From b6abb022f68dfcdf06b2e9bd302423154b7b559b Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 19 Jan 2021 16:37:02 -0800 Subject: [PATCH 001/203] Add initial implementation of impersonation proxy. Signed-off-by: Margo Crawford Signed-off-by: Matt Moyer --- cmd/debug-make-impersonation-token/main.go | 41 +++ deploy/concierge/rbac.yaml | 3 + .../concierge/impersonator/impersonator.go | 156 +++++++++++ .../impersonator/impersonator_test.go | 259 ++++++++++++++++++ internal/concierge/server/server.go | 35 +++ proxy-kubeconfig.yaml | 16 ++ 6 files changed, 510 insertions(+) create mode 100644 cmd/debug-make-impersonation-token/main.go create mode 100644 internal/concierge/impersonator/impersonator.go create mode 100644 internal/concierge/impersonator/impersonator_test.go create mode 100644 proxy-kubeconfig.yaml diff --git a/cmd/debug-make-impersonation-token/main.go b/cmd/debug-make-impersonation-token/main.go new file mode 100644 index 00000000..b33f1581 --- /dev/null +++ b/cmd/debug-make-impersonation-token/main.go @@ -0,0 +1,41 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" +) + +func main() { + reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: os.Getenv("PINNIPED_TEST_CONCIERGE_NAMESPACE"), + }, + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginv1alpha1.GroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: os.Getenv("PINNIPED_TEST_USER_TOKEN"), + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, + Kind: os.Getenv("PINNIPED_AUTHENTICATOR_KIND"), + Name: os.Getenv("PINNIPED_AUTHENTICATOR_NAME"), + }, + }, + }) + if err != nil { + panic(err) + } + fmt.Println(base64.RawURLEncoding.EncodeToString(reqJSON)) +} diff --git a/deploy/concierge/rbac.yaml b/deploy/concierge/rbac.yaml index 8df8734b..b81fc14e 100644 --- a/deploy/concierge/rbac.yaml +++ b/deploy/concierge/rbac.yaml @@ -28,6 +28,9 @@ rules: resources: [ securitycontextconstraints ] verbs: [ use ] resourceNames: [ nonroot ] + - apiGroups: [""] + resources: ["users", "groups"] + verbs: ["impersonate"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go new file mode 100644 index 00000000..12f9e1b1 --- /dev/null +++ b/internal/concierge/impersonator/impersonator.go @@ -0,0 +1,156 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httputil" + "net/url" + "strings" + + "github.com/go-logr/logr" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/client-go/rest" + "k8s.io/client-go/transport" + + "go.pinniped.dev/generated/1.20/apis/concierge/login" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/controller/authenticator/authncache" +) + +// allowedHeaders are the set of HTTP headers that are allowed to be forwarded through the impersonation proxy. +//nolint: gochecknoglobals +var allowedHeaders = []string{ + "Accept", + "Accept-Encoding", + "User-Agent", + "Connection", + "Upgrade", +} + +type Proxy struct { + cache *authncache.Cache + proxy *httputil.ReverseProxy + log logr.Logger +} + +func New(cache *authncache.Cache, log logr.Logger) (*Proxy, error) { + return newInternal(cache, log, rest.InClusterConfig) +} + +func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*rest.Config, error)) (*Proxy, error) { + kubeconfig, err := getConfig() + if err != nil { + return nil, fmt.Errorf("could not get in-cluster config: %w", err) + } + + serverURL, err := url.Parse(kubeconfig.Host) + if err != nil { + return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err) + } + + kubeTransportConfig, err := kubeconfig.TransportConfig() + if err != nil { + return nil, fmt.Errorf("could not get in-cluster transport config: %w", err) + } + kubeTransportConfig.TLS.NextProtos = []string{"http/1.1"} + + kubeRoundTripper, err := transport.New(kubeTransportConfig) + if err != nil { + return nil, fmt.Errorf("could not get in-cluster transport: %w", err) + } + + proxy := httputil.NewSingleHostReverseProxy(serverURL) + proxy.Transport = kubeRoundTripper + + return &Proxy{ + cache: cache, + proxy: proxy, + log: log, + }, nil +} + +func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + log := p.log.WithValues( + "url", r.URL.String(), + "method", r.Method, + ) + + tokenCredentialReq, err := extractToken(r) + if err != nil { + log.Error(err, "invalid token encoding") + http.Error(w, "invalid token encoding", http.StatusBadRequest) + return + } + log = log.WithValues( + "authenticator", tokenCredentialReq.Spec.Authenticator, + "authenticatorNamespace", tokenCredentialReq.Namespace, + ) + + userInfo, err := p.cache.AuthenticateTokenCredentialRequest(r.Context(), tokenCredentialReq) + if err != nil { + log.Error(err, "received invalid token") + http.Error(w, "invalid token", http.StatusUnauthorized) + return + } + + if userInfo == nil { + log.Info("received token that did not authenticate") + http.Error(w, "not authenticated", http.StatusUnauthorized) + return + } + log = log.WithValues( + "user", userInfo.GetName(), + "groups", userInfo.GetGroups(), + ) + + newHeaders := getProxyHeaders(userInfo, r.Header) + r.Header = newHeaders + + log.Info("proxying authenticated request") + p.proxy.ServeHTTP(w, r) +} + +func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header { + newHeaders := http.Header{} + newHeaders.Set("Impersonate-User", userInfo.GetName()) + for _, group := range userInfo.GetGroups() { + newHeaders.Add("Impersonate-Group", group) + } + for _, header := range allowedHeaders { + values := requestHeaders.Values(header) + for i := range values { + newHeaders.Add(header, values[i]) + } + } + return newHeaders +} + +func extractToken(req *http.Request) (*login.TokenCredentialRequest, error) { + authHeader := req.Header.Get("Authorization") + if authHeader == "" { + return nil, fmt.Errorf("missing authorization header") + } + if !strings.HasPrefix(authHeader, "Bearer ") { + return nil, fmt.Errorf("authorization header must be of type Bearer") + } + encoded := strings.TrimPrefix(authHeader, "Bearer ") + tokenCredentialRequestJSON, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return nil, fmt.Errorf("invalid base64 in encoded bearer token: %w", err) + } + + var v1alpha1Req loginv1alpha1.TokenCredentialRequest + if err := json.Unmarshal(tokenCredentialRequestJSON, &v1alpha1Req); err != nil { + return nil, fmt.Errorf("invalid TokenCredentialRequest encoded in bearer token: %w", err) + } + var internalReq login.TokenCredentialRequest + if err := loginv1alpha1.Convert_v1alpha1_TokenCredentialRequest_To_login_TokenCredentialRequest(&v1alpha1Req, &internalReq, nil); err != nil { + return nil, fmt.Errorf("failed to convert v1alpha1 TokenCredentialRequest to internal version: %w", err) + } + return &internalReq, nil +} diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go new file mode 100644 index 00000000..9a8d1508 --- /dev/null +++ b/internal/concierge/impersonator/impersonator_test.go @@ -0,0 +1,259 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd/api" + + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/controller/authenticator/authncache" + "go.pinniped.dev/internal/mocks/mocktokenauthenticator" + "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/internal/testutil/testlogger" +) + +func TestImpersonator(t *testing.T) { + validURL, _ := url.Parse("http://pinniped.dev/blah") + testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + // Expect that the request is authenticated based on the kubeconfig credential. + if r.Header.Get("Authorization") != "Bearer some-service-account-token" { + http.Error(w, "expected to see service account token", http.StatusForbidden) + return + } + // Fail if we see the malicious header passed through the proxy (it's not on the allowlist). + if r.Header.Get("Malicious-Header") != "" { + http.Error(w, "didn't expect to see malicious header", http.StatusForbidden) + return + } + // Expect to see the user agent header passed through. + if r.Header.Get("User-Agent") != "test-user-agent" { + http.Error(w, "got unexpected user agent header", http.StatusBadRequest) + return + } + _, _ = w.Write([]byte("successful proxied response")) + }) + testServerKubeconfig := rest.Config{ + Host: testServerURL, + BearerToken: "some-service-account-token", + TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testServerCA)}, + } + + tests := []struct { + name string + getKubeconfig func() (*rest.Config, error) + wantCreationErr string + request *http.Request + wantHTTPBody string + wantHTTPStatus int + wantLogs []string + expectMockToken func(*testing.T, *mocktokenauthenticator.MockTokenMockRecorder) + }{ + { + name: "fail to get in-cluster config", + getKubeconfig: func() (*rest.Config, error) { + return nil, fmt.Errorf("some kubernetes error") + }, + wantCreationErr: "could not get in-cluster config: some kubernetes error", + }, + { + name: "invalid kubeconfig host", + getKubeconfig: func() (*rest.Config, error) { + return &rest.Config{Host: ":"}, nil + }, + wantCreationErr: "could not parse host URL from in-cluster config: parse \":\": missing protocol scheme", + }, + { + name: "invalid transport config", + getKubeconfig: func() (*rest.Config, error) { + return &rest.Config{ + Host: "pinniped.dev/blah", + ExecProvider: &api.ExecConfig{}, + AuthProvider: &api.AuthProviderConfig{}, + }, nil + }, + wantCreationErr: "could not get in-cluster transport config: execProvider and authProvider cannot be used in combination", + }, + { + name: "fail to get transport from config", + getKubeconfig: func() (*rest.Config, error) { + return &rest.Config{ + Host: "pinniped.dev/blah", + BearerToken: "test-bearer-token", + Transport: http.DefaultTransport, + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, + }, nil + }, + wantCreationErr: "could not get in-cluster transport: using a custom transport with TLS certificate options or the insecure flag is not allowed", + }, + { + name: "missing authorization header", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{}, + URL: validURL, + }, + wantHTTPBody: "invalid token encoding\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"missing authorization header\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "authorization header missing bearer prefix", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{"Authorization": {makeTestTokenRequest("foo", "authenticator-one", "test-token")}}, + URL: validURL, + }, + wantHTTPBody: "invalid token encoding\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"authorization header must be of type Bearer\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "token is not base64 encoded", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{"Authorization": {"Bearer !!!"}}, + URL: validURL, + }, + wantHTTPBody: "invalid token encoding\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"invalid base64 in encoded bearer token: illegal base64 data at input byte 0\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "base64 encoded token is not valid json", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{"Authorization": {"Bearer abc"}}, + URL: validURL, + }, + wantHTTPBody: "invalid token encoding\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"invalid TokenCredentialRequest encoded in bearer token: invalid character 'i' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "token could not be authenticated", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("", "", "")}}, + URL: validURL, + }, + wantHTTPBody: "invalid token\n", + wantHTTPStatus: http.StatusUnauthorized, + wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"\"} \"authenticatorNamespace\"=\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "token authenticates as nil", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}}, + URL: validURL, + }, + expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { + recorder.AuthenticateToken(gomock.Any(), "test-token").Return(nil, false, nil) + }, + wantHTTPBody: "not authenticated\n", + wantHTTPStatus: http.StatusUnauthorized, + wantLogs: []string{"\"level\"=0 \"msg\"=\"received token that did not authenticate\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"authenticatorNamespace\"=\"foo\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + // happy path + { + name: "token validates", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: &http.Request{ + Method: "GET", + Header: map[string][]string{ + "Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}, + "Malicious-Header": {"test-header-value-1"}, + "User-Agent": {"test-user-agent"}, + }, + URL: validURL, + }, + expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { + userInfo := user.DefaultInfo{Name: "test-user", Groups: []string{"test-group-1", "test-group-2"}} + response := &authenticator.Response{User: &userInfo} + recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"authenticatorNamespace\"=\"foo\" \"groups\"=[\"test-group-1\",\"test-group-2\"] \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"user\"=\"test-user\""}, + }, + } + + for _, tt := range tests { + tt := tt + testLog := testlogger.New(t) + t.Run(tt.name, func(t *testing.T) { + // stole this from cache_test, hopefully it is sufficient + cacheWithMockAuthenticator := authncache.New() + ctrl := gomock.NewController(t) + defer ctrl.Finish() + key := authncache.Key{Namespace: "foo", Name: "authenticator-one"} + mockToken := mocktokenauthenticator.NewMockToken(ctrl) + cacheWithMockAuthenticator.Store(key, mockToken) + + if tt.expectMockToken != nil { + tt.expectMockToken(t, mockToken.EXPECT()) + } + + proxy, err := newInternal(cacheWithMockAuthenticator, testLog, tt.getKubeconfig) + if tt.wantCreationErr != "" { + require.EqualError(t, err, tt.wantCreationErr) + return + } + require.NoError(t, err) + require.NotNil(t, proxy) + w := httptest.NewRecorder() + proxy.ServeHTTP(w, tt.request) + if tt.wantHTTPStatus != 0 { + require.Equal(t, tt.wantHTTPStatus, w.Code) + } + if tt.wantHTTPBody != "" { + require.Equal(t, tt.wantHTTPBody, w.Body.String()) + } + if tt.wantLogs != nil { + require.Equal(t, tt.wantLogs, testLog.Lines()) + } + }) + } +} + +func makeTestTokenRequest(namespace string, name string, token string) string { + reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginv1alpha1.GroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: token, + Authenticator: corev1.TypedLocalObjectReference{Name: name}, + }, + }) + if err != nil { + panic(err) + } + return base64.RawURLEncoding.EncodeToString(reqJSON) +} diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index 9c1a0a40..9b22ba28 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -6,8 +6,11 @@ package server import ( "context" + "crypto/tls" + "crypto/x509/pkix" "fmt" "io" + "net/http" "time" "github.com/spf13/cobra" @@ -18,11 +21,15 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" loginapi "go.pinniped.dev/generated/1.20/apis/concierge/login" loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/certauthority/dynamiccertauthority" "go.pinniped.dev/internal/concierge/apiserver" + "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/internal/config/concierge" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/controllermanager" @@ -162,6 +169,34 @@ func (a *App) runServer(ctx context.Context) error { return fmt.Errorf("could not create aggregated API server: %w", err) } + // run proxy handler + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) + if err != nil { + return fmt.Errorf("could not create impersonation CA: %w", err) + } + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, []string{"impersonation-proxy"}, nil, 24*time.Hour) + if err != nil { + return fmt.Errorf("could not create impersonation cert: %w", err) + } + impersonationProxy, err := impersonator.New(authenticators, klogr.New().WithName("impersonation-proxy")) + if err != nil { + return fmt.Errorf("could not create impersonation proxy: %w", err) + } + + impersonationProxyServer := http.Server{ + Addr: "0.0.0.0:8444", + Handler: impersonationProxy, + TLSConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{*impersonationCert}, + }, + } + go func() { + if err := impersonationProxyServer.ListenAndServeTLS("", ""); err != nil { + klog.ErrorS(err, "could not serve impersonation proxy") + } + }() + // Run the server. Its post-start hook will start the controllers. return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } diff --git a/proxy-kubeconfig.yaml b/proxy-kubeconfig.yaml new file mode 100644 index 00000000..37c60516 --- /dev/null +++ b/proxy-kubeconfig.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +clusters: + - cluster: + server: https://127.0.0.1:8444 + insecure-skip-tls-verify: true + name: kind-pinniped +contexts: + - context: + cluster: kind-pinniped + user: kind-pinniped + name: kind-pinniped +current-context: kind-pinniped +kind: Config +preferences: {} +users: + - name: kind-pinniped From 1299231a487af8308249961be391d4b4cffdffdc Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 22 Jan 2021 12:00:27 -0600 Subject: [PATCH 002/203] Add integration test for impersonation proxy. Signed-off-by: Margo Crawford --- deploy/concierge/deployment.yaml | 14 +++ .../concierge_impersonation_proxy_test.go | 94 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 test/integration/concierge_impersonation_proxy_test.go diff --git a/deploy/concierge/deployment.yaml b/deploy/concierge/deployment.yaml index 3d7f4243..8a59030b 100644 --- a/deploy/concierge/deployment.yaml +++ b/deploy/concierge/deployment.yaml @@ -189,6 +189,20 @@ spec: port: 443 targetPort: 8443 --- +apiVersion: v1 +kind: Service +metadata: + name: #@ defaultResourceNameWithSuffix("proxy") + namespace: #@ namespace() + labels: #@ labels() +spec: + type: ClusterIP + selector: #@ defaultLabel() + ports: + - protocol: TCP + port: 443 + targetPort: 8444 +--- apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go new file mode 100644 index 00000000..f3e81631 --- /dev/null +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -0,0 +1,94 @@ +// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package integration + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/test/library" +) + +// Smoke test to see if the kubeconfig works and the cluster is reachable. +func TestImpersonationProxy(t *testing.T) { + env := library.IntegrationEnv(t) + if env.Proxy == "" { + t.Skip("this test can only run in environments with the in-cluster proxy right now") + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + defer cancel() + + // Create a client using the admin kubeconfig. + adminClient := library.NewClientset(t) + + // Create a WebhookAuthenticator. + authenticator := library.CreateTestWebhookAuthenticator(ctx, t) + + // Find the address of the ClusterIP service. + proxyServiceURL := fmt.Sprintf("https://%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) + t.Logf("making kubeconfig that points to %q", proxyServiceURL) + + kubeconfig := &rest.Config{ + Host: proxyServiceURL, + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, + BearerToken: makeImpersonationTestToken(t, authenticator), + Proxy: func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + }, + } + + clientset, err := kubernetes.NewForConfig(kubeconfig) + require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") + + t.Run( + "access as user", + library.AccessAsUserTest(ctx, adminClient, env.TestUser.ExpectedUsername, clientset), + ) + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run( + "access as group "+group, + library.AccessAsGroupTest(ctx, adminClient, group, clientset), + ) + } +} + +func makeImpersonationTestToken(t *testing.T, authenticator corev1.TypedLocalObjectReference) string { + t.Helper() + + env := library.IntegrationEnv(t) + reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: env.ConciergeNamespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginv1alpha1.GroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: authenticator, + }, + }) + require.NoError(t, err) + return base64.RawURLEncoding.EncodeToString(reqJSON) +} From 64aff7b98370b6c140f30d4f43d496ebef376080 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 22 Jan 2021 12:12:12 -0600 Subject: [PATCH 003/203] Only log user ID, not user name/groups. Signed-off-by: Margo Crawford --- internal/concierge/impersonator/impersonator.go | 5 +---- internal/concierge/impersonator/impersonator_test.go | 8 ++++++-- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 12f9e1b1..8c8cfd1c 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -103,10 +103,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "not authenticated", http.StatusUnauthorized) return } - log = log.WithValues( - "user", userInfo.GetName(), - "groups", userInfo.GetGroups(), - ) + log = log.WithValues("userID", userInfo.GetUID()) newHeaders := getProxyHeaders(userInfo, r.Header) r.Header = newHeaders diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 9a8d1508..5ae19e02 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -190,13 +190,17 @@ func TestImpersonator(t *testing.T) { URL: validURL, }, expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { - userInfo := user.DefaultInfo{Name: "test-user", Groups: []string{"test-group-1", "test-group-2"}} + userInfo := user.DefaultInfo{ + Name: "test-user", + Groups: []string{"test-group-1", "test-group-2"}, + UID: "test-uid", + } response := &authenticator.Response{User: &userInfo} recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, - wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"authenticatorNamespace\"=\"foo\" \"groups\"=[\"test-group-1\",\"test-group-2\"] \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"user\"=\"test-user\""}, + wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"authenticatorNamespace\"=\"foo\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, }, } From 07b7b743b40d0e7f2d6bc04692c66ea12fb0c00e Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 26 Jan 2021 11:39:42 -0800 Subject: [PATCH 004/203] Impersonation proxy cli arguments --- cmd/pinniped/cmd/kubeconfig.go | 37 +++++++++++++---- cmd/pinniped/cmd/kubeconfig_test.go | 64 +++++++++++++++++++++++++++++ cmd/pinniped/cmd/login_oidc.go | 51 ++++++++++++++++++++++- cmd/pinniped/cmd/login_oidc_test.go | 1 + 4 files changed, 145 insertions(+), 8 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 1b108c02..8959370f 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -73,11 +73,14 @@ type getKubeconfigOIDCParams struct { } type getKubeconfigConciergeParams struct { - disabled bool - namespace string - authenticatorName string - authenticatorType string - apiGroupSuffix string + disabled bool + namespace string + authenticatorName string + authenticatorType string + apiGroupSuffix string + caBundleData string + endpoint string + useImpersonationProxy bool } type getKubeconfigParams struct { @@ -110,6 +113,10 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", "pinniped.dev", "Concierge API group suffix") + f.StringVar(&flags.concierge.caBundleData, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") + f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") + f.BoolVar(&flags.concierge.useImpersonationProxy, "use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") + f.StringVar(&flags.oidc.issuer, "oidc-issuer", "", "OpenID Connect issuer URL (default: autodiscover)") f.StringVar(&flags.oidc.clientID, "oidc-client-id", "pinniped-cli", "OpenID Connect client ID (default: autodiscover)") f.Uint16Var(&flags.oidc.listenPort, "oidc-listen-port", 0, "TCP port for localhost listener (authorization code flow only)") @@ -162,6 +169,10 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if err != nil { return fmt.Errorf("could not load --kubeconfig/--kubeconfig-context: %w", err) } + if flags.concierge.useImpersonationProxy { + cluster.CertificateAuthorityData = []byte(flags.concierge.caBundleData) + cluster.Server = flags.concierge.endpoint + } clientset, err := deps.getClientset(clientConfig, flags.concierge.apiGroupSuffix) if err != nil { return fmt.Errorf("could not configure Kubernetes client: %w", err) @@ -266,6 +277,13 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, } } + if flags.concierge.endpoint == "" { + flags.concierge.endpoint = v1Cluster.Server + } + if flags.concierge.caBundleData == "" { + flags.concierge.caBundleData = base64.StdEncoding.EncodeToString(v1Cluster.CertificateAuthorityData) + } + // Append the flags to configure the Concierge credential exchange at runtime. execConfig.Args = append(execConfig.Args, "--enable-concierge", @@ -273,9 +291,14 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, "--concierge-namespace="+flags.concierge.namespace, "--concierge-authenticator-name="+flags.concierge.authenticatorName, "--concierge-authenticator-type="+flags.concierge.authenticatorType, - "--concierge-endpoint="+v1Cluster.Server, - "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(v1Cluster.CertificateAuthorityData), + "--concierge-endpoint="+flags.concierge.endpoint, + "--concierge-ca-bundle-data="+flags.concierge.caBundleData, ) + if flags.concierge.useImpersonationProxy { + execConfig.Args = append(execConfig.Args, + "--use-impersonation-proxy", + ) + } return nil } diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index cbae7146..07e0d8c1 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -61,6 +61,8 @@ func TestGetKubeconfig(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name (default: autodiscover) --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) + --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge + --concierge-endpoint string API base for the Pinniped concierge endpoint --concierge-namespace string Namespace in which the concierge was installed (default "pinniped-concierge") -h, --help help for kubeconfig --kubeconfig string Path to kubeconfig file @@ -76,6 +78,7 @@ func TestGetKubeconfig(t *testing.T) { --oidc-skip-browser During OpenID Connect login, skip opening the browser (just print the URL) --static-token string Instead of doing an OIDC-based login, specify a static token --static-token-env string Instead of doing an OIDC-based login, read a static token from the environment + --use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy `), }, { @@ -506,6 +509,67 @@ func TestGetKubeconfig(t *testing.T) { `, base64.StdEncoding.EncodeToString(testCA.Bundle())), wantAPIGroupSuffix: "tuna.io", }, + { + name: "configure impersonation proxy with autodetected JWT authenticator", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-ca-bundle-data", "blah", // TODO make this more realistic, maybe do some validation? + "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", + "--use-impersonation-proxy", + }, + conciergeObjects: []runtime.Object{ + &conciergev1alpha1.JWTAuthenticator{ + ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator", Namespace: "pinniped-concierge"}, + Spec: conciergev1alpha1.JWTAuthenticatorSpec{ + Issuer: "https://example.com/issuer", + Audience: "test-audience", + TLS: &conciergev1alpha1.TLSSpec{ + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testCA.Bundle()), + }, + }, + }, + }, + wantStdout: here.Docf(` + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: YmxhaA== + server: https://impersonation-proxy-endpoint.test + name: pinniped + contexts: + - context: + cluster: pinniped + user: pinniped + name: pinniped + current-context: pinniped + kind: Config + preferences: {} + users: + - name: pinniped + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + args: + - login + - oidc + - --enable-concierge + - --concierge-api-group-suffix=pinniped.dev + - --concierge-namespace=pinniped-concierge + - --concierge-authenticator-name=test-authenticator + - --concierge-authenticator-type=jwt + - --concierge-endpoint=https://impersonation-proxy-endpoint.test + - --concierge-ca-bundle-data=blah + - --use-impersonation-proxy + - --issuer=https://example.com/issuer + - --client-id=pinniped-cli + - --scopes=offline_access,openid,pinniped:request-audience + - --ca-bundle-data=%s + - --request-audience=test-audience + command: '.../path/to/pinniped' + env: [] + provideClusterInfo: true + `, base64.StdEncoding.EncodeToString(testCA.Bundle())), + }, } for _, tt := range tests { tt := tt diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 7b5f53f3..9d986530 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -16,6 +16,11 @@ import ( "path/filepath" "time" + corev1 "k8s.io/api/core/v1" + + authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "github.com/coreos/go-oidc/v3/oidc" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -65,6 +70,7 @@ type oidcLoginFlags struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string + useImpersonationProxy bool } func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { @@ -94,6 +100,7 @@ func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", "pinniped.dev", "Concierge API group suffix") + cmd.Flags().BoolVar(&flags.useImpersonationProxy, "use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") mustMarkHidden(&cmd, "debug-session-cache") mustMarkRequired(&cmd, "issuer") @@ -171,12 +178,21 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - if concierge != nil { + // do a credential exchange request, unless impersonation proxy is configured + if concierge != nil && !flags.useImpersonationProxy { cred, err = deps.exchangeToken(ctx, concierge, token.IDToken.Token) if err != nil { return fmt.Errorf("could not complete concierge credential exchange: %w", err) } } + if concierge != nil && flags.useImpersonationProxy { + // TODO add the right header??? + req, err := execCredentialForImpersonationProxy(token, flags) + if err != nil { + return err + } + return json.NewEncoder(cmd.OutOrStdout()).Encode(req) + } return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) } func makeClient(caBundlePaths []string, caBundleData []string) (*http.Client, error) { @@ -238,3 +254,36 @@ func mustGetConfigDir() string { } return filepath.Join(home, ".config", xdgAppName) } + +func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLoginFlags) (*clientauthv1beta1.ExecCredential, error) { + reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: flags.conciergeNamespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginv1alpha1.GroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: token.AccessToken.Token, // TODO + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, + Kind: os.Getenv(flags.conciergeAuthenticatorType), + Name: os.Getenv(flags.conciergeAuthenticatorName), + }, + }, + }) + if err != nil { + return nil, err + } + encodedToken := base64.RawURLEncoding.EncodeToString(reqJSON) + return &clientauthv1beta1.ExecCredential{ + TypeMeta: metav1.TypeMeta{ + Kind: "ExecCredential", + APIVersion: "client.authentication.k8s.io/v1beta1", + }, + Status: &clientauthv1beta1.ExecCredentialStatus{ + Token: encodedToken, + }, + }, nil +} diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index 41607e4e..70c81924 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -74,6 +74,7 @@ func TestLoginOIDCCommand(t *testing.T) { --scopes strings OIDC scopes to request during login (default [offline_access,openid,pinniped:request-audience]) --session-cache string Path to session cache file (default "` + cfgDir + `/sessions.yaml") --skip-browser Skip opening the browser (just print the URL) + --use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy `), }, { From 170b86d0c6653288cdeee62264fc30619de6c3f5 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 26 Jan 2021 13:34:09 -0800 Subject: [PATCH 005/203] Add happy path test for login oidc --- cmd/pinniped/cmd/login_oidc.go | 25 +++++++++++++---- cmd/pinniped/cmd/login_oidc_test.go | 42 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 9d986530..831e4f01 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "time" corev1 "k8s.io/api/core/v1" @@ -256,6 +257,16 @@ func mustGetConfigDir() string { } func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLoginFlags) (*clientauthv1beta1.ExecCredential, error) { + // TODO maybe de-dup this with conciergeclient.go + var kind string + switch strings.ToLower(flags.conciergeAuthenticatorType) { + case "webhook": + kind = "WebhookAuthenticator" + case "jwt": + kind = "JWTAuthenticator" + default: + return nil, fmt.Errorf(`invalid authenticator type: %q, supported values are "webhook" and "jwt"`, kind) + } reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: flags.conciergeNamespace, @@ -265,11 +276,11 @@ func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLogin APIVersion: loginv1alpha1.GroupName + "/v1alpha1", }, Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: token.AccessToken.Token, // TODO + Token: token.IDToken.Token, // TODO Authenticator: corev1.TypedLocalObjectReference{ APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, - Kind: os.Getenv(flags.conciergeAuthenticatorType), - Name: os.Getenv(flags.conciergeAuthenticatorName), + Kind: kind, + Name: flags.conciergeAuthenticatorName, }, }, }) @@ -277,7 +288,7 @@ func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLogin return nil, err } encodedToken := base64.RawURLEncoding.EncodeToString(reqJSON) - return &clientauthv1beta1.ExecCredential{ + cred := &clientauthv1beta1.ExecCredential{ TypeMeta: metav1.TypeMeta{ Kind: "ExecCredential", APIVersion: "client.authentication.k8s.io/v1beta1", @@ -285,5 +296,9 @@ func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLogin Status: &clientauthv1beta1.ExecCredentialStatus{ Token: encodedToken, }, - }, nil + } + if !token.IDToken.Expiry.IsZero() { + cred.Status.ExpirationTimestamp = &token.IDToken.Expiry + } + return cred, nil } diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index 70c81924..ba02bd8d 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -8,12 +8,18 @@ import ( "context" "crypto/x509/pkix" "encoding/base64" + "encoding/json" "fmt" "io/ioutil" "path/filepath" "testing" "time" + corev1 "k8s.io/api/core/v1" + + authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientauthv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" @@ -198,6 +204,21 @@ func TestLoginOIDCCommand(t *testing.T) { wantOptionsCount: 7, wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"token":"exchanged-token"}}` + "\n", }, + { + name: "success with impersonation proxy", + args: []string{ + "--client-id", "test-client-id", + "--issuer", "test-issuer", + "--enable-concierge", + "--use-impersonation-proxy", + "--concierge-authenticator-type", "webhook", + "--concierge-authenticator-name", "test-authenticator", + "--concierge-endpoint", "https://127.0.0.1:1234/", + "--concierge-ca-bundle-data", base64.StdEncoding.EncodeToString(testCA.Bundle()), + }, + wantOptionsCount: 3, + wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"expirationTimestamp":"3020-10-12T13:14:15Z","token":"` + impersonationProxyToken("test-id-token") + `"}}` + "\n", + }, } for _, tt := range tests { tt := tt @@ -254,3 +275,24 @@ func TestLoginOIDCCommand(t *testing.T) { }) } } + +func impersonationProxyToken(token string) string { + reqJSON, _ := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "pinniped-concierge", + }, + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginv1alpha1.GroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: token, + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, + Kind: "WebhookAuthenticator", + Name: "test-authenticator", + }, + }, + }) + return base64.RawURLEncoding.EncodeToString(reqJSON) +} From 2f891b4bfbfdb1699617a3e8663a546451dd3026 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 26 Jan 2021 16:08:27 -0800 Subject: [PATCH 006/203] Add --concierge-use-impersonation-proxy to static login - also renamed --use-impersonation-proxy to --concierge-use-impersonation-proxy --- cmd/pinniped/cmd/kubeconfig.go | 4 +- cmd/pinniped/cmd/kubeconfig_test.go | 6 +-- cmd/pinniped/cmd/login_oidc.go | 5 ++- cmd/pinniped/cmd/login_oidc_test.go | 8 ++-- cmd/pinniped/cmd/login_static.go | 64 ++++++++++++++++++++++++++- cmd/pinniped/cmd/login_static_test.go | 13 ++++++ 6 files changed, 88 insertions(+), 12 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 8959370f..a4e74883 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -115,7 +115,7 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.concierge.caBundleData, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") - f.BoolVar(&flags.concierge.useImpersonationProxy, "use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") + f.BoolVar(&flags.concierge.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") f.StringVar(&flags.oidc.issuer, "oidc-issuer", "", "OpenID Connect issuer URL (default: autodiscover)") f.StringVar(&flags.oidc.clientID, "oidc-client-id", "pinniped-cli", "OpenID Connect client ID (default: autodiscover)") @@ -296,7 +296,7 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, ) if flags.concierge.useImpersonationProxy { execConfig.Args = append(execConfig.Args, - "--use-impersonation-proxy", + "--concierge-use-impersonation-proxy", ) } return nil diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 07e0d8c1..9c588adb 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -64,6 +64,7 @@ func TestGetKubeconfig(t *testing.T) { --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge --concierge-endpoint string API base for the Pinniped concierge endpoint --concierge-namespace string Namespace in which the concierge was installed (default "pinniped-concierge") + --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy -h, --help help for kubeconfig --kubeconfig string Path to kubeconfig file --kubeconfig-context string Kubeconfig context name (default: current active context) @@ -78,7 +79,6 @@ func TestGetKubeconfig(t *testing.T) { --oidc-skip-browser During OpenID Connect login, skip opening the browser (just print the URL) --static-token string Instead of doing an OIDC-based login, specify a static token --static-token-env string Instead of doing an OIDC-based login, read a static token from the environment - --use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy `), }, { @@ -515,7 +515,7 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", "--concierge-ca-bundle-data", "blah", // TODO make this more realistic, maybe do some validation? "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", - "--use-impersonation-proxy", + "--concierge-use-impersonation-proxy", }, conciergeObjects: []runtime.Object{ &conciergev1alpha1.JWTAuthenticator{ @@ -559,7 +559,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=jwt - --concierge-endpoint=https://impersonation-proxy-endpoint.test - --concierge-ca-bundle-data=blah - - --use-impersonation-proxy + - --concierge-use-impersonation-proxy - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 831e4f01..999fd8a7 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -101,7 +101,7 @@ func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", "pinniped.dev", "Concierge API group suffix") - cmd.Flags().BoolVar(&flags.useImpersonationProxy, "use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") + cmd.Flags().BoolVar(&flags.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") mustMarkHidden(&cmd, "debug-session-cache") mustMarkRequired(&cmd, "issuer") @@ -187,7 +187,8 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin } } if concierge != nil && flags.useImpersonationProxy { - // TODO add the right header??? + // Put the token into a TokenCredentialRequest + // put the TokenCredentialRequest in an ExecCredential req, err := execCredentialForImpersonationProxy(token, flags) if err != nil { return err diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index ba02bd8d..883cc31f 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -72,6 +72,7 @@ func TestLoginOIDCCommand(t *testing.T) { --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge --concierge-endpoint string API base for the Pinniped concierge endpoint --concierge-namespace string Namespace in which the concierge was installed (default "pinniped-concierge") + --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy --enable-concierge Exchange the OIDC ID token with the Pinniped concierge during login -h, --help help for oidc --issuer string OpenID Connect issuer URL @@ -80,7 +81,6 @@ func TestLoginOIDCCommand(t *testing.T) { --scopes strings OIDC scopes to request during login (default [offline_access,openid,pinniped:request-audience]) --session-cache string Path to session cache file (default "` + cfgDir + `/sessions.yaml") --skip-browser Skip opening the browser (just print the URL) - --use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy `), }, { @@ -210,14 +210,14 @@ func TestLoginOIDCCommand(t *testing.T) { "--client-id", "test-client-id", "--issuer", "test-issuer", "--enable-concierge", - "--use-impersonation-proxy", + "--concierge-use-impersonation-proxy", "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", "--concierge-endpoint", "https://127.0.0.1:1234/", "--concierge-ca-bundle-data", base64.StdEncoding.EncodeToString(testCA.Bundle()), }, wantOptionsCount: 3, - wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"expirationTimestamp":"3020-10-12T13:14:15Z","token":"` + impersonationProxyToken("test-id-token") + `"}}` + "\n", + wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"expirationTimestamp":"3020-10-12T13:14:15Z","token":"` + impersonationProxyTestToken("test-id-token") + `"}}` + "\n", }, } for _, tt := range tests { @@ -276,7 +276,7 @@ func TestLoginOIDCCommand(t *testing.T) { } } -func impersonationProxyToken(token string) string { +func impersonationProxyTestToken(token string) string { reqJSON, _ := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ ObjectMeta: metav1.ObjectMeta{ Namespace: "pinniped-concierge", diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 52fcf095..7477dfd4 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -5,15 +5,22 @@ package cmd import ( "context" + "encoding/base64" "encoding/json" "fmt" "io" "os" + "strings" "time" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/spf13/cobra" clientauthv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" + authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" "go.pinniped.dev/pkg/conciergeclient" "go.pinniped.dev/pkg/oidcclient/oidctypes" ) @@ -47,6 +54,7 @@ type staticLoginParams struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string + useImpersonationProxy bool } func staticLoginCommand(deps staticLoginDeps) *cobra.Command { @@ -68,6 +76,7 @@ func staticLoginCommand(deps staticLoginDeps) *cobra.Command { cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", "pinniped.dev", "Concierge API group suffix") + cmd.Flags().BoolVar(&flags.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") cmd.RunE = func(cmd *cobra.Command, args []string) error { return runStaticLogin(cmd.OutOrStdout(), deps, flags) } return &cmd } @@ -109,7 +118,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams cred := tokenCredential(&oidctypes.Token{IDToken: &oidctypes.IDToken{Token: token}}) // Exchange that token with the concierge, if configured. - if concierge != nil { + if concierge != nil && !flags.useImpersonationProxy { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -119,5 +128,58 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams return fmt.Errorf("could not complete concierge credential exchange: %w", err) } } + if concierge != nil && flags.useImpersonationProxy { + // Put the token into a TokenCredentialRequest + // put the TokenCredentialRequest in an ExecCredential + req, err := execCredentialForImpersonationProxyStatic(token, flags) + if err != nil { + return err + } + return json.NewEncoder(out).Encode(req) + } return json.NewEncoder(out).Encode(cred) } + +func execCredentialForImpersonationProxyStatic(token string, flags staticLoginParams) (*clientauthv1beta1.ExecCredential, error) { + // TODO maybe de-dup this with conciergeclient.go + var kind string + switch strings.ToLower(flags.conciergeAuthenticatorType) { + case "webhook": + kind = "WebhookAuthenticator" + case "jwt": + kind = "JWTAuthenticator" + default: + return nil, fmt.Errorf(`invalid authenticator type: %q, supported values are "webhook" and "jwt"`, kind) + } + reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: flags.conciergeNamespace, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginv1alpha1.GroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: token, // TODO + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, + Kind: kind, + Name: flags.conciergeAuthenticatorName, + }, + }, + }) + if err != nil { + return nil, err + } + encodedToken := base64.RawURLEncoding.EncodeToString(reqJSON) + cred := &clientauthv1beta1.ExecCredential{ + TypeMeta: metav1.TypeMeta{ + Kind: "ExecCredential", + APIVersion: "client.authentication.k8s.io/v1beta1", + }, + Status: &clientauthv1beta1.ExecCredentialStatus{ + Token: encodedToken, + }, + } + return cred, nil +} diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index 0152dbd0..a19f0909 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -57,6 +57,7 @@ func TestLoginStaticCommand(t *testing.T) { --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge --concierge-endpoint string API base for the Pinniped concierge endpoint --concierge-namespace string Namespace in which the concierge was installed (default "pinniped-concierge") + --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy --enable-concierge Exchange the token with the Pinniped concierge during login -h, --help help for static --token string Static token to present during login @@ -153,6 +154,18 @@ func TestLoginStaticCommand(t *testing.T) { }, wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"token":"test-token"}}` + "\n", }, + { + name: "impersonation proxy success", + args: []string{ + "--enable-concierge", + "--concierge-use-impersonation-proxy", + "--token", "test-token", + "--concierge-endpoint", "https://127.0.0.1/", + "--concierge-authenticator-type", "webhook", + "--concierge-authenticator-name", "test-authenticator", + }, + wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"token":"` + impersonationProxyTestToken("test-token") + `"}}` + "\n", + }, } for _, tt := range tests { tt := tt From 12e41d783fcb5fa58edd61c31cc55781bda4c6f6 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 26 Jan 2021 16:49:03 -0800 Subject: [PATCH 007/203] Refactored execCredentialForImpersonationProxy to be shared --- cmd/pinniped/cmd/login_oidc.go | 22 +++++++++----- cmd/pinniped/cmd/login_static.go | 52 ++------------------------------ 2 files changed, 16 insertions(+), 58 deletions(-) diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 999fd8a7..b4eb62ee 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -189,7 +189,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin if concierge != nil && flags.useImpersonationProxy { // Put the token into a TokenCredentialRequest // put the TokenCredentialRequest in an ExecCredential - req, err := execCredentialForImpersonationProxy(token, flags) + req, err := execCredentialForImpersonationProxy(token.IDToken.Token, flags.conciergeAuthenticatorType, flags.conciergeNamespace, flags.conciergeAuthenticatorName, token.IDToken.Expiry) if err != nil { return err } @@ -257,10 +257,16 @@ func mustGetConfigDir() string { return filepath.Join(home, ".config", xdgAppName) } -func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLoginFlags) (*clientauthv1beta1.ExecCredential, error) { +func execCredentialForImpersonationProxy( + idToken string, + conciergeAuthenticatorType string, + conciergeNamespace string, + conciergeAuthenticatorName string, + tokenExpiry metav1.Time, +) (*clientauthv1beta1.ExecCredential, error) { // TODO maybe de-dup this with conciergeclient.go var kind string - switch strings.ToLower(flags.conciergeAuthenticatorType) { + switch strings.ToLower(conciergeAuthenticatorType) { case "webhook": kind = "WebhookAuthenticator" case "jwt": @@ -270,18 +276,18 @@ func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLogin } reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ ObjectMeta: metav1.ObjectMeta{ - Namespace: flags.conciergeNamespace, + Namespace: conciergeNamespace, }, TypeMeta: metav1.TypeMeta{ Kind: "TokenCredentialRequest", APIVersion: loginv1alpha1.GroupName + "/v1alpha1", }, Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: token.IDToken.Token, // TODO + Token: idToken, // TODO Authenticator: corev1.TypedLocalObjectReference{ APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, Kind: kind, - Name: flags.conciergeAuthenticatorName, + Name: conciergeAuthenticatorName, }, }, }) @@ -298,8 +304,8 @@ func execCredentialForImpersonationProxy(token *oidctypes.Token, flags oidcLogin Token: encodedToken, }, } - if !token.IDToken.Expiry.IsZero() { - cred.Status.ExpirationTimestamp = &token.IDToken.Expiry + if !tokenExpiry.IsZero() { + cred.Status.ExpirationTimestamp = &tokenExpiry } return cred, nil } diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 7477dfd4..3b90c29c 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -5,22 +5,17 @@ package cmd import ( "context" - "encoding/base64" "encoding/json" "fmt" "io" "os" - "strings" "time" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/spf13/cobra" clientauthv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" - authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" - loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" "go.pinniped.dev/pkg/conciergeclient" "go.pinniped.dev/pkg/oidcclient/oidctypes" ) @@ -129,9 +124,10 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams } } if concierge != nil && flags.useImpersonationProxy { + var nilExpiry metav1.Time // Put the token into a TokenCredentialRequest // put the TokenCredentialRequest in an ExecCredential - req, err := execCredentialForImpersonationProxyStatic(token, flags) + req, err := execCredentialForImpersonationProxy(token, flags.conciergeAuthenticatorType, flags.conciergeNamespace, flags.conciergeAuthenticatorName, nilExpiry) if err != nil { return err } @@ -139,47 +135,3 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams } return json.NewEncoder(out).Encode(cred) } - -func execCredentialForImpersonationProxyStatic(token string, flags staticLoginParams) (*clientauthv1beta1.ExecCredential, error) { - // TODO maybe de-dup this with conciergeclient.go - var kind string - switch strings.ToLower(flags.conciergeAuthenticatorType) { - case "webhook": - kind = "WebhookAuthenticator" - case "jwt": - kind = "JWTAuthenticator" - default: - return nil, fmt.Errorf(`invalid authenticator type: %q, supported values are "webhook" and "jwt"`, kind) - } - reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: flags.conciergeNamespace, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginv1alpha1.GroupName + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: token, // TODO - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, - Kind: kind, - Name: flags.conciergeAuthenticatorName, - }, - }, - }) - if err != nil { - return nil, err - } - encodedToken := base64.RawURLEncoding.EncodeToString(reqJSON) - cred := &clientauthv1beta1.ExecCredential{ - TypeMeta: metav1.TypeMeta{ - Kind: "ExecCredential", - APIVersion: "client.authentication.k8s.io/v1beta1", - }, - Status: &clientauthv1beta1.ExecCredentialStatus{ - Token: encodedToken, - }, - } - return cred, nil -} From 343c275f4609f6af7d1492115f055005d9627c77 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 27 Jan 2021 15:39:16 -0800 Subject: [PATCH 008/203] Path to ci bundle rather than the actual value for get kubeconfig Also changed a function param to a pointer --- cmd/pinniped/cmd/kubeconfig.go | 27 ++++++++++++++----- cmd/pinniped/cmd/kubeconfig_test.go | 42 ++++++++++++++++++++--------- cmd/pinniped/cmd/login_oidc.go | 8 +++--- cmd/pinniped/cmd/login_static.go | 5 +--- 4 files changed, 55 insertions(+), 27 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index a4e74883..4937b03d 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -78,7 +78,7 @@ type getKubeconfigConciergeParams struct { authenticatorName string authenticatorType string apiGroupSuffix string - caBundleData string + caBundlePath string endpoint string useImpersonationProxy bool } @@ -113,7 +113,7 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", "pinniped.dev", "Concierge API group suffix") - f.StringVar(&flags.concierge.caBundleData, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") + f.StringVar(&flags.concierge.caBundlePath, "concierge-ca-bundle", "", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the concierge") f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") f.BoolVar(&flags.concierge.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") @@ -170,7 +170,14 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar return fmt.Errorf("could not load --kubeconfig/--kubeconfig-context: %w", err) } if flags.concierge.useImpersonationProxy { - cluster.CertificateAuthorityData = []byte(flags.concierge.caBundleData) + // TODO what to do if --use-impersonation-proxy is set but flags.concierge.caBundlePath is not??? + // TODO dont do this twice + conciergeCaBundleData, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) + if err != nil { + return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + } + + cluster.CertificateAuthorityData = []byte(conciergeCaBundleData) cluster.Server = flags.concierge.endpoint } clientset, err := deps.getClientset(clientConfig, flags.concierge.apiGroupSuffix) @@ -280,8 +287,16 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, if flags.concierge.endpoint == "" { flags.concierge.endpoint = v1Cluster.Server } - if flags.concierge.caBundleData == "" { - flags.concierge.caBundleData = base64.StdEncoding.EncodeToString(v1Cluster.CertificateAuthorityData) + + var encodedConciergeCaBundleData string + if flags.concierge.caBundlePath == "" { + encodedConciergeCaBundleData = base64.StdEncoding.EncodeToString(v1Cluster.CertificateAuthorityData) + } else { + conciergeCaBundleData, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) + if err != nil { + return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + } + encodedConciergeCaBundleData = base64.StdEncoding.EncodeToString([]byte(conciergeCaBundleData)) } // Append the flags to configure the Concierge credential exchange at runtime. @@ -292,7 +307,7 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, "--concierge-authenticator-name="+flags.concierge.authenticatorName, "--concierge-authenticator-type="+flags.concierge.authenticatorType, "--concierge-endpoint="+flags.concierge.endpoint, - "--concierge-ca-bundle-data="+flags.concierge.caBundleData, + "--concierge-ca-bundle-data="+encodedConciergeCaBundleData, ) if flags.concierge.useImpersonationProxy { execConfig.Args = append(execConfig.Args, diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 9c588adb..c806e5e3 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -28,11 +28,14 @@ import ( ) func TestGetKubeconfig(t *testing.T) { - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testOIDCCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) - testCABundlePath := filepath.Join(tmpdir, "testca.pem") - require.NoError(t, ioutil.WriteFile(testCABundlePath, testCA.Bundle(), 0600)) + testOIDCCABundlePath := filepath.Join(tmpdir, "testca.pem") + require.NoError(t, ioutil.WriteFile(testOIDCCABundlePath, testOIDCCA.Bundle(), 0600)) + + testConciergeCABundlePath := filepath.Join(tmpdir, "testconciergeca.pem") + require.NoError(t, ioutil.WriteFile(testConciergeCABundlePath, []byte("test-concierge-ca"), 0600)) tests := []struct { name string @@ -61,7 +64,7 @@ func TestGetKubeconfig(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name (default: autodiscover) --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) - --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge + --concierge-ca-bundle string Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the concierge --concierge-endpoint string API base for the Pinniped concierge endpoint --concierge-namespace string Namespace in which the concierge was installed (default "pinniped-concierge") --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy @@ -268,6 +271,19 @@ func TestGetKubeconfig(t *testing.T) { Error: tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator test-namespace/test-authenticator has invalid spec.tls.certificateAuthorityData: illegal base64 data at input byte 7 `), }, + { + name: " invalid concierge ca bundle", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-ca-bundle", "./does/not/exist", + "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", + "--concierge-use-impersonation-proxy", + }, + wantError: true, + wantStderr: here.Doc(` + Error: could not read --concierge-ca-bundle: open ./does/not/exist: no such file or directory + `), + }, { name: "invalid static token flags", args: []string{ @@ -399,7 +415,7 @@ func TestGetKubeconfig(t *testing.T) { Issuer: "https://example.com/issuer", Audience: "test-audience", TLS: &conciergev1alpha1.TLSSpec{ - CertificateAuthorityData: base64.StdEncoding.EncodeToString(testCA.Bundle()), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), }, }, }, @@ -442,7 +458,7 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testCA.Bundle())), + `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), }, { name: "autodetect nothing, set a bunch of options", @@ -454,7 +470,7 @@ func TestGetKubeconfig(t *testing.T) { "--oidc-issuer", "https://example.com/issuer", "--oidc-skip-browser", "--oidc-listen-port", "1234", - "--oidc-ca-bundle", testCABundlePath, + "--oidc-ca-bundle", testOIDCCABundlePath, "--oidc-session-cache", "/path/to/cache/dir/sessions.yaml", "--oidc-debug-session-cache", "--oidc-request-audience", "test-audience", @@ -506,14 +522,14 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testCA.Bundle())), + `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), wantAPIGroupSuffix: "tuna.io", }, { name: "configure impersonation proxy with autodetected JWT authenticator", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", - "--concierge-ca-bundle-data", "blah", // TODO make this more realistic, maybe do some validation? + "--concierge-ca-bundle", testConciergeCABundlePath, "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", "--concierge-use-impersonation-proxy", }, @@ -524,7 +540,7 @@ func TestGetKubeconfig(t *testing.T) { Issuer: "https://example.com/issuer", Audience: "test-audience", TLS: &conciergev1alpha1.TLSSpec{ - CertificateAuthorityData: base64.StdEncoding.EncodeToString(testCA.Bundle()), + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), }, }, }, @@ -533,7 +549,7 @@ func TestGetKubeconfig(t *testing.T) { apiVersion: v1 clusters: - cluster: - certificate-authority-data: YmxhaA== + certificate-authority-data: dGVzdC1jb25jaWVyZ2UtY2E= server: https://impersonation-proxy-endpoint.test name: pinniped contexts: @@ -558,7 +574,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-name=test-authenticator - --concierge-authenticator-type=jwt - --concierge-endpoint=https://impersonation-proxy-endpoint.test - - --concierge-ca-bundle-data=blah + - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= - --concierge-use-impersonation-proxy - --issuer=https://example.com/issuer - --client-id=pinniped-cli @@ -568,7 +584,7 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testCA.Bundle())), + `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), }, } for _, tt := range tests { diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index b4eb62ee..3d38fb2d 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -189,7 +189,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin if concierge != nil && flags.useImpersonationProxy { // Put the token into a TokenCredentialRequest // put the TokenCredentialRequest in an ExecCredential - req, err := execCredentialForImpersonationProxy(token.IDToken.Token, flags.conciergeAuthenticatorType, flags.conciergeNamespace, flags.conciergeAuthenticatorName, token.IDToken.Expiry) + req, err := execCredentialForImpersonationProxy(token.IDToken.Token, flags.conciergeAuthenticatorType, flags.conciergeNamespace, flags.conciergeAuthenticatorName, &token.IDToken.Expiry) if err != nil { return err } @@ -262,7 +262,7 @@ func execCredentialForImpersonationProxy( conciergeAuthenticatorType string, conciergeNamespace string, conciergeAuthenticatorName string, - tokenExpiry metav1.Time, + tokenExpiry *metav1.Time, ) (*clientauthv1beta1.ExecCredential, error) { // TODO maybe de-dup this with conciergeclient.go var kind string @@ -292,7 +292,7 @@ func execCredentialForImpersonationProxy( }, }) if err != nil { - return nil, err + return nil, fmt.Errorf("Error creating TokenCredentialRequest for impersonation proxy: %w", err) } encodedToken := base64.RawURLEncoding.EncodeToString(reqJSON) cred := &clientauthv1beta1.ExecCredential{ @@ -305,7 +305,7 @@ func execCredentialForImpersonationProxy( }, } if !tokenExpiry.IsZero() { - cred.Status.ExpirationTimestamp = &tokenExpiry + cred.Status.ExpirationTimestamp = tokenExpiry } return cred, nil } diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 3b90c29c..a8090935 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -11,8 +11,6 @@ import ( "os" "time" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/spf13/cobra" clientauthv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" @@ -124,10 +122,9 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams } } if concierge != nil && flags.useImpersonationProxy { - var nilExpiry metav1.Time // Put the token into a TokenCredentialRequest // put the TokenCredentialRequest in an ExecCredential - req, err := execCredentialForImpersonationProxy(token, flags.conciergeAuthenticatorType, flags.conciergeNamespace, flags.conciergeAuthenticatorName, nilExpiry) + req, err := execCredentialForImpersonationProxy(token, flags.conciergeAuthenticatorType, flags.conciergeNamespace, flags.conciergeAuthenticatorName, nil) if err != nil { return err } From ab60396ac4d3166965d0e5dd70f3c6c0832c7ae7 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 29 Jan 2021 16:38:50 -0800 Subject: [PATCH 009/203] CredentialIssuer contains Impersonation Proxy spec --- .../v1alpha1/types_credentialissuer.go.tmpl | 23 +++++++- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ++++++++++++ generated/1.17/README.adoc | 54 +++++++++++++++++++ .../config/v1alpha1/types_credentialissuer.go | 23 +++++++- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +++++++++++++++++- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ++++++++++++ generated/1.18/README.adoc | 54 +++++++++++++++++++ .../config/v1alpha1/types_credentialissuer.go | 23 +++++++- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +++++++++++++++++- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ++++++++++++ generated/1.19/README.adoc | 54 +++++++++++++++++++ .../config/v1alpha1/types_credentialissuer.go | 23 +++++++- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +++++++++++++++++- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ++++++++++++ generated/1.20/README.adoc | 54 +++++++++++++++++++ .../config/v1alpha1/types_credentialissuer.go | 23 +++++++- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +++++++++++++++++- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ++++++++++++ 18 files changed, 699 insertions(+), 9 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index f462056d..305ae886 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,6 +65,25 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } +type TLSConfig struct { + // The CA that clients should validate when connecting to the impersonation proxy endpoint + CertificateAuthorityData string `json:"certificateAuthorityData"` + // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint + SecretName string `json:"secretName"` +} + +type ImpersonationProxySpec struct { + // specify the external endpoint name that will route to the impersonation proxy port + ExternalEndpoint string `json:"externalEndpoint"` + // TLS configuration to communicate with the impersonation proxy + TLS TLSConfig `json:"tls"` +} + +// Spec for the credential issuer +type CredentialIssuerSpec struct { + ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` +} + // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -75,6 +94,8 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` + + Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index 9b4c0056..996c6bfa 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,6 +35,38 @@ spec: type: string metadata: type: object + spec: + description: Spec for the credential issuer + properties: + impersonationProxy: + properties: + externalEndpoint: + description: specify the external endpoint name that will route + to the impersonation proxy port + type: string + tls: + description: TLS configuration to communicate with the impersonation + proxy + properties: + certificateAuthorityData: + description: The CA that clients should validate when connecting + to the impersonation proxy endpoint + type: string + secretName: + description: The name of a secret of type "kubernetes.io/tls" + that will be used to serve the endpoint + type: string + required: + - certificateAuthorityData + - secretName + type: object + required: + - externalEndpoint + - tls + type: object + required: + - impersonationProxy + type: object status: description: Status of the credential issuer. properties: @@ -99,6 +131,7 @@ spec: - strategies type: object required: + - spec - status type: object served: true diff --git a/generated/1.17/README.adoc b/generated/1.17/README.adoc index 78e691e8..2bd7c08a 100644 --- a/generated/1.17/README.adoc +++ b/generated/1.17/README.adoc @@ -231,6 +231,7 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. +| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -254,6 +255,23 @@ Describes the configuration status of a Pinniped credential issuer. +[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerspec"] +==== CredentialIssuerSpec + +Spec for the credential issuer + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -293,6 +311,42 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyspec"] +==== ImpersonationProxySpec + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port +| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy +|=== + + +[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-tlsconfig"] +==== TLSConfig + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint +| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint +|=== + + [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index f462056d..305ae886 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,6 +65,25 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } +type TLSConfig struct { + // The CA that clients should validate when connecting to the impersonation proxy endpoint + CertificateAuthorityData string `json:"certificateAuthorityData"` + // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint + SecretName string `json:"secretName"` +} + +type ImpersonationProxySpec struct { + // specify the external endpoint name that will route to the impersonation proxy port + ExternalEndpoint string `json:"externalEndpoint"` + // TLS configuration to communicate with the impersonation proxy + TLS TLSConfig `json:"tls"` +} + +// Spec for the credential issuer +type CredentialIssuerSpec struct { + ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` +} + // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -75,6 +94,8 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` + + Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ef9877e8..e5167501 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,6 +17,7 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) + out.Spec = in.Spec return } @@ -87,6 +88,23 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { + *out = *in + out.ImpersonationProxy = in.ImpersonationProxy + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. +func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { + if in == nil { + return nil + } + out := new(CredentialIssuerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -131,3 +149,36 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { + *out = *in + out.TLS = in.TLS + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. +func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { + if in == nil { + return nil + } + out := new(ImpersonationProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { + if in == nil { + return nil + } + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 9b4c0056..996c6bfa 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,6 +35,38 @@ spec: type: string metadata: type: object + spec: + description: Spec for the credential issuer + properties: + impersonationProxy: + properties: + externalEndpoint: + description: specify the external endpoint name that will route + to the impersonation proxy port + type: string + tls: + description: TLS configuration to communicate with the impersonation + proxy + properties: + certificateAuthorityData: + description: The CA that clients should validate when connecting + to the impersonation proxy endpoint + type: string + secretName: + description: The name of a secret of type "kubernetes.io/tls" + that will be used to serve the endpoint + type: string + required: + - certificateAuthorityData + - secretName + type: object + required: + - externalEndpoint + - tls + type: object + required: + - impersonationProxy + type: object status: description: Status of the credential issuer. properties: @@ -99,6 +131,7 @@ spec: - strategies type: object required: + - spec - status type: object served: true diff --git a/generated/1.18/README.adoc b/generated/1.18/README.adoc index 2be2ab9b..00b6e07f 100644 --- a/generated/1.18/README.adoc +++ b/generated/1.18/README.adoc @@ -231,6 +231,7 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. +| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -254,6 +255,23 @@ Describes the configuration status of a Pinniped credential issuer. +[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerspec"] +==== CredentialIssuerSpec + +Spec for the credential issuer + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -293,6 +311,42 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyspec"] +==== ImpersonationProxySpec + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port +| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy +|=== + + +[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-tlsconfig"] +==== TLSConfig + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint +| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint +|=== + + [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index f462056d..305ae886 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,6 +65,25 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } +type TLSConfig struct { + // The CA that clients should validate when connecting to the impersonation proxy endpoint + CertificateAuthorityData string `json:"certificateAuthorityData"` + // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint + SecretName string `json:"secretName"` +} + +type ImpersonationProxySpec struct { + // specify the external endpoint name that will route to the impersonation proxy port + ExternalEndpoint string `json:"externalEndpoint"` + // TLS configuration to communicate with the impersonation proxy + TLS TLSConfig `json:"tls"` +} + +// Spec for the credential issuer +type CredentialIssuerSpec struct { + ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` +} + // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -75,6 +94,8 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` + + Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ef9877e8..e5167501 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,6 +17,7 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) + out.Spec = in.Spec return } @@ -87,6 +88,23 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { + *out = *in + out.ImpersonationProxy = in.ImpersonationProxy + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. +func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { + if in == nil { + return nil + } + out := new(CredentialIssuerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -131,3 +149,36 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { + *out = *in + out.TLS = in.TLS + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. +func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { + if in == nil { + return nil + } + out := new(ImpersonationProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { + if in == nil { + return nil + } + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 9b4c0056..996c6bfa 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,6 +35,38 @@ spec: type: string metadata: type: object + spec: + description: Spec for the credential issuer + properties: + impersonationProxy: + properties: + externalEndpoint: + description: specify the external endpoint name that will route + to the impersonation proxy port + type: string + tls: + description: TLS configuration to communicate with the impersonation + proxy + properties: + certificateAuthorityData: + description: The CA that clients should validate when connecting + to the impersonation proxy endpoint + type: string + secretName: + description: The name of a secret of type "kubernetes.io/tls" + that will be used to serve the endpoint + type: string + required: + - certificateAuthorityData + - secretName + type: object + required: + - externalEndpoint + - tls + type: object + required: + - impersonationProxy + type: object status: description: Status of the credential issuer. properties: @@ -99,6 +131,7 @@ spec: - strategies type: object required: + - spec - status type: object served: true diff --git a/generated/1.19/README.adoc b/generated/1.19/README.adoc index c1eae62c..870499c3 100644 --- a/generated/1.19/README.adoc +++ b/generated/1.19/README.adoc @@ -231,6 +231,7 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. +| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -254,6 +255,23 @@ Describes the configuration status of a Pinniped credential issuer. +[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerspec"] +==== CredentialIssuerSpec + +Spec for the credential issuer + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -293,6 +311,42 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyspec"] +==== ImpersonationProxySpec + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port +| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy +|=== + + +[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-tlsconfig"] +==== TLSConfig + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint +| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint +|=== + + [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index f462056d..305ae886 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,6 +65,25 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } +type TLSConfig struct { + // The CA that clients should validate when connecting to the impersonation proxy endpoint + CertificateAuthorityData string `json:"certificateAuthorityData"` + // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint + SecretName string `json:"secretName"` +} + +type ImpersonationProxySpec struct { + // specify the external endpoint name that will route to the impersonation proxy port + ExternalEndpoint string `json:"externalEndpoint"` + // TLS configuration to communicate with the impersonation proxy + TLS TLSConfig `json:"tls"` +} + +// Spec for the credential issuer +type CredentialIssuerSpec struct { + ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` +} + // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -75,6 +94,8 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` + + Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ef9877e8..e5167501 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,6 +17,7 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) + out.Spec = in.Spec return } @@ -87,6 +88,23 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { + *out = *in + out.ImpersonationProxy = in.ImpersonationProxy + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. +func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { + if in == nil { + return nil + } + out := new(CredentialIssuerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -131,3 +149,36 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { + *out = *in + out.TLS = in.TLS + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. +func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { + if in == nil { + return nil + } + out := new(ImpersonationProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { + if in == nil { + return nil + } + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 9b4c0056..996c6bfa 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,6 +35,38 @@ spec: type: string metadata: type: object + spec: + description: Spec for the credential issuer + properties: + impersonationProxy: + properties: + externalEndpoint: + description: specify the external endpoint name that will route + to the impersonation proxy port + type: string + tls: + description: TLS configuration to communicate with the impersonation + proxy + properties: + certificateAuthorityData: + description: The CA that clients should validate when connecting + to the impersonation proxy endpoint + type: string + secretName: + description: The name of a secret of type "kubernetes.io/tls" + that will be used to serve the endpoint + type: string + required: + - certificateAuthorityData + - secretName + type: object + required: + - externalEndpoint + - tls + type: object + required: + - impersonationProxy + type: object status: description: Status of the credential issuer. properties: @@ -99,6 +131,7 @@ spec: - strategies type: object required: + - spec - status type: object served: true diff --git a/generated/1.20/README.adoc b/generated/1.20/README.adoc index b516494c..31a65720 100644 --- a/generated/1.20/README.adoc +++ b/generated/1.20/README.adoc @@ -231,6 +231,7 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.2/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. +| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -254,6 +255,23 @@ Describes the configuration status of a Pinniped credential issuer. +[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerspec"] +==== CredentialIssuerSpec + +Spec for the credential issuer + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -293,6 +311,42 @@ Status of a credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyspec"] +==== ImpersonationProxySpec + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port +| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy +|=== + + +[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-tlsconfig"] +==== TLSConfig + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint +| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint +|=== + + [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index f462056d..305ae886 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,6 +65,25 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } +type TLSConfig struct { + // The CA that clients should validate when connecting to the impersonation proxy endpoint + CertificateAuthorityData string `json:"certificateAuthorityData"` + // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint + SecretName string `json:"secretName"` +} + +type ImpersonationProxySpec struct { + // specify the external endpoint name that will route to the impersonation proxy port + ExternalEndpoint string `json:"externalEndpoint"` + // TLS configuration to communicate with the impersonation proxy + TLS TLSConfig `json:"tls"` +} + +// Spec for the credential issuer +type CredentialIssuerSpec struct { + ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` +} + // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -75,6 +94,8 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` + + Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index ef9877e8..e5167501 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,6 +17,7 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) + out.Spec = in.Spec return } @@ -87,6 +88,23 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { + *out = *in + out.ImpersonationProxy = in.ImpersonationProxy + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. +func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { + if in == nil { + return nil + } + out := new(CredentialIssuerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -131,3 +149,36 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { + *out = *in + out.TLS = in.TLS + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. +func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { + if in == nil { + return nil + } + out := new(ImpersonationProxySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. +func (in *TLSConfig) DeepCopy() *TLSConfig { + if in == nil { + return nil + } + out := new(TLSConfig) + in.DeepCopyInto(out) + return out +} diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 9b4c0056..996c6bfa 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,6 +35,38 @@ spec: type: string metadata: type: object + spec: + description: Spec for the credential issuer + properties: + impersonationProxy: + properties: + externalEndpoint: + description: specify the external endpoint name that will route + to the impersonation proxy port + type: string + tls: + description: TLS configuration to communicate with the impersonation + proxy + properties: + certificateAuthorityData: + description: The CA that clients should validate when connecting + to the impersonation proxy endpoint + type: string + secretName: + description: The name of a secret of type "kubernetes.io/tls" + that will be used to serve the endpoint + type: string + required: + - certificateAuthorityData + - secretName + type: object + required: + - externalEndpoint + - tls + type: object + required: + - impersonationProxy + type: object status: description: Status of the credential issuer. properties: @@ -99,6 +131,7 @@ spec: - strategies type: object required: + - spec - status type: object served: true From 23e8c3591888c7d539b83cd6eaab02516027a42d Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Mon, 1 Feb 2021 12:43:34 -0800 Subject: [PATCH 010/203] Revert "CredentialIssuer contains Impersonation Proxy spec" This reverts commit 83bbd1fa9314508030ea9fcf26c6720212d65dc0. --- .../v1alpha1/types_credentialissuer.go.tmpl | 23 +------- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ------------ generated/1.17/README.adoc | 54 ------------------- .../config/v1alpha1/types_credentialissuer.go | 23 +------- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +----------------- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ------------ generated/1.18/README.adoc | 54 ------------------- .../config/v1alpha1/types_credentialissuer.go | 23 +------- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +----------------- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ------------ generated/1.19/README.adoc | 54 ------------------- .../config/v1alpha1/types_credentialissuer.go | 23 +------- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +----------------- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ------------ generated/1.20/README.adoc | 54 ------------------- .../config/v1alpha1/types_credentialissuer.go | 23 +------- .../config/v1alpha1/zz_generated.deepcopy.go | 53 +----------------- ...cierge.pinniped.dev_credentialissuers.yaml | 33 ------------ 18 files changed, 9 insertions(+), 699 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index 305ae886..f462056d 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -1,4 +1,4 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,25 +65,6 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } -type TLSConfig struct { - // The CA that clients should validate when connecting to the impersonation proxy endpoint - CertificateAuthorityData string `json:"certificateAuthorityData"` - // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint - SecretName string `json:"secretName"` -} - -type ImpersonationProxySpec struct { - // specify the external endpoint name that will route to the impersonation proxy port - ExternalEndpoint string `json:"externalEndpoint"` - // TLS configuration to communicate with the impersonation proxy - TLS TLSConfig `json:"tls"` -} - -// Spec for the credential issuer -type CredentialIssuerSpec struct { - ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` -} - // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -94,8 +75,6 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` - - Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index 996c6bfa..9b4c0056 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,38 +35,6 @@ spec: type: string metadata: type: object - spec: - description: Spec for the credential issuer - properties: - impersonationProxy: - properties: - externalEndpoint: - description: specify the external endpoint name that will route - to the impersonation proxy port - type: string - tls: - description: TLS configuration to communicate with the impersonation - proxy - properties: - certificateAuthorityData: - description: The CA that clients should validate when connecting - to the impersonation proxy endpoint - type: string - secretName: - description: The name of a secret of type "kubernetes.io/tls" - that will be used to serve the endpoint - type: string - required: - - certificateAuthorityData - - secretName - type: object - required: - - externalEndpoint - - tls - type: object - required: - - impersonationProxy - type: object status: description: Status of the credential issuer. properties: @@ -131,7 +99,6 @@ spec: - strategies type: object required: - - spec - status type: object served: true diff --git a/generated/1.17/README.adoc b/generated/1.17/README.adoc index 2bd7c08a..78e691e8 100644 --- a/generated/1.17/README.adoc +++ b/generated/1.17/README.adoc @@ -231,7 +231,6 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. -| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -255,23 +254,6 @@ Describes the configuration status of a Pinniped credential issuer. -[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerspec"] -==== CredentialIssuerSpec - -Spec for the credential issuer - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | -|=== - - [id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -311,42 +293,6 @@ Status of a credential issuer. |=== -[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyspec"] -==== ImpersonationProxySpec - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port -| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy -|=== - - -[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-tlsconfig"] -==== TLSConfig - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint -| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint -|=== - - [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index 305ae886..f462056d 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,25 +65,6 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } -type TLSConfig struct { - // The CA that clients should validate when connecting to the impersonation proxy endpoint - CertificateAuthorityData string `json:"certificateAuthorityData"` - // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint - SecretName string `json:"secretName"` -} - -type ImpersonationProxySpec struct { - // specify the external endpoint name that will route to the impersonation proxy port - ExternalEndpoint string `json:"externalEndpoint"` - // TLS configuration to communicate with the impersonation proxy - TLS TLSConfig `json:"tls"` -} - -// Spec for the credential issuer -type CredentialIssuerSpec struct { - ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` -} - // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -94,8 +75,6 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` - - Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index e5167501..ef9877e8 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,7 +17,6 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) - out.Spec = in.Spec return } @@ -88,23 +87,6 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { - *out = *in - out.ImpersonationProxy = in.ImpersonationProxy - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. -func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { - if in == nil { - return nil - } - out := new(CredentialIssuerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -149,36 +131,3 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { - *out = *in - out.TLS = in.TLS - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. -func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { - if in == nil { - return nil - } - out := new(ImpersonationProxySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. -func (in *TLSConfig) DeepCopy() *TLSConfig { - if in == nil { - return nil - } - out := new(TLSConfig) - in.DeepCopyInto(out) - return out -} diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 996c6bfa..9b4c0056 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,38 +35,6 @@ spec: type: string metadata: type: object - spec: - description: Spec for the credential issuer - properties: - impersonationProxy: - properties: - externalEndpoint: - description: specify the external endpoint name that will route - to the impersonation proxy port - type: string - tls: - description: TLS configuration to communicate with the impersonation - proxy - properties: - certificateAuthorityData: - description: The CA that clients should validate when connecting - to the impersonation proxy endpoint - type: string - secretName: - description: The name of a secret of type "kubernetes.io/tls" - that will be used to serve the endpoint - type: string - required: - - certificateAuthorityData - - secretName - type: object - required: - - externalEndpoint - - tls - type: object - required: - - impersonationProxy - type: object status: description: Status of the credential issuer. properties: @@ -131,7 +99,6 @@ spec: - strategies type: object required: - - spec - status type: object served: true diff --git a/generated/1.18/README.adoc b/generated/1.18/README.adoc index 00b6e07f..2be2ab9b 100644 --- a/generated/1.18/README.adoc +++ b/generated/1.18/README.adoc @@ -231,7 +231,6 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. -| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -255,23 +254,6 @@ Describes the configuration status of a Pinniped credential issuer. -[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerspec"] -==== CredentialIssuerSpec - -Spec for the credential issuer - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | -|=== - - [id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -311,42 +293,6 @@ Status of a credential issuer. |=== -[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyspec"] -==== ImpersonationProxySpec - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port -| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy -|=== - - -[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-tlsconfig"] -==== TLSConfig - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint -| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint -|=== - - [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index 305ae886..f462056d 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,25 +65,6 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } -type TLSConfig struct { - // The CA that clients should validate when connecting to the impersonation proxy endpoint - CertificateAuthorityData string `json:"certificateAuthorityData"` - // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint - SecretName string `json:"secretName"` -} - -type ImpersonationProxySpec struct { - // specify the external endpoint name that will route to the impersonation proxy port - ExternalEndpoint string `json:"externalEndpoint"` - // TLS configuration to communicate with the impersonation proxy - TLS TLSConfig `json:"tls"` -} - -// Spec for the credential issuer -type CredentialIssuerSpec struct { - ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` -} - // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -94,8 +75,6 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` - - Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index e5167501..ef9877e8 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,7 +17,6 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) - out.Spec = in.Spec return } @@ -88,23 +87,6 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { - *out = *in - out.ImpersonationProxy = in.ImpersonationProxy - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. -func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { - if in == nil { - return nil - } - out := new(CredentialIssuerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -149,36 +131,3 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { - *out = *in - out.TLS = in.TLS - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. -func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { - if in == nil { - return nil - } - out := new(ImpersonationProxySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. -func (in *TLSConfig) DeepCopy() *TLSConfig { - if in == nil { - return nil - } - out := new(TLSConfig) - in.DeepCopyInto(out) - return out -} diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 996c6bfa..9b4c0056 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,38 +35,6 @@ spec: type: string metadata: type: object - spec: - description: Spec for the credential issuer - properties: - impersonationProxy: - properties: - externalEndpoint: - description: specify the external endpoint name that will route - to the impersonation proxy port - type: string - tls: - description: TLS configuration to communicate with the impersonation - proxy - properties: - certificateAuthorityData: - description: The CA that clients should validate when connecting - to the impersonation proxy endpoint - type: string - secretName: - description: The name of a secret of type "kubernetes.io/tls" - that will be used to serve the endpoint - type: string - required: - - certificateAuthorityData - - secretName - type: object - required: - - externalEndpoint - - tls - type: object - required: - - impersonationProxy - type: object status: description: Status of the credential issuer. properties: @@ -131,7 +99,6 @@ spec: - strategies type: object required: - - spec - status type: object served: true diff --git a/generated/1.19/README.adoc b/generated/1.19/README.adoc index 870499c3..c1eae62c 100644 --- a/generated/1.19/README.adoc +++ b/generated/1.19/README.adoc @@ -231,7 +231,6 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.19/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. -| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -255,23 +254,6 @@ Describes the configuration status of a Pinniped credential issuer. -[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerspec"] -==== CredentialIssuerSpec - -Spec for the credential issuer - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | -|=== - - [id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -311,42 +293,6 @@ Status of a credential issuer. |=== -[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyspec"] -==== ImpersonationProxySpec - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port -| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy -|=== - - -[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-tlsconfig"] -==== TLSConfig - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint -| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint -|=== - - [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index 305ae886..f462056d 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,25 +65,6 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } -type TLSConfig struct { - // The CA that clients should validate when connecting to the impersonation proxy endpoint - CertificateAuthorityData string `json:"certificateAuthorityData"` - // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint - SecretName string `json:"secretName"` -} - -type ImpersonationProxySpec struct { - // specify the external endpoint name that will route to the impersonation proxy port - ExternalEndpoint string `json:"externalEndpoint"` - // TLS configuration to communicate with the impersonation proxy - TLS TLSConfig `json:"tls"` -} - -// Spec for the credential issuer -type CredentialIssuerSpec struct { - ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` -} - // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -94,8 +75,6 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` - - Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index e5167501..ef9877e8 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,7 +17,6 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) - out.Spec = in.Spec return } @@ -88,23 +87,6 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { - *out = *in - out.ImpersonationProxy = in.ImpersonationProxy - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. -func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { - if in == nil { - return nil - } - out := new(CredentialIssuerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -149,36 +131,3 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { - *out = *in - out.TLS = in.TLS - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. -func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { - if in == nil { - return nil - } - out := new(ImpersonationProxySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. -func (in *TLSConfig) DeepCopy() *TLSConfig { - if in == nil { - return nil - } - out := new(TLSConfig) - in.DeepCopyInto(out) - return out -} diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 996c6bfa..9b4c0056 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,38 +35,6 @@ spec: type: string metadata: type: object - spec: - description: Spec for the credential issuer - properties: - impersonationProxy: - properties: - externalEndpoint: - description: specify the external endpoint name that will route - to the impersonation proxy port - type: string - tls: - description: TLS configuration to communicate with the impersonation - proxy - properties: - certificateAuthorityData: - description: The CA that clients should validate when connecting - to the impersonation proxy endpoint - type: string - secretName: - description: The name of a secret of type "kubernetes.io/tls" - that will be used to serve the endpoint - type: string - required: - - certificateAuthorityData - - secretName - type: object - required: - - externalEndpoint - - tls - type: object - required: - - impersonationProxy - type: object status: description: Status of the credential issuer. properties: @@ -131,7 +99,6 @@ spec: - strategies type: object required: - - spec - status type: object served: true diff --git a/generated/1.20/README.adoc b/generated/1.20/README.adoc index 31a65720..b516494c 100644 --- a/generated/1.20/README.adoc +++ b/generated/1.20/README.adoc @@ -231,7 +231,6 @@ Describes the configuration status of a Pinniped credential issuer. | *`metadata`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.2/#objectmeta-v1-meta[$$ObjectMeta$$]__ | Refer to Kubernetes API documentation for fields of `metadata`. | *`status`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$]__ | Status of the credential issuer. -| *`spec`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$]__ | |=== @@ -255,23 +254,6 @@ Describes the configuration status of a Pinniped credential issuer. -[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerspec"] -==== CredentialIssuerSpec - -Spec for the credential issuer - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuer[$$CredentialIssuer$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`impersonationProxy`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$]__ | -|=== - - [id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerstatus"] ==== CredentialIssuerStatus @@ -311,42 +293,6 @@ Status of a credential issuer. |=== -[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyspec"] -==== ImpersonationProxySpec - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerspec[$$CredentialIssuerSpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`externalEndpoint`* __string__ | specify the external endpoint name that will route to the impersonation proxy port -| *`tls`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-tlsconfig[$$TLSConfig$$]__ | TLS configuration to communicate with the impersonation proxy -|=== - - -[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-tlsconfig"] -==== TLSConfig - - - -.Appears In: -**** -- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-impersonationproxyspec[$$ImpersonationProxySpec$$] -**** - -[cols="25a,75a", options="header"] -|=== -| Field | Description -| *`certificateAuthorityData`* __string__ | The CA that clients should validate when connecting to the impersonation proxy endpoint -| *`secretName`* __string__ | The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint -|=== - - [id="{anchor_prefix}-config-supervisor-pinniped-dev-v1alpha1"] === config.supervisor.pinniped.dev/v1alpha1 diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index 305ae886..f462056d 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -65,25 +65,6 @@ type CredentialIssuerStrategy struct { LastUpdateTime metav1.Time `json:"lastUpdateTime"` } -type TLSConfig struct { - // The CA that clients should validate when connecting to the impersonation proxy endpoint - CertificateAuthorityData string `json:"certificateAuthorityData"` - // The name of a secret of type "kubernetes.io/tls" that will be used to serve the endpoint - SecretName string `json:"secretName"` -} - -type ImpersonationProxySpec struct { - // specify the external endpoint name that will route to the impersonation proxy port - ExternalEndpoint string `json:"externalEndpoint"` - // TLS configuration to communicate with the impersonation proxy - TLS TLSConfig `json:"tls"` -} - -// Spec for the credential issuer -type CredentialIssuerSpec struct { - ImpersonationProxy ImpersonationProxySpec `json:"impersonationProxy"` -} - // Describes the configuration status of a Pinniped credential issuer. // +genclient // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object @@ -94,8 +75,6 @@ type CredentialIssuer struct { // Status of the credential issuer. Status CredentialIssuerStatus `json:"status"` - - Spec CredentialIssuerSpec `json:"spec"` } // List of CredentialIssuer objects. diff --git a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index e5167501..ef9877e8 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -1,6 +1,6 @@ // +build !ignore_autogenerated -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2020 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Code generated by deepcopy-gen. DO NOT EDIT. @@ -17,7 +17,6 @@ func (in *CredentialIssuer) DeepCopyInto(out *CredentialIssuer) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Status.DeepCopyInto(&out.Status) - out.Spec = in.Spec return } @@ -88,23 +87,6 @@ func (in *CredentialIssuerList) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CredentialIssuerSpec) DeepCopyInto(out *CredentialIssuerSpec) { - *out = *in - out.ImpersonationProxy = in.ImpersonationProxy - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerSpec. -func (in *CredentialIssuerSpec) DeepCopy() *CredentialIssuerSpec { - if in == nil { - return nil - } - out := new(CredentialIssuerSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = *in @@ -149,36 +131,3 @@ func (in *CredentialIssuerStrategy) DeepCopy() *CredentialIssuerStrategy { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ImpersonationProxySpec) DeepCopyInto(out *ImpersonationProxySpec) { - *out = *in - out.TLS = in.TLS - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImpersonationProxySpec. -func (in *ImpersonationProxySpec) DeepCopy() *ImpersonationProxySpec { - if in == nil { - return nil - } - out := new(ImpersonationProxySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TLSConfig) DeepCopyInto(out *TLSConfig) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSConfig. -func (in *TLSConfig) DeepCopy() *TLSConfig { - if in == nil { - return nil - } - out := new(TLSConfig) - in.DeepCopyInto(out) - return out -} diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 996c6bfa..9b4c0056 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -35,38 +35,6 @@ spec: type: string metadata: type: object - spec: - description: Spec for the credential issuer - properties: - impersonationProxy: - properties: - externalEndpoint: - description: specify the external endpoint name that will route - to the impersonation proxy port - type: string - tls: - description: TLS configuration to communicate with the impersonation - proxy - properties: - certificateAuthorityData: - description: The CA that clients should validate when connecting - to the impersonation proxy endpoint - type: string - secretName: - description: The name of a secret of type "kubernetes.io/tls" - that will be used to serve the endpoint - type: string - required: - - certificateAuthorityData - - secretName - type: object - required: - - externalEndpoint - - tls - type: object - required: - - impersonationProxy - type: object status: description: Status of the credential issuer. properties: @@ -131,7 +99,6 @@ spec: - strategies type: object required: - - spec - status type: object served: true From 6b46bae6c68eb44af60811666e01e64415a58cac Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 3 Feb 2021 11:32:29 -0800 Subject: [PATCH 011/203] Fixed integration test compile failures after rebase --- test/integration/concierge_impersonation_proxy_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index f3e81631..d1228707 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package integration @@ -35,7 +35,7 @@ func TestImpersonationProxy(t *testing.T) { defer cancel() // Create a client using the admin kubeconfig. - adminClient := library.NewClientset(t) + // adminClient := library.NewClientset(t) // Create a WebhookAuthenticator. authenticator := library.CreateTestWebhookAuthenticator(ctx, t) @@ -61,13 +61,13 @@ func TestImpersonationProxy(t *testing.T) { t.Run( "access as user", - library.AccessAsUserTest(ctx, adminClient, env.TestUser.ExpectedUsername, clientset), + library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, clientset), ) for _, group := range env.TestUser.ExpectedGroups { group := group t.Run( "access as group "+group, - library.AccessAsGroupTest(ctx, adminClient, group, clientset), + library.AccessAsGroupTest(ctx, group, clientset), ) } } From 812f5084a12290d4282f6ea94103e1d64c125498 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Tue, 9 Feb 2021 13:25:24 -0500 Subject: [PATCH 012/203] internal/concierge/impersonator: don't mutate ServeHTTP() req I added that test helper to create an http.Request since I wanted to properly initialize the http.Request's context. Signed-off-by: Andrew Keesler --- .../concierge/impersonator/impersonator.go | 7 +- .../impersonator/impersonator_test.go | 79 +++++++------------ 2 files changed, 34 insertions(+), 52 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 8c8cfd1c..befe390e 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -105,11 +105,12 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } log = log.WithValues("userID", userInfo.GetUID()) - newHeaders := getProxyHeaders(userInfo, r.Header) - r.Header = newHeaders + // Never mutate request (see http.Handler docs). + newR := r.WithContext(r.Context()) + newR.Header = getProxyHeaders(userInfo, r.Header) log.Info("proxying authenticated request") - p.proxy.ServeHTTP(w, r) + p.proxy.ServeHTTP(w, newR) } func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header { diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 5ae19e02..bf1c5981 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -4,6 +4,7 @@ package impersonator import ( + "context" "encoding/base64" "encoding/json" "fmt" @@ -53,6 +54,12 @@ func TestImpersonator(t *testing.T) { BearerToken: "some-service-account-token", TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testServerCA)}, } + newRequest := func(h http.Header) *http.Request { + r, err := http.NewRequestWithContext(context.Background(), http.MethodGet, validURL.String(), nil) + require.NoError(t, err) + r.Header = h + return r + } tests := []struct { name string @@ -102,61 +109,41 @@ func TestImpersonator(t *testing.T) { wantCreationErr: "could not get in-cluster transport: using a custom transport with TLS certificate options or the insecure flag is not allowed", }, { - name: "missing authorization header", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{}, - URL: validURL, - }, + name: "missing authorization header", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, wantLogs: []string{"\"error\"=\"missing authorization header\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { - name: "authorization header missing bearer prefix", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{"Authorization": {makeTestTokenRequest("foo", "authenticator-one", "test-token")}}, - URL: validURL, - }, + name: "authorization header missing bearer prefix", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Authorization": {makeTestTokenRequest("foo", "authenticator-one", "test-token")}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, wantLogs: []string{"\"error\"=\"authorization header must be of type Bearer\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { - name: "token is not base64 encoded", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{"Authorization": {"Bearer !!!"}}, - URL: validURL, - }, + name: "token is not base64 encoded", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Authorization": {"Bearer !!!"}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, wantLogs: []string{"\"error\"=\"invalid base64 in encoded bearer token: illegal base64 data at input byte 0\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { - name: "base64 encoded token is not valid json", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{"Authorization": {"Bearer abc"}}, - URL: validURL, - }, + name: "base64 encoded token is not valid json", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Authorization": {"Bearer abc"}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, wantLogs: []string{"\"error\"=\"invalid TokenCredentialRequest encoded in bearer token: invalid character 'i' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { - name: "token could not be authenticated", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("", "", "")}}, - URL: validURL, - }, + name: "token could not be authenticated", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("", "", "")}}), wantHTTPBody: "invalid token\n", wantHTTPStatus: http.StatusUnauthorized, wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"\"} \"authenticatorNamespace\"=\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, @@ -164,11 +151,7 @@ func TestImpersonator(t *testing.T) { { name: "token authenticates as nil", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}}, - URL: validURL, - }, + request: newRequest(map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}}), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { recorder.AuthenticateToken(gomock.Any(), "test-token").Return(nil, false, nil) }, @@ -180,15 +163,11 @@ func TestImpersonator(t *testing.T) { { name: "token validates", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: &http.Request{ - Method: "GET", - Header: map[string][]string{ - "Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}, - "Malicious-Header": {"test-header-value-1"}, - "User-Agent": {"test-user-agent"}, - }, - URL: validURL, - }, + request: newRequest(map[string][]string{ + "Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}, + "Malicious-Header": {"test-header-value-1"}, + "User-Agent": {"test-user-agent"}, + }), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { userInfo := user.DefaultInfo{ Name: "test-user", @@ -228,7 +207,9 @@ func TestImpersonator(t *testing.T) { require.NoError(t, err) require.NotNil(t, proxy) w := httptest.NewRecorder() + requestBeforeServe := tt.request.Clone(tt.request.Context()) proxy.ServeHTTP(w, tt.request) + require.Equal(t, requestBeforeServe, tt.request, "ServeHTTP() mutated the request, and it should not per http.Handler docs") if tt.wantHTTPStatus != 0 { require.Equal(t, tt.wantHTTPStatus, w.Code) } From dfcc2a1eb851214f0cac23e24e2219c44674750e Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 5 Feb 2021 17:01:39 -0800 Subject: [PATCH 013/203] Introduce clusterhost package to determine whether a cluster has control plane nodes Also added hasExternalLoadBalancerProvider key to cluster capabilities for integration testing. Signed-off-by: Ryan Richard --- internal/clusterhost/clusterhost.go | 63 +++++++++ internal/clusterhost/clusterhost_test.go | 169 +++++++++++++++++++++++ internal/concierge/server/server.go | 36 +++++ test/cluster_capabilities/gke.yaml | 5 +- test/cluster_capabilities/kind.yaml | 5 +- test/cluster_capabilities/tkgs.yaml | 5 +- test/library/env.go | 3 +- 7 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 internal/clusterhost/clusterhost.go create mode 100644 internal/clusterhost/clusterhost_test.go diff --git a/internal/clusterhost/clusterhost.go b/internal/clusterhost/clusterhost.go new file mode 100644 index 00000000..bfdbd6c7 --- /dev/null +++ b/internal/clusterhost/clusterhost.go @@ -0,0 +1,63 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clusterhost + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + labelNodeRolePrefix = "node-role.kubernetes.io/" + nodeLabelRole = "kubernetes.io/node-role" + controlPlaneNodeRole = "control-plane" + // this role was deprecated by kubernetes 1.20. + masterNodeRole = "master" +) + +type ClusterHost struct { + client kubernetes.Interface +} + +func New(client kubernetes.Interface) *ClusterHost { + return &ClusterHost{client: client} +} + +func (c *ClusterHost) HasControlPlaneNodes(ctx context.Context) (bool, error) { + nodes, err := c.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return false, fmt.Errorf("error fetching nodes: %v", err) + } + if len(nodes.Items) == 0 { + return false, fmt.Errorf("no nodes found") + } + for _, node := range nodes.Items { + for k, v := range node.Labels { + if isControlPlaneNodeRole(k, v) { + return true, nil + } + } + } + + return false, nil +} + +func isControlPlaneNodeRole(k string, v string) bool { + if k == labelNodeRolePrefix+controlPlaneNodeRole { + return true + } + if k == labelNodeRolePrefix+masterNodeRole { + return true + } + if k == nodeLabelRole && v == controlPlaneNodeRole { + return true + } + if k == nodeLabelRole && v == masterNodeRole { + return true + } + return false +} diff --git a/internal/clusterhost/clusterhost_test.go b/internal/clusterhost/clusterhost_test.go new file mode 100644 index 00000000..11405096 --- /dev/null +++ b/internal/clusterhost/clusterhost_test.go @@ -0,0 +1,169 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package clusterhost + +import ( + "context" + "errors" + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + + v1 "k8s.io/api/core/v1" +) + +func TestHasControlPlaneNodes(t *testing.T) { + tests := []struct { + name string + nodes []*v1.Node + listNodesErr error + wantErr error + wantReturnValue bool + }{ + { + name: "Fetching nodes returns an error", + listNodesErr: errors.New("couldn't get nodes"), + wantErr: errors.New("error fetching nodes: couldn't get nodes"), + }, + { + name: "Fetching nodes returns an empty array", + nodes: []*v1.Node{}, + wantErr: errors.New("no nodes found"), + }, + { + name: "Nodes found, but not control plane nodes", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{ + "not-control-plane-label": "some-value", + "kubernetes.io/node-role": "worker", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{"node-role.kubernetes.io/worker": ""}, + }, + }, + }, + wantReturnValue: false, + }, + { + name: "Nodes found, including a control-plane role in node-role.kubernetes.io/ format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "node-role.kubernetes.io/control-plane": "", + }, + }, + }, + }, + wantReturnValue: true, + }, + { + name: "Nodes found, including a master role in node-role.kubernetes.io/ format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "node-role.kubernetes.io/master": "", + }, + }, + }, + }, + wantReturnValue: true, + }, + { + name: "Nodes found, including a control-plane role in kubernetes.io/node-role= format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "kubernetes.io/node-role": "control-plane", + }, + }, + }, + }, + wantReturnValue: true, + }, + { + name: "Nodes found, including a master role in kubernetes.io/node-role= format", + nodes: []*v1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-1", + Labels: map[string]string{"unrelated-label": "some-value"}, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node-2", + Labels: map[string]string{ + "some-other-label": "some-value", + "kubernetes.io/node-role": "master", + }, + }, + }, + }, + wantReturnValue: true, + }, + } + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + kubeClient := kubernetesfake.NewSimpleClientset() + if test.listNodesErr != nil { + listNodesErr := test.listNodesErr + kubeClient.PrependReactor( + "list", + "nodes", + func(_ coretesting.Action) (bool, runtime.Object, error) { + return true, nil, listNodesErr + }, + ) + } + for _, node := range test.nodes { + err := kubeClient.Tracker().Add(node) + require.NoError(t, err) + } + clusterHost := New(kubeClient) + hasControlPlaneNodes, err := clusterHost.HasControlPlaneNodes(context.Background()) + require.Equal(t, test.wantErr, err) + require.Equal(t, test.wantReturnValue, hasControlPlaneNodes) + }) + } +} diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index 9b22ba28..dbf94dc2 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -13,6 +13,12 @@ import ( "net/http" "time" + "k8s.io/apimachinery/pkg/util/intstr" + + v1 "k8s.io/api/core/v1" + + "go.pinniped.dev/internal/kubeclient" + "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -169,6 +175,35 @@ func (a *App) runServer(ctx context.Context) error { return fmt.Errorf("could not create aggregated API server: %w", err) } + client, err := kubeclient.New() + if err != nil { + plog.WarningErr("could not create client", err) + } else { + appNameLabel := cfg.Labels["app"] + loadBalancer := v1.Service{ + Spec: v1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []v1.ServicePort{ + { + TargetPort: intstr.FromInt(8444), + Port: 443, + Protocol: v1.ProtocolTCP, + }, + }, + Selector: map[string]string{"app": appNameLabel}, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "impersonation-proxy-load-balancer", + Namespace: podInfo.Namespace, + Labels: cfg.Labels, + }, + } + _, err = client.Kubernetes.CoreV1().Services(podInfo.Namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) + if err != nil { + plog.WarningErr("could not create load balancer", err) + } + } + // run proxy handler impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) if err != nil { @@ -191,6 +226,7 @@ func (a *App) runServer(ctx context.Context) error { Certificates: []tls.Certificate{*impersonationCert}, }, } + // todo store CA, cert etc. on the authenticator status go func() { if err := impersonationProxyServer.ListenAndServeTLS("", ""); err != nil { klog.ErrorS(err, "could not serve impersonation proxy") diff --git a/test/cluster_capabilities/gke.yaml b/test/cluster_capabilities/gke.yaml index 4852280d..2f97168e 100644 --- a/test/cluster_capabilities/gke.yaml +++ b/test/cluster_capabilities/gke.yaml @@ -1,4 +1,4 @@ -# Copyright 2020 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Describe the capabilities of the cluster against which the integration tests will run. @@ -6,3 +6,6 @@ capabilities: # Is it possible to borrow the cluster's signing key from the kube API server? clusterSigningKeyIsAvailable: false + + # Will the cluster successfully provision a load balancer if requested? + hasExternalLoadBalancerProvider: true diff --git a/test/cluster_capabilities/kind.yaml b/test/cluster_capabilities/kind.yaml index c81f6687..ba9099fa 100644 --- a/test/cluster_capabilities/kind.yaml +++ b/test/cluster_capabilities/kind.yaml @@ -1,4 +1,4 @@ -# Copyright 2020 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Describe the capabilities of the cluster against which the integration tests will run. @@ -6,3 +6,6 @@ capabilities: # Is it possible to borrow the cluster's signing key from the kube API server? clusterSigningKeyIsAvailable: true + + # Will the cluster successfully provision a load balancer if requested? + hasExternalLoadBalancerProvider: false diff --git a/test/cluster_capabilities/tkgs.yaml b/test/cluster_capabilities/tkgs.yaml index c81f6687..a45b92b3 100644 --- a/test/cluster_capabilities/tkgs.yaml +++ b/test/cluster_capabilities/tkgs.yaml @@ -1,4 +1,4 @@ -# Copyright 2020 the Pinniped contributors. All Rights Reserved. +# Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # Describe the capabilities of the cluster against which the integration tests will run. @@ -6,3 +6,6 @@ capabilities: # Is it possible to borrow the cluster's signing key from the kube API server? clusterSigningKeyIsAvailable: true + + # Will the cluster successfully provision a load balancer if requested? + hasExternalLoadBalancerProvider: true diff --git a/test/library/env.go b/test/library/env.go index 8fa53a93..ada5ee48 100644 --- a/test/library/env.go +++ b/test/library/env.go @@ -18,7 +18,8 @@ import ( type Capability string const ( - ClusterSigningKeyIsAvailable Capability = "clusterSigningKeyIsAvailable" + ClusterSigningKeyIsAvailable Capability = "clusterSigningKeyIsAvailable" + HasExternalLoadBalancerProvider Capability = "hasExternalLoadBalancerProvider" ) // TestEnv captures all the external parameters consumed by our integration tests. From 8697488126dfabc96cd4b7fc2465c00a2fa4d75d Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Tue, 9 Feb 2021 15:28:56 -0500 Subject: [PATCH 014/203] internal/concierge/impersonator: use kubeconfig from kubeclient Signed-off-by: Andrew Keesler --- internal/concierge/impersonator/impersonator.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index befe390e..c80957b1 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -20,6 +20,7 @@ import ( "go.pinniped.dev/generated/1.20/apis/concierge/login" loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/controller/authenticator/authncache" + "go.pinniped.dev/internal/kubeclient" ) // allowedHeaders are the set of HTTP headers that are allowed to be forwarded through the impersonation proxy. @@ -39,7 +40,13 @@ type Proxy struct { } func New(cache *authncache.Cache, log logr.Logger) (*Proxy, error) { - return newInternal(cache, log, rest.InClusterConfig) + return newInternal(cache, log, func() (*rest.Config, error) { + client, err := kubeclient.New() + if err != nil { + return nil, err + } + return client.JSONConfig, nil + }) } func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*rest.Config, error)) (*Proxy, error) { From 268ca5b7f638d83cae7f7a2057cd627bef6b5b9f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 9 Feb 2021 13:42:56 -0800 Subject: [PATCH 015/203] Add config structs in impersonator package Signed-off-by: Margo Crawford --- internal/concierge/impersonator/config.go | 71 ++++++++++++++ .../concierge/impersonator/config_test.go | 98 +++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 internal/concierge/impersonator/config.go create mode 100644 internal/concierge/impersonator/config_test.go diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go new file mode 100644 index 00000000..b2c2d10e --- /dev/null +++ b/internal/concierge/impersonator/config.go @@ -0,0 +1,71 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" +) + +type Mode string + +const ( + // Explicitly enable the impersonation proxy. + ModeEnabled Mode = "enabled" + + // Explicitly disable the impersonation proxy. + ModeDisabled Mode = "disabled" + + // Allow the proxy to decide if it should be enabled or disabled based upon the cluster in which it is running. + ModeAuto Mode = "auto" +) + +const ( + ConfigMapDataKey = "config.yaml" +) + +// When specified, both CertificateAuthoritySecretName and TLSSecretName are required. They may be specified to +// both point at the same Secret or to point at different Secrets. +type TLSConfig struct { + // CertificateAuthoritySecretName contains the name of a namespace-local Secret resource. The corresponding Secret + // must contain a key called "ca.crt" whose value is the CA certificate which clients should trust when connecting + // to the impersonation proxy. + CertificateAuthoritySecretName string `json:"certificateAuthoritySecretName"` + + // TLSSecretName contains the name of a namespace-local Secret resource. The corresponding Secret must be of type + // "kubernetes.io/tls" and contain keys called "tls.crt" and "tls.key" whose values are the TLS certificate and + // private key that will be used by the impersonation proxy to serve its endpoints. + TLSSecretName string `json:"tlsSecretName"` +} + +type Config struct { + // Enable or disable the impersonation proxy. Optional. Defaults to ModeAuto. + Mode Mode `json:"mode,omitempty"` + + // The HTTPS URL of the impersonation proxy for clients to use from outside the cluster. Used when creating TLS + // certificates and for clients to discover the endpoint. Optional. When not specified, if the impersonation proxy + // is started, then it will automatically create a LoadBalancer Service and use its ingress as the endpoint. + Endpoint string `json:"endpoint,omitempty"` + + // The TLS configuration of the impersonation proxy's endpoints. Optional. When not specified, a CA and TLS + // certificate will be automatically created based on the Endpoint setting. + TLS *TLSConfig `json:"tls,omitempty"` +} + +func FromConfigMap(configMap *v1.ConfigMap) (*Config, error) { + stringConfig, ok := configMap.Data[ConfigMapDataKey] + if !ok { + return nil, fmt.Errorf(`ConfigMap is missing expected key "%s"`, ConfigMapDataKey) + } + var config Config + if err := yaml.Unmarshal([]byte(stringConfig), &config); err != nil { + return nil, fmt.Errorf("decode yaml: %w", err) + } + if config.Mode == "" { + config.Mode = ModeAuto // set the default value + } + return &config, nil +} diff --git a/internal/concierge/impersonator/config_test.go b/internal/concierge/impersonator/config_test.go new file mode 100644 index 00000000..8406072b --- /dev/null +++ b/internal/concierge/impersonator/config_test.go @@ -0,0 +1,98 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonator + +import ( + "testing" + + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.pinniped.dev/internal/here" +) + +func TestFromConfigMap(t *testing.T) { + tests := []struct { + name string + configMap *v1.ConfigMap + wantConfig *Config + wantError string + }{ + { + name: "fully configured, valid config", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": here.Doc(` + mode: enabled + endpoint: https://proxy.example.com:8443/ + tls: + certificateAuthoritySecretName: my-ca-crt + tlsSecretName: my-tls-certificate-and-key + `), + }, + }, + wantConfig: &Config{ + Mode: "enabled", + Endpoint: "https://proxy.example.com:8443/", + TLS: &TLSConfig{ + CertificateAuthoritySecretName: "my-ca-crt", + TLSSecretName: "my-tls-certificate-and-key", + }, + }, + }, + { + name: "empty, valid config", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "", + }, + }, + wantConfig: &Config{ + Mode: "auto", + Endpoint: "", + TLS: nil, + }, + }, + { + name: "wrong key in configmap", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "wrong-key": "", + }, + }, + wantError: `ConfigMap is missing expected key "config.yaml"`, + }, + { + name: "illegal yaml in configmap", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "this is not yaml", + }, + }, + wantError: "decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config", + }, + } + + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + config, err := FromConfigMap(test.configMap) + require.Equal(t, test.wantConfig, config) + if test.wantError != "" { + require.EqualError(t, err, test.wantError) + } else { + require.NoError(t, err) + } + }) + } +} From 5cd60fa5f9c7c6ffce7be4fbacd4ad0723996c2e Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Feb 2021 17:22:47 -0800 Subject: [PATCH 016/203] Move starting/stopping impersonation proxy server to a new controller - Watch a configmap to read the configuration of the impersonation proxy and reconcile it. - Implements "auto" mode by querying the API for control plane nodes. - WIP: does not create a load balancer or proper TLS certificates yet. Those will come in future commits. Signed-off-by: Margo Crawford --- deploy/concierge/rbac.yaml | 18 +- internal/concierge/impersonator/config.go | 16 +- .../concierge/impersonator/config_test.go | 75 ++- .../concierge/impersonator/impersonator.go | 16 +- internal/concierge/server/server.go | 71 --- .../impersonatorconfig/impersonator_config.go | 222 ++++++++ .../impersonator_config_test.go | 535 ++++++++++++++++++ .../controllermanager/prepare_controllers.go | 26 + .../concierge_impersonation_proxy_test.go | 83 ++- 9 files changed, 958 insertions(+), 104 deletions(-) create mode 100644 internal/controller/impersonatorconfig/impersonator_config.go create mode 100644 internal/controller/impersonatorconfig/impersonator_config_test.go diff --git a/deploy/concierge/rbac.yaml b/deploy/concierge/rbac.yaml index 503ad07a..3f3796ec 100644 --- a/deploy/concierge/rbac.yaml +++ b/deploy/concierge/rbac.yaml @@ -31,9 +31,12 @@ rules: resources: [ securitycontextconstraints ] verbs: [ use ] resourceNames: [ nonroot ] - - apiGroups: [""] - resources: ["users", "groups"] - verbs: ["impersonate"] + - apiGroups: [ "" ] + resources: [ "users", "groups" ] + verbs: [ "impersonate" ] + - apiGroups: [ "" ] + resources: [ nodes ] + verbs: [ list ] - apiGroups: - #@ pinnipedDevAPIGroupWithPrefix("config.concierge") resources: [ credentialissuers ] @@ -84,9 +87,12 @@ rules: - apiGroups: [ "" ] resources: [ pods/exec ] verbs: [ create ] - - apiGroups: [apps] - resources: [replicasets,deployments] - verbs: [get] + - apiGroups: [ apps ] + resources: [ replicasets,deployments ] + verbs: [ get ] + - apiGroups: [ "" ] + resources: [ configmaps ] + verbs: [ list, get, watch ] --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go index b2c2d10e..7ced9b66 100644 --- a/internal/concierge/impersonator/config.go +++ b/internal/concierge/impersonator/config.go @@ -55,17 +55,21 @@ type Config struct { TLS *TLSConfig `json:"tls,omitempty"` } -func FromConfigMap(configMap *v1.ConfigMap) (*Config, error) { +func NewConfig() *Config { + return &Config{Mode: ModeAuto} +} + +func ConfigFromConfigMap(configMap *v1.ConfigMap) (*Config, error) { stringConfig, ok := configMap.Data[ConfigMapDataKey] if !ok { return nil, fmt.Errorf(`ConfigMap is missing expected key "%s"`, ConfigMapDataKey) } - var config Config - if err := yaml.Unmarshal([]byte(stringConfig), &config); err != nil { + config := NewConfig() + if err := yaml.Unmarshal([]byte(stringConfig), config); err != nil { return nil, fmt.Errorf("decode yaml: %w", err) } - if config.Mode == "" { - config.Mode = ModeAuto // set the default value + if config.Mode != ModeAuto && config.Mode != ModeEnabled && config.Mode != ModeDisabled { + return nil, fmt.Errorf(`illegal value for "mode": %s`, config.Mode) } - return &config, nil + return config, nil } diff --git a/internal/concierge/impersonator/config_test.go b/internal/concierge/impersonator/config_test.go index 8406072b..9b12b6ba 100644 --- a/internal/concierge/impersonator/config_test.go +++ b/internal/concierge/impersonator/config_test.go @@ -13,7 +13,12 @@ import ( "go.pinniped.dev/internal/here" ) -func TestFromConfigMap(t *testing.T) { +func TestNewConfig(t *testing.T) { + // It defaults the mode. + require.Equal(t, &Config{Mode: ModeAuto}, NewConfig()) +} + +func TestConfigFromConfigMap(t *testing.T) { tests := []struct { name string configMap *v1.ConfigMap @@ -27,11 +32,11 @@ func TestFromConfigMap(t *testing.T) { ObjectMeta: metav1.ObjectMeta{}, Data: map[string]string{ "config.yaml": here.Doc(` - mode: enabled - endpoint: https://proxy.example.com:8443/ - tls: - certificateAuthoritySecretName: my-ca-crt - tlsSecretName: my-tls-certificate-and-key + mode: enabled + endpoint: https://proxy.example.com:8443/ + tls: + certificateAuthoritySecretName: my-ca-crt + tlsSecretName: my-tls-certificate-and-key `), }, }, @@ -59,6 +64,51 @@ func TestFromConfigMap(t *testing.T) { TLS: nil, }, }, + { + name: "valid config with mode enabled", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: enabled", + }, + }, + wantConfig: &Config{ + Mode: "enabled", + Endpoint: "", + TLS: nil, + }, + }, + { + name: "valid config with mode disabled", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: disabled", + }, + }, + wantConfig: &Config{ + Mode: "disabled", + Endpoint: "", + TLS: nil, + }, + }, + { + name: "valid config with mode auto", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: auto", + }, + }, + wantConfig: &Config{ + Mode: "auto", + Endpoint: "", + TLS: nil, + }, + }, { name: "wrong key in configmap", configMap: &v1.ConfigMap{ @@ -81,12 +131,23 @@ func TestFromConfigMap(t *testing.T) { }, wantError: "decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config", }, + { + name: "illegal value for mode in configmap", + configMap: &v1.ConfigMap{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Data: map[string]string{ + "config.yaml": "mode: unexpected-value", + }, + }, + wantError: `illegal value for "mode": unexpected-value`, + }, } for _, tt := range tests { test := tt t.Run(test.name, func(t *testing.T) { - config, err := FromConfigMap(test.configMap) + config, err := ConfigFromConfigMap(test.configMap) require.Equal(t, test.wantConfig, config) if test.wantError != "" { require.EqualError(t, err, test.wantError) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index c80957b1..0ded0510 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -33,13 +33,13 @@ var allowedHeaders = []string{ "Upgrade", } -type Proxy struct { +type proxy struct { cache *authncache.Cache proxy *httputil.ReverseProxy log logr.Logger } -func New(cache *authncache.Cache, log logr.Logger) (*Proxy, error) { +func New(cache *authncache.Cache, log logr.Logger) (http.Handler, error) { return newInternal(cache, log, func() (*rest.Config, error) { client, err := kubeclient.New() if err != nil { @@ -49,7 +49,7 @@ func New(cache *authncache.Cache, log logr.Logger) (*Proxy, error) { }) } -func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*rest.Config, error)) (*Proxy, error) { +func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*rest.Config, error)) (*proxy, error) { kubeconfig, err := getConfig() if err != nil { return nil, fmt.Errorf("could not get in-cluster config: %w", err) @@ -71,17 +71,17 @@ func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*re return nil, fmt.Errorf("could not get in-cluster transport: %w", err) } - proxy := httputil.NewSingleHostReverseProxy(serverURL) - proxy.Transport = kubeRoundTripper + reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) + reverseProxy.Transport = kubeRoundTripper - return &Proxy{ + return &proxy{ cache: cache, - proxy: proxy, + proxy: reverseProxy, log: log, }, nil } -func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { log := p.log.WithValues( "url", r.URL.String(), "method", r.Method, diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index 08bfb2bf..dbcafb0a 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -6,19 +6,10 @@ package server import ( "context" - "crypto/tls" - "crypto/x509/pkix" "fmt" "io" - "net/http" "time" - "k8s.io/apimachinery/pkg/util/intstr" - - v1 "k8s.io/api/core/v1" - - "go.pinniped.dev/internal/kubeclient" - "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -27,15 +18,11 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" - "k8s.io/klog/v2" - "k8s.io/klog/v2/klogr" loginapi "go.pinniped.dev/generated/1.20/apis/concierge/login" loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" - "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/certauthority/dynamiccertauthority" "go.pinniped.dev/internal/concierge/apiserver" - "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/internal/config/concierge" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/controllermanager" @@ -175,64 +162,6 @@ func (a *App) runServer(ctx context.Context) error { return fmt.Errorf("could not create aggregated API server: %w", err) } - client, err := kubeclient.New() - if err != nil { - plog.WarningErr("could not create client", err) - } else { - appNameLabel := cfg.Labels["app"] - loadBalancer := v1.Service{ - Spec: v1.ServiceSpec{ - Type: "LoadBalancer", - Ports: []v1.ServicePort{ - { - TargetPort: intstr.FromInt(8444), - Port: 443, - Protocol: v1.ProtocolTCP, - }, - }, - Selector: map[string]string{"app": appNameLabel}, - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "impersonation-proxy-load-balancer", - Namespace: podInfo.Namespace, - Labels: cfg.Labels, - }, - } - _, err = client.Kubernetes.CoreV1().Services(podInfo.Namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) - if err != nil { - plog.WarningErr("could not create load balancer", err) - } - } - - // run proxy handler - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) - if err != nil { - return fmt.Errorf("could not create impersonation CA: %w", err) - } - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, []string{"impersonation-proxy"}, nil, 24*time.Hour) - if err != nil { - return fmt.Errorf("could not create impersonation cert: %w", err) - } - impersonationProxy, err := impersonator.New(authenticators, klogr.New().WithName("impersonation-proxy")) - if err != nil { - return fmt.Errorf("could not create impersonation proxy: %w", err) - } - - impersonationProxyServer := http.Server{ - Addr: "0.0.0.0:8444", - Handler: impersonationProxy, - TLSConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, - Certificates: []tls.Certificate{*impersonationCert}, - }, - } - // todo store CA, cert etc. on the authenticator status - go func() { - if err := impersonationProxyServer.ListenAndServeTLS("", ""); err != nil { - klog.ErrorS(err, "could not serve impersonation proxy") - } - }() - // Run the server. Its post-start hook will start the controllers. return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go new file mode 100644 index 00000000..cfa4f5ca --- /dev/null +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -0,0 +1,222 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonatorconfig + +import ( + "crypto/tls" + "crypto/x509/pkix" + "errors" + "fmt" + "net" + "net/http" + "time" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/clusterhost" + "go.pinniped.dev/internal/concierge/impersonator" + pinnipedcontroller "go.pinniped.dev/internal/controller" + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/plog" +) + +const ( + impersonationProxyPort = ":8444" +) + +type impersonatorConfigController struct { + namespace string + configMapResourceName string + k8sClient kubernetes.Interface + configMapsInformer corev1informers.ConfigMapInformer + generatedLoadBalancerServiceName string + startTLSListenerFunc StartTLSListenerFunc + httpHandlerFactory func() (http.Handler, error) + + server *http.Server + hasControlPlaneNodes *bool +} + +type StartTLSListenerFunc func(network, listenAddress string, config *tls.Config) (net.Listener, error) + +func NewImpersonatorConfigController( + namespace string, + configMapResourceName string, + k8sClient kubernetes.Interface, + configMapsInformer corev1informers.ConfigMapInformer, + withInformer pinnipedcontroller.WithInformerOptionFunc, + withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, + generatedLoadBalancerServiceName string, + startTLSListenerFunc StartTLSListenerFunc, + httpHandlerFactory func() (http.Handler, error), +) controllerlib.Controller { + return controllerlib.New( + controllerlib.Config{ + Name: "impersonator-config-controller", + Syncer: &impersonatorConfigController{ + namespace: namespace, + configMapResourceName: configMapResourceName, + k8sClient: k8sClient, + configMapsInformer: configMapsInformer, + generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, + startTLSListenerFunc: startTLSListenerFunc, + httpHandlerFactory: httpHandlerFactory, + }, + }, + withInformer( + configMapsInformer, + pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(configMapResourceName, namespace), + controllerlib.InformerOption{}, + ), + // Be sure to run once even if the ConfigMap that the informer is watching doesn't exist. + withInitialEvent(controllerlib.Key{ + Namespace: namespace, + Name: configMapResourceName, + }), + ) +} + +func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { + plog.Info("impersonatorConfigController Sync") + + configMap, err := c.configMapsInformer.Lister().ConfigMaps(c.namespace).Get(c.configMapResourceName) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return fmt.Errorf("failed to get %s/%s configmap: %w", c.namespace, c.configMapResourceName, err) + } + + var config *impersonator.Config + if notFound { + plog.Info("Did not find impersonation proxy config: using default config values", + "configmap", c.configMapResourceName, + "namespace", c.namespace, + ) + config = impersonator.NewConfig() // use default configuration options + } else { + config, err = impersonator.ConfigFromConfigMap(configMap) + if err != nil { + return fmt.Errorf("invalid impersonator configuration: %v", err) + } + plog.Info("Read impersonation proxy config", + "configmap", c.configMapResourceName, + "namespace", c.namespace, + ) + } + + // Make a live API call to avoid the cost of having an informer watch all node changes on the cluster, + // since there could be lots and we don't especially care about node changes. + // Once we have concluded that there is or is not a visible control plane, then cache that decision + // to avoid listing nodes very often. + if c.hasControlPlaneNodes == nil { + hasControlPlaneNodes, err := clusterhost.New(c.k8sClient).HasControlPlaneNodes(ctx.Context) + if err != nil { + return err + } + c.hasControlPlaneNodes = &hasControlPlaneNodes + plog.Debug("Queried for control plane nodes", "foundControlPlaneNodes", hasControlPlaneNodes) + } + + if (config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes) || config.Mode == impersonator.ModeEnabled { + if err = c.startImpersonator(); err != nil { + return err + } + } else { + if err = c.stopImpersonator(); err != nil { + return err + } + } + + // TODO when the proxy is going to run, and the endpoint goes from being not specified to being specified, then the LoadBalancer is deleted + // TODO when the proxy is going to run, and when the endpoint goes from being specified to being not specified, then the LoadBalancer is created + // TODO when auto mode decides that the proxy should be disabled, then it also does not create the LoadBalancer (or it deletes it) + + // client, err := kubeclient.New() + // if err != nil { + // plog.WarningErr("could not create client", err) + // } else { + // appNameLabel := cfg.Labels["app"] + // loadBalancer := v1.Service{ + // Spec: v1.ServiceSpec{ + // Type: "LoadBalancer", + // Ports: []v1.ServicePort{ + // { + // TargetPort: intstr.FromInt(8444), + // Port: 443, + // Protocol: v1.ProtocolTCP, + // }, + // }, + // Selector: map[string]string{"app": appNameLabel}, + // }, + // ObjectMeta: metav1.ObjectMeta{ + // Name: "impersonation-proxy-load-balancer", + // Namespace: podInfo.Namespace, + // Labels: cfg.Labels, + // }, + // } + // _, err = client.Kubernetes.CoreV1().Services(podInfo.Namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) + // if err != nil { + // plog.WarningErr("could not create load balancer", err) + // } + // } + + return nil +} + +func (c *impersonatorConfigController) stopImpersonator() error { + if c.server != nil { + plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) + err := c.server.Close() + c.server = nil + if err != nil { + return err + } + } + return nil +} + +func (c *impersonatorConfigController) startImpersonator() error { + if c.server != nil { + return nil + } + + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) + if err != nil { + return fmt.Errorf("could not create impersonation CA: %w", err) + } + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, []string{"impersonation-proxy"}, nil, 24*time.Hour) + if err != nil { + return fmt.Errorf("could not create impersonation cert: %w", err) + } + + handler, err := c.httpHandlerFactory() + if err != nil { + return err + } + + listener, err := c.startTLSListenerFunc("tcp", impersonationProxyPort, &tls.Config{ + MinVersion: tls.VersionTLS12, // Allow v1.2 because clients like the default `curl` on MacOS don't support 1.3 yet. + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + return impersonationCert, nil + }, + }) + if err != nil { + return err + } + + c.server = &http.Server{Handler: handler} + + go func() { + plog.Info("Starting impersonation proxy", "port", impersonationProxyPort) + err = c.server.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + plog.Info("The impersonation proxy server has shut down") + } else { + plog.Error("Unexpected shutdown of the impersonation proxy server", err) + } + }() + return nil +} diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go new file mode 100644 index 00000000..65d35546 --- /dev/null +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -0,0 +1,535 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package impersonatorconfig + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + kubeinformers "k8s.io/client-go/informers" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/testutil" +) + +type tlsListenerWrapper struct { + listener net.Listener + closeError error +} + +func (t *tlsListenerWrapper) Accept() (net.Conn, error) { + return t.listener.Accept() +} + +func (t *tlsListenerWrapper) Close() error { + if t.closeError != nil { + // Really close the connection and then "pretend" that there was an error during close. + _ = t.listener.Close() + return t.closeError + } + return t.listener.Close() +} + +func (t *tlsListenerWrapper) Addr() net.Addr { + return t.listener.Addr() +} + +func TestImpersonatorConfigControllerOptions(t *testing.T) { + spec.Run(t, "options", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + const configMapResourceName = "some-configmap-resource-name" + const generatedLoadBalancerServiceName = "some-service-resource-name" + + var r *require.Assertions + var observableWithInformerOption *testutil.ObservableWithInformerOption + var observableWithInitialEventOption *testutil.ObservableWithInitialEventOption + var configMapsInformerFilter controllerlib.Filter + + it.Before(func() { + r = require.New(t) + observableWithInformerOption = testutil.NewObservableWithInformerOption() + observableWithInitialEventOption = testutil.NewObservableWithInitialEventOption() + configMapsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().ConfigMaps() + _ = NewImpersonatorConfigController( + installedInNamespace, + configMapResourceName, + nil, + configMapsInformer, + observableWithInformerOption.WithInformer, + observableWithInitialEventOption.WithInitialEvent, + generatedLoadBalancerServiceName, + nil, + nil, + ) + configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) + }) + + when("watching ConfigMap objects", func() { + var subject controllerlib.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.ConfigMap + + it.Before(func() { + subject = configMapsInformerFilter + target = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: configMapResourceName, Namespace: installedInNamespace}} + wrongNamespace = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: configMapResourceName, Namespace: "wrong-namespace"}} + wrongName = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target ConfigMap changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a ConfigMap from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a ConfigMap with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a ConfigMap with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + + when("starting up", func() { + it("asks for an initial event because the ConfigMap may not exist yet and it needs to run anyway", func() { + r.Equal(&controllerlib.Key{ + Namespace: installedInNamespace, + Name: configMapResourceName, + }, observableWithInitialEventOption.GetInitialEventKey()) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} + +func TestImpersonatorConfigControllerSync(t *testing.T) { + spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + const configMapResourceName = "some-configmap-resource-name" + const generatedLoadBalancerServiceName = "some-service-resource-name" + + var r *require.Assertions + + var subject controllerlib.Controller + var kubeAPIClient *kubernetesfake.Clientset + var kubeInformerClient *kubernetesfake.Clientset + var kubeInformers kubeinformers.SharedInformerFactory + var timeoutContext context.Context + var timeoutContextCancel context.CancelFunc + var syncContext *controllerlib.Context + var startTLSListenerFuncWasCalled int + var startTLSListenerFuncError error + var startTLSListenerUponCloseError error + var httpHanderFactoryFuncError error + var startedTLSListener net.Listener + + var startTLSListenerFunc = func(network, listenAddress string, config *tls.Config) (net.Listener, error) { + startTLSListenerFuncWasCalled++ + r.Equal("tcp", network) + r.Equal(":8444", listenAddress) + r.Equal(uint16(tls.VersionTLS12), config.MinVersion) + if startTLSListenerFuncError != nil { + return nil, startTLSListenerFuncError + } + var err error + //nolint: gosec // Intentionally binding to all network interfaces. + startedTLSListener, err = tls.Listen(network, ":0", config) // automatically choose the port for unit tests + r.NoError(err) + return &tlsListenerWrapper{listener: startedTLSListener, closeError: startTLSListenerUponCloseError}, nil + } + + var closeTLSListener = func() { + if startedTLSListener != nil { + err := startedTLSListener.Close() + // Ignore when the production code has already closed the server because there is nothing to + // clean up in that case. + if err != nil && !strings.Contains(err.Error(), "use of closed network connection") { + r.NoError(err) + } + } + } + + var requireTLSServerIsRunning = func() { + r.Greater(startTLSListenerFuncWasCalled, 0) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // TODO once we're using certs, do not skip verify + } + client := &http.Client{Transport: tr} + url := "https://" + startedTLSListener.Addr().String() + req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) + r.NoError(err) + resp, err := client.Do(req) + r.NoError(err) + + r.Equal(http.StatusOK, resp.StatusCode) + body, err := ioutil.ReadAll(resp.Body) + r.NoError(resp.Body.Close()) + r.NoError(err) + r.Equal("hello world", string(body)) + } + + var requireTLSServerIsNoLongerRunning = func() { + r.Greater(startTLSListenerFuncWasCalled, 0) + _, err := tls.Dial( + startedTLSListener.Addr().Network(), + startedTLSListener.Addr().String(), + &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // TODO once we're using certs, do not skip verify + ) + r.Error(err) + r.Regexp(`dial tcp \[::\]:[0-9]+: connect: connection refused`, err.Error()) + } + + var requireTLSServerWasNeverStarted = func() { + r.Equal(0, startTLSListenerFuncWasCalled) + } + + var waitForInformerCacheToSeeResourceVersion = func(informer cache.SharedIndexInformer, wantVersion string) { + r.Eventually(func() bool { + return informer.LastSyncResourceVersion() == wantVersion + }, 10*time.Second, time.Millisecond) + } + + // Defer starting the informers until the last possible moment so that the + // nested Before's can keep adding things to the informer caches. + var startInformersAndController = func() { + // Set this at the last second to allow for injection of server override. + subject = NewImpersonatorConfigController( + installedInNamespace, + configMapResourceName, + kubeAPIClient, + kubeInformers.Core().V1().ConfigMaps(), + controllerlib.WithInformer, + controllerlib.WithInitialEvent, + generatedLoadBalancerServiceName, + startTLSListenerFunc, + func() (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, err := fmt.Fprintf(w, "hello world") + r.NoError(err) + }), httpHanderFactoryFuncError + }, + ) + + // Set this at the last second to support calling subject.Name(). + syncContext = &controllerlib.Context{ + Context: timeoutContext, + Name: subject.Name(), + Key: controllerlib.Key{ + Namespace: installedInNamespace, + Name: configMapResourceName, + }, + } + + // Must start informers before calling TestRunSynchronously() + kubeInformers.Start(timeoutContext.Done()) + controllerlib.TestRunSynchronously(t, subject) + } + + var addImpersonatorConfigMapToTracker = func(resourceName, configYAML string) { + impersonatorConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Data: map[string]string{ + "config.yaml": configYAML, + }, + } + r.NoError(kubeInformerClient.Tracker().Add(impersonatorConfigMap)) + } + + var updateImpersonatorConfigMapInTracker = func(resourceName, configYAML, newResourceVersion string) { + impersonatorConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Different resource version compared to the initial version when this resource was created + // so we can tell when the informer cache has cached this newly updated version. + ResourceVersion: newResourceVersion, + }, + Data: map[string]string{ + "config.yaml": configYAML, + }, + } + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + impersonatorConfigMap, + installedInNamespace, + )) + } + + var addNodeWithRoleToTracker = func(role string) { + r.NoError(kubeAPIClient.Tracker().Add( + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + Labels: map[string]string{"kubernetes.io/node-role": role}, + }, + }, + )) + } + + it.Before(func() { + r = require.New(t) + + timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + + kubeInformerClient = kubernetesfake.NewSimpleClientset() + kubeInformers = kubeinformers.NewSharedInformerFactoryWithOptions(kubeInformerClient, 0, + kubeinformers.WithNamespace(installedInNamespace), + ) + kubeAPIClient = kubernetesfake.NewSimpleClientset() + }) + + it.After(func() { + timeoutContextCancel() + closeTLSListener() + }) + + when("the ConfigMap does not yet exist in the installation namespace or it was deleted (defaults to auto mode)", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker("some-other-ConfigMap", "foo: bar") + }) + + when("there are visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane") + }) + + it("does not start the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerWasNeverStarted() + }) + }) + + when("there are not visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + }) + + it("automatically starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + }) + }) + + when("sync is called more than once", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + }) + + it("only starts the impersonator once and only lists the cluster's nodes once", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal( + []coretesting.Action{ + coretesting.NewListAction( + schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, + schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, + "", + metav1.ListOptions{}), + }, + kubeAPIClient.Actions(), + ) + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + requireTLSServerIsRunning() // still running + r.Equal(1, len(kubeAPIClient.Actions())) // no new API calls + }) + }) + + when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { + it("returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "no nodes found") + requireTLSServerWasNeverStarted() + }) + }) + + when("the http handler factory function returns an error", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + httpHanderFactoryFuncError = errors.New("some factory error") + }) + + it("returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "some factory error") + requireTLSServerWasNeverStarted() + }) + }) + + when("the configmap is invalid", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "not yaml") + }) + + it("returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config") + requireTLSServerWasNeverStarted() + }) + }) + + when("the ConfigMap is already in the installation namespace", func() { + when("the configuration is auto mode with an endpoint", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` + mode: auto + endpoint: https://proxy.example.com:8443/ + `), + ) + }) + + when("there are visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane") + }) + + it("does not start the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerWasNeverStarted() + }) + }) + + when("there are not visible control plane nodes", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + }) + + it("starts the impersonator according to the settings in the ConfigMap", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + }) + }) + + when("the configuration is disabled mode", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: disabled") + addNodeWithRoleToTracker("worker") + }) + + it("does not start the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerWasNeverStarted() + }) + }) + + when("the configuration is enabled mode", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("control-plane") + }) + + it("starts the impersonator regardless of the visibility of control plane nodes", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + }) + }) + + when("the configuration switches from enabled to disabled mode", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("control-plane") + }) + + it("starts the impersonator, then shuts it down, then starts it again", func() { + startInformersAndController() + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsNoLongerRunning() + + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + + when("there is an error while shutting down the server", func() { + it.Before(func() { + startTLSListenerUponCloseError = errors.New("fake server close error") + }) + + it("returns the error from the sync function", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "fake server close error") + requireTLSServerIsNoLongerRunning() + }) + }) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 5f6c6d16..99b2012e 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -7,7 +7,9 @@ package controllermanager import ( "context" + "crypto/tls" "fmt" + "net/http" "time" "k8s.io/apimachinery/pkg/util/clock" @@ -19,12 +21,14 @@ import ( pinnipedclientset "go.pinniped.dev/generated/1.20/client/concierge/clientset/versioned" pinnipedinformers "go.pinniped.dev/generated/1.20/client/concierge/informers/externalversions" "go.pinniped.dev/internal/apiserviceref" + "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/internal/config/concierge" "go.pinniped.dev/internal/controller/apicerts" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/controller/authenticator/cachecleaner" "go.pinniped.dev/internal/controller/authenticator/jwtcachefiller" "go.pinniped.dev/internal/controller/authenticator/webhookcachefiller" + "go.pinniped.dev/internal/controller/impersonatorconfig" "go.pinniped.dev/internal/controller/issuerconfig" "go.pinniped.dev/internal/controller/kubecertagent" "go.pinniped.dev/internal/controllerlib" @@ -271,6 +275,28 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { klogr.New(), ), singletonWorker, + ). + + // The impersonation proxy configuration controllers dynamically configure the impersonation proxy feature. + WithController( + impersonatorconfig.NewImpersonatorConfigController( + c.ServerInstallationInfo.Namespace, + "pinniped-concierge-impersonation-proxy-config", // TODO this string should come from `c.NamesConfig` + client.Kubernetes, + informers.installationNamespaceK8s.Core().V1().ConfigMaps(), + controllerlib.WithInformer, + controllerlib.WithInitialEvent, + "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` + tls.Listen, + func() (http.Handler, error) { + impersonationProxyHandler, err := impersonator.New(c.AuthenticatorCache, klogr.New().WithName("impersonation-proxy")) + if err != nil { + return nil, fmt.Errorf("could not create impersonation proxy: %w", err) + } + return impersonationProxyHandler, nil + }, + ), + singletonWorker, ) // Return a function which starts the informers and controllers. diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index d1228707..6f53d93f 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -18,8 +18,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "sigs.k8s.io/yaml" loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/test/library" ) @@ -31,16 +33,16 @@ func TestImpersonationProxy(t *testing.T) { return } - ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() // Create a client using the admin kubeconfig. - // adminClient := library.NewClientset(t) + adminClient := library.NewKubernetesClientset(t) // Create a WebhookAuthenticator. authenticator := library.CreateTestWebhookAuthenticator(ctx, t) - // Find the address of the ClusterIP service. + // The address of the ClusterIP service that points at the impersonation proxy's port proxyServiceURL := fmt.Sprintf("https://%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) t.Logf("making kubeconfig that points to %q", proxyServiceURL) @@ -56,20 +58,89 @@ func TestImpersonationProxy(t *testing.T) { }, } - clientset, err := kubernetes.NewForConfig(kubeconfig) + impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") + // TODO if there is already a ConfigMap, remember its contents and delete it, which puts the proxy into its default settings + // TODO and in a t.Cleanup() if there was already a ConfigMap at the start of the test, then restore the original contents + + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + // Check that load balancer has been created + require.Eventually(t, func() bool { + return hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) + }, 10*time.Second, 500*time.Millisecond) + } else { + // Check that no load balancer has been created + require.Never(t, func() bool { + return hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) + }, 10*time.Second, 500*time.Millisecond) + + // Check that we can't use the impersonation proxy to execute kubectl commands yet + _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + require.EqualError(t, err, "Get \"https://pinniped-concierge-proxy.concierge.svc.cluster.local/api/v1/namespaces\": Service Unavailable") + + // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer) + configMap := configMapForConfig(t, impersonator.Config{ + Mode: impersonator.ModeEnabled, + Endpoint: proxyServiceURL, + TLS: nil, + }) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Cleanup(func() { + // TODO clean up the ConfigMap at the end of the test, and make sure that it happens before the t.Cleanup() above which is trying to restore the original ConfigMap + }) + } + t.Run( "access as user", - library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, clientset), + library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyClient), ) for _, group := range env.TestUser.ExpectedGroups { group := group t.Run( "access as group "+group, - library.AccessAsGroupTest(ctx, group, clientset), + library.AccessAsGroupTest(ctx, group, impersonationProxyClient), ) } + + // Update configuration to force the proxy to disabled mode + configMap := configMapForConfig(t, impersonator.Config{Mode: impersonator.ModeDisabled}) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) + require.NoError(t, err) + + // Check that we can't use the impersonation proxy to execute kubectl commands again + _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + require.EqualError(t, err, "Get \"https://pinniped-concierge-proxy.concierge.svc.cluster.local/api/v1/namespaces\": Service Unavailable") + + // if env.HasCapability(library.HasExternalLoadBalancerProvider) { + // TODO we started the test with a load balancer, so after forcing the proxy to disable, assert that the LoadBalancer was deleted + // } +} + +func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigMap { + configString, err := yaml.Marshal(config) + require.NoError(t, err) + configMap := corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "pinniped-concierge-impersonation-proxy-config"}, + Data: map[string]string{ + "config.yaml": string(configString), + }} + return configMap +} + +func hasLoadBalancerService(ctx context.Context, t *testing.T, client kubernetes.Interface, namespace string) bool { + t.Helper() + + services, err := client.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + for _, service := range services.Items { + if service.Spec.Type == corev1.ServiceTypeLoadBalancer { + return true + } + } + return false } func makeImpersonationTestToken(t *testing.T, authenticator corev1.TypedLocalObjectReference) string { From 6512ab1351030b984b65bbca3dca4b0eb593f5fc Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 11 Feb 2021 17:27:27 -0500 Subject: [PATCH 017/203] internal/concierge/impersonator: don't care about namespace Concierge APIs are no longer namespaced (see f015ad58528). Signed-off-by: Andrew Keesler --- internal/concierge/impersonator/impersonator.go | 1 - internal/concierge/impersonator/impersonator_test.go | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 0ded0510..409ae60f 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -95,7 +95,6 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } log = log.WithValues( "authenticator", tokenCredentialReq.Spec.Authenticator, - "authenticatorNamespace", tokenCredentialReq.Namespace, ) userInfo, err := p.cache.AuthenticateTokenCredentialRequest(r.Context(), tokenCredentialReq) diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 72f38e17..449a2419 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -146,7 +146,7 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("", "", "")}}), wantHTTPBody: "invalid token\n", wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"\"} \"authenticatorNamespace\"=\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token authenticates as nil", @@ -157,7 +157,7 @@ func TestImpersonator(t *testing.T) { }, wantHTTPBody: "not authenticated\n", wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"level\"=0 \"msg\"=\"received token that did not authenticate\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"authenticatorNamespace\"=\"foo\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"level\"=0 \"msg\"=\"received token that did not authenticate\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, // happy path { @@ -179,7 +179,7 @@ func TestImpersonator(t *testing.T) { }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, - wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"authenticatorNamespace\"=\"foo\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, + wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, }, } From 25bc8dd8a98ac9dbdd69a130b02f9a11275c0b7c Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Mon, 15 Feb 2021 18:04:12 -0500 Subject: [PATCH 018/203] test/integration: hopefully fix TestImpersonationProxy I think we were assuming the name of our Concierge app, and getting lucky because it was the name we use when testing locally (but not in CI). Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 6f53d93f..979060ed 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -64,6 +64,7 @@ func TestImpersonationProxy(t *testing.T) { // TODO if there is already a ConfigMap, remember its contents and delete it, which puts the proxy into its default settings // TODO and in a t.Cleanup() if there was already a ConfigMap at the start of the test, then restore the original contents + serviceUnavailableError := fmt.Sprintf(`Get "%s/api/v1/namespaces": Service Unavailable`, proxyServiceURL) if env.HasCapability(library.HasExternalLoadBalancerProvider) { // Check that load balancer has been created require.Eventually(t, func() bool { @@ -77,7 +78,7 @@ func TestImpersonationProxy(t *testing.T) { // Check that we can't use the impersonation proxy to execute kubectl commands yet _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - require.EqualError(t, err, "Get \"https://pinniped-concierge-proxy.concierge.svc.cluster.local/api/v1/namespaces\": Service Unavailable") + require.EqualError(t, err, serviceUnavailableError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer) configMap := configMapForConfig(t, impersonator.Config{ @@ -112,7 +113,7 @@ func TestImpersonationProxy(t *testing.T) { // Check that we can't use the impersonation proxy to execute kubectl commands again _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - require.EqualError(t, err, "Get \"https://pinniped-concierge-proxy.concierge.svc.cluster.local/api/v1/namespaces\": Service Unavailable") + require.EqualError(t, err, serviceUnavailableError) // if env.HasCapability(library.HasExternalLoadBalancerProvider) { // TODO we started the test with a load balancer, so after forcing the proxy to disable, assert that the LoadBalancer was deleted From fdd8ef58353d152cd5ec8c97b7ee360a112d483b Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Mon, 15 Feb 2021 18:00:10 -0500 Subject: [PATCH 019/203] internal/concierge/impersonator: handle custom login API group Signed-off-by: Andrew Keesler --- cmd/pinniped/cmd/login_oidc.go | 1 + .../concierge/impersonator/impersonator.go | 42 ++-- .../impersonator/impersonator_test.go | 148 ++++++++++---- internal/concierge/scheme/scheme.go | 113 +++++++++++ internal/concierge/scheme/scheme_test.go | 184 ++++++++++++++++++ internal/concierge/server/server.go | 120 +++--------- internal/concierge/server/server_test.go | 173 ---------------- .../controllermanager/prepare_controllers.go | 7 +- .../impersonationtoken/impersonationtoken.go | 67 +++++++ .../concierge_impersonation_proxy_test.go | 27 +-- 10 files changed, 529 insertions(+), 353 deletions(-) create mode 100644 internal/concierge/scheme/scheme.go create mode 100644 internal/concierge/scheme/scheme_test.go create mode 100644 internal/testutil/impersonationtoken/impersonationtoken.go diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 4d04d5f5..ceb60c74 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -267,6 +267,7 @@ func execCredentialForImpersonationProxy( tokenExpiry *metav1.Time, ) (*clientauthv1beta1.ExecCredential, error) { // TODO maybe de-dup this with conciergeclient.go + // TODO reuse code from internal/testutil/impersonationtoken here to create token var kind string switch strings.ToLower(conciergeAuthenticatorType) { case "webhook": diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 409ae60f..710df348 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -5,7 +5,6 @@ package impersonator import ( "encoding/base64" - "encoding/json" "fmt" "net/http" "net/http/httputil" @@ -13,12 +12,12 @@ import ( "strings" "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/rest" "k8s.io/client-go/transport" "go.pinniped.dev/generated/1.20/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/kubeclient" ) @@ -34,13 +33,14 @@ var allowedHeaders = []string{ } type proxy struct { - cache *authncache.Cache - proxy *httputil.ReverseProxy - log logr.Logger + cache *authncache.Cache + jsonDecoder runtime.Decoder + proxy *httputil.ReverseProxy + log logr.Logger } -func New(cache *authncache.Cache, log logr.Logger) (http.Handler, error) { - return newInternal(cache, log, func() (*rest.Config, error) { +func New(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr.Logger) (http.Handler, error) { + return newInternal(cache, jsonDecoder, log, func() (*rest.Config, error) { client, err := kubeclient.New() if err != nil { return nil, err @@ -49,7 +49,7 @@ func New(cache *authncache.Cache, log logr.Logger) (http.Handler, error) { }) } -func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*rest.Config, error)) (*proxy, error) { +func newInternal(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr.Logger, getConfig func() (*rest.Config, error)) (*proxy, error) { kubeconfig, err := getConfig() if err != nil { return nil, fmt.Errorf("could not get in-cluster config: %w", err) @@ -75,9 +75,10 @@ func newInternal(cache *authncache.Cache, log logr.Logger, getConfig func() (*re reverseProxy.Transport = kubeRoundTripper return &proxy{ - cache: cache, - proxy: reverseProxy, - log: log, + cache: cache, + jsonDecoder: jsonDecoder, + proxy: reverseProxy, + log: log, }, nil } @@ -87,7 +88,7 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { "method", r.Method, ) - tokenCredentialReq, err := extractToken(r) + tokenCredentialReq, err := extractToken(r, p.jsonDecoder) if err != nil { log.Error(err, "invalid token encoding") http.Error(w, "invalid token encoding", http.StatusBadRequest) @@ -134,7 +135,7 @@ func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header return newHeaders } -func extractToken(req *http.Request) (*login.TokenCredentialRequest, error) { +func extractToken(req *http.Request, jsonDecoder runtime.Decoder) (*login.TokenCredentialRequest, error) { authHeader := req.Header.Get("Authorization") if authHeader == "" { return nil, fmt.Errorf("missing authorization header") @@ -148,13 +149,14 @@ func extractToken(req *http.Request) (*login.TokenCredentialRequest, error) { return nil, fmt.Errorf("invalid base64 in encoded bearer token: %w", err) } - var v1alpha1Req loginv1alpha1.TokenCredentialRequest - if err := json.Unmarshal(tokenCredentialRequestJSON, &v1alpha1Req); err != nil { - return nil, fmt.Errorf("invalid TokenCredentialRequest encoded in bearer token: %w", err) + obj, err := runtime.Decode(jsonDecoder, tokenCredentialRequestJSON) + if err != nil { + return nil, fmt.Errorf("invalid object encoded in bearer token: %w", err) } - var internalReq login.TokenCredentialRequest - if err := loginv1alpha1.Convert_v1alpha1_TokenCredentialRequest_To_login_TokenCredentialRequest(&v1alpha1Req, &internalReq, nil); err != nil { - return nil, fmt.Errorf("failed to convert v1alpha1 TokenCredentialRequest to internal version: %w", err) + tokenCredentialRequest, ok := obj.(*login.TokenCredentialRequest) + if !ok { + return nil, fmt.Errorf("invalid TokenCredentialRequest encoded in bearer token: got %T", obj) } - return &internalReq, nil + + return tokenCredentialRequest, nil } diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 449a2419..94a46f28 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -5,8 +5,6 @@ package impersonator import ( "context" - "encoding/base64" - "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -16,20 +14,31 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd/api" - loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" + "go.pinniped.dev/generated/1.20/apis/concierge/login" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/controller/authenticator/authncache" + "go.pinniped.dev/internal/groupsuffix" "go.pinniped.dev/internal/mocks/mocktokenauthenticator" "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/internal/testutil/impersonationtoken" "go.pinniped.dev/internal/testutil/testlogger" ) func TestImpersonator(t *testing.T) { + const ( + defaultAPIGroup = "pinniped.dev" + customAPIGroup = "walrus.tld" + ) + validURL, _ := url.Parse("http://pinniped.dev/blah") testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { // Expect that the request is authenticated based on the kubeconfig credential. @@ -61,15 +70,25 @@ func TestImpersonator(t *testing.T) { return r } + goodAuthenticator := corev1.TypedLocalObjectReference{ + Name: "authenticator-one", + APIGroup: stringPtr(authenticationv1alpha1.GroupName), + } + badAuthenticator := corev1.TypedLocalObjectReference{ + Name: "", + APIGroup: stringPtr(authenticationv1alpha1.GroupName), + } + tests := []struct { - name string - getKubeconfig func() (*rest.Config, error) - wantCreationErr string - request *http.Request - wantHTTPBody string - wantHTTPStatus int - wantLogs []string - expectMockToken func(*testing.T, *mocktokenauthenticator.MockTokenMockRecorder) + name string + apiGroupOverride string + getKubeconfig func() (*rest.Config, error) + wantCreationErr string + request *http.Request + wantHTTPBody string + wantHTTPStatus int + wantLogs []string + expectMockToken func(*testing.T, *mocktokenauthenticator.MockTokenMockRecorder) }{ { name: "fail to get in-cluster config", @@ -119,7 +138,7 @@ func TestImpersonator(t *testing.T) { { name: "authorization header missing bearer prefix", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: newRequest(map[string][]string{"Authorization": {makeTestTokenRequest("foo", "authenticator-one", "test-token")}}), + request: newRequest(map[string][]string{"Authorization": {impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, wantLogs: []string{"\"error\"=\"authorization header must be of type Bearer\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, @@ -138,33 +157,50 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{"Authorization": {"Bearer abc"}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"invalid TokenCredentialRequest encoded in bearer token: invalid character 'i' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: couldn't get version/kind; json parse error: invalid character 'i' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "base64 encoded token is encoded with default api group but we are expecting custom api group", + apiGroupOverride: customAPIGroup, + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), + wantHTTPBody: "invalid token encoding\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.pinniped.dev/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "base64 encoded token is encoded with custom api group but we are expecting default api group", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}}), + wantHTTPBody: "invalid token encoding\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.walrus.tld/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token could not be authenticated", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: newRequest(map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("", "", "")}}), + request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "", &badAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token\n", wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token authenticates as nil", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: newRequest(map[string][]string{"Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}}), + request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { recorder.AuthenticateToken(gomock.Any(), "test-token").Return(nil, false, nil) }, wantHTTPBody: "not authenticated\n", wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"level\"=0 \"msg\"=\"received token that did not authenticate\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"level\"=0 \"msg\"=\"received token that did not authenticate\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, // happy path { name: "token validates", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + makeTestTokenRequest("foo", "authenticator-one", "test-token")}, + "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, "Malicious-Header": {"test-header-value-1"}, "User-Agent": {"test-user-agent"}, }), @@ -179,7 +215,29 @@ func TestImpersonator(t *testing.T) { }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, - wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":null,\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, + wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, + }, + { + name: "token validates with custom api group", + apiGroupOverride: customAPIGroup, + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{ + "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}, + "Malicious-Header": {"test-header-value-1"}, + "User-Agent": {"test-user-agent"}, + }), + expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { + userInfo := user.DefaultInfo{ + Name: "test-user", + Groups: []string{"test-group-1", "test-group-2"}, + UID: "test-uid", + } + response := &authenticator.Response{User: &userInfo} + recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, }, } @@ -187,11 +245,19 @@ func TestImpersonator(t *testing.T) { tt := tt testLog := testlogger.New(t) t.Run(tt.name, func(t *testing.T) { + defer func() { + if t.Failed() { + for i, line := range testLog.Lines() { + t.Logf("testLog line %d: %q", i+1, line) + } + } + }() + // stole this from cache_test, hopefully it is sufficient cacheWithMockAuthenticator := authncache.New() ctrl := gomock.NewController(t) defer ctrl.Finish() - key := authncache.Key{Name: "authenticator-one"} + key := authncache.Key{Name: "authenticator-one", APIGroup: *goodAuthenticator.APIGroup} mockToken := mocktokenauthenticator.NewMockToken(ctrl) cacheWithMockAuthenticator.Store(key, mockToken) @@ -199,7 +265,12 @@ func TestImpersonator(t *testing.T) { tt.expectMockToken(t, mockToken.EXPECT()) } - proxy, err := newInternal(cacheWithMockAuthenticator, testLog, tt.getKubeconfig) + apiGroup := defaultAPIGroup + if tt.apiGroupOverride != "" { + apiGroup = tt.apiGroupOverride + } + + proxy, err := newInternal(cacheWithMockAuthenticator, makeDecoder(t, apiGroup), testLog, tt.getKubeconfig) if tt.wantCreationErr != "" { require.EqualError(t, err, tt.wantCreationErr) return @@ -223,22 +294,21 @@ func TestImpersonator(t *testing.T) { } } -func makeTestTokenRequest(namespace string, name string, token string) string { - reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginv1alpha1.GroupName + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: token, - Authenticator: corev1.TypedLocalObjectReference{Name: name}, - }, +func stringPtr(s string) *string { return &s } + +func makeDecoder(t *testing.T, apiGroupSuffix string) runtime.Decoder { + t.Helper() + + loginConciergeGroupName, ok := groupsuffix.Replace(login.GroupName, apiGroupSuffix) + require.True(t, ok, "couldn't replace suffix of %q with %q", login.GroupName, apiGroupSuffix) + + scheme := conciergescheme.New(loginConciergeGroupName, apiGroupSuffix) + codecs := serializer.NewCodecFactory(scheme) + respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) + require.True(t, ok, "couldn't find serializer info for media type") + + return codecs.DecoderToVersion(respInfo.Serializer, schema.GroupVersion{ + Group: loginConciergeGroupName, + Version: login.SchemeGroupVersion.Version, }) - if err != nil { - panic(err) - } - return base64.RawURLEncoding.EncodeToString(reqJSON) } diff --git a/internal/concierge/scheme/scheme.go b/internal/concierge/scheme/scheme.go new file mode 100644 index 00000000..480d6f65 --- /dev/null +++ b/internal/concierge/scheme/scheme.go @@ -0,0 +1,113 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package scheme contains code to construct a proper runtime.Scheme for the Concierge aggregated +// API. +package scheme + +import ( + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + loginapi "go.pinniped.dev/generated/1.20/apis/concierge/login" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/groupsuffix" + "go.pinniped.dev/internal/plog" +) + +// New returns a runtime.Scheme for use by the Concierge aggregated API. The provided +// loginConciergeAPIGroup should be the API group that the Concierge is serving (e.g., +// login.concierge.pinniped.dev, login.concierge.walrus.tld, etc.). The provided apiGroupSuffix is +// the API group suffix of the provided loginConciergeAPIGroup (e.g., pinniped.dev, walrus.tld, +// etc.). +func New(loginConciergeAPIGroup, apiGroupSuffix string) *runtime.Scheme { + // standard set up of the server side scheme + scheme := runtime.NewScheme() + + // add the options to empty v1 + metav1.AddToGroupVersion(scheme, metav1.Unversioned) + + // nothing fancy is required if using the standard group + if loginConciergeAPIGroup == loginv1alpha1.GroupName { + utilruntime.Must(loginv1alpha1.AddToScheme(scheme)) + utilruntime.Must(loginapi.AddToScheme(scheme)) + return scheme + } + + // we need a temporary place to register our types to avoid double registering them + tmpScheme := runtime.NewScheme() + utilruntime.Must(loginv1alpha1.AddToScheme(tmpScheme)) + utilruntime.Must(loginapi.AddToScheme(tmpScheme)) + + for gvk := range tmpScheme.AllKnownTypes() { + if gvk.GroupVersion() == metav1.Unversioned { + continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore + } + + if gvk.Group != loginv1alpha1.GroupName { + panic("tmp scheme has types not in the aggregated API group: " + gvk.Group) // programmer error + } + + obj, err := tmpScheme.New(gvk) + if err != nil { + panic(err) // programmer error, scheme internal code is broken + } + newGVK := schema.GroupVersionKind{ + Group: loginConciergeAPIGroup, + Version: gvk.Version, + Kind: gvk.Kind, + } + + // register the existing type but with the new group in the correct scheme + scheme.AddKnownTypeWithName(newGVK, obj) + } + + // manually register conversions and defaulting into the correct scheme since we cannot directly call loginv1alpha1.AddToScheme + utilruntime.Must(loginv1alpha1.RegisterConversions(scheme)) + utilruntime.Must(loginv1alpha1.RegisterDefaults(scheme)) + + // we do not want to return errors from the scheme and instead would prefer to defer + // to the REST storage layer for consistency. The simplest way to do this is to force + // a cache miss from the authenticator cache. Kube API groups are validated via the + // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never + // to be in the authenticator cache. Add a timestamp just to be extra sure. + const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" + authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() + + // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest + // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites + // any previously registered defaulting function. Thus to make sure that we catch + // a situation where we add a defaulting func, we attempt to call it here with a nil + // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no + // defaulting func registered, but it will almost certainly panic if one is added. + scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) + + // on incoming requests, restore the authenticator API group to the standard group + // note that we are responsible for duplicating this logic for every external API version + scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { + credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) + + if credentialRequest.Spec.Authenticator.APIGroup == nil { + // force a cache miss because this is an invalid request + plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) + credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss + return + } + + restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) + if !ok { + // force a cache miss because this is an invalid request + plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) + credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss + return + } + + credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup + }) + + return scheme +} diff --git a/internal/concierge/scheme/scheme_test.go b/internal/concierge/scheme/scheme_test.go new file mode 100644 index 00000000..7f2c3be5 --- /dev/null +++ b/internal/concierge/scheme/scheme_test.go @@ -0,0 +1,184 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package scheme + +import ( + "reflect" + "strings" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + loginapi "go.pinniped.dev/generated/1.20/apis/concierge/login" + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + "go.pinniped.dev/internal/groupsuffix" +) + +func TestNew(t *testing.T) { + // the standard group + regularGV := schema.GroupVersion{ + Group: "login.concierge.pinniped.dev", + Version: "v1alpha1", + } + regularGVInternal := schema.GroupVersion{ + Group: "login.concierge.pinniped.dev", + Version: runtime.APIVersionInternal, + } + + // the canonical other group + otherGV := schema.GroupVersion{ + Group: "login.concierge.walrus.tld", + Version: "v1alpha1", + } + otherGVInternal := schema.GroupVersion{ + Group: "login.concierge.walrus.tld", + Version: runtime.APIVersionInternal, + } + + // kube's core internal + internalGV := schema.GroupVersion{ + Group: "", + Version: runtime.APIVersionInternal, + } + + tests := []struct { + name string + apiGroupSuffix string + want map[schema.GroupVersionKind]reflect.Type + }{ + { + name: "regular api group", + apiGroupSuffix: "pinniped.dev", + want: map[schema.GroupVersionKind]reflect.Type{ + // all the types that are in the aggregated API group + + regularGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), + regularGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), + + regularGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), + regularGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), + + regularGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + regularGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + regularGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + regularGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + regularGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + regularGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + regularGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + regularGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + + regularGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + // the types below this line do not really matter to us because they are in the core group + + internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), + metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), + metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), + metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), + metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), + metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + }, + }, + { + name: "other api group", + apiGroupSuffix: "walrus.tld", + want: map[schema.GroupVersionKind]reflect.Type{ + // all the types that are in the aggregated API group + + otherGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), + otherGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), + + otherGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), + otherGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), + + otherGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + otherGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + otherGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + otherGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + otherGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + otherGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + otherGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + otherGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + + otherGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + // the types below this line do not really matter to us because they are in the core group + + internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), + + metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), + metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), + metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), + metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), + metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), + metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), + metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), + metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), + metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), + metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), + metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), + metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), + metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + loginConciergeAPIGroup, ok := groupsuffix.Replace("login.concierge.pinniped.dev", tt.apiGroupSuffix) + require.True(t, ok) + + scheme := New(loginConciergeAPIGroup, tt.apiGroupSuffix) + require.Equal(t, tt.want, scheme.AllKnownTypes()) + + // make a credential request like a client would send + authenticationConciergeAPIGroup := "authentication.concierge." + tt.apiGroupSuffix + credentialRequest := &loginv1alpha1.TokenCredentialRequest{ + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &authenticationConciergeAPIGroup, + }, + }, + } + + // run defaulting on it + scheme.Default(credentialRequest) + + // make sure the group is restored if needed + require.Equal(t, "authentication.concierge.pinniped.dev", *credentialRequest.Spec.Authenticator.APIGroup) + + // make a credential request in the standard group + defaultAuthenticationConciergeAPIGroup := "authentication.concierge.pinniped.dev" + defaultCredentialRequest := &loginv1alpha1.TokenCredentialRequest{ + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &defaultAuthenticationConciergeAPIGroup, + }, + }, + } + + // run defaulting on it + scheme.Default(defaultCredentialRequest) + + if tt.apiGroupSuffix == "pinniped.dev" { // when using the standard group, this should just work + require.Equal(t, "authentication.concierge.pinniped.dev", *defaultCredentialRequest.Spec.Authenticator.APIGroup) + } else { // when using any other group, this should always be a cache miss + require.True(t, strings.HasPrefix(*defaultCredentialRequest.Spec.Authenticator.APIGroup, "_INVALID_API_GROUP_2")) + } + }) + } +} diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index dbcafb0a..00b75ef3 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -11,18 +11,17 @@ import ( "time" "github.com/spf13/cobra" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" - loginapi "go.pinniped.dev/generated/1.20/apis/concierge/login" + "go.pinniped.dev/generated/1.20/apis/concierge/login" loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/certauthority/dynamiccertauthority" "go.pinniped.dev/internal/concierge/apiserver" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/config/concierge" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/controllermanager" @@ -123,6 +122,14 @@ func (a *App) runServer(ctx context.Context) error { // cert issuer used to issue certs to Pinniped clients wishing to login. dynamicSigningCertProvider := dynamiccert.New() + // Get the "real" name of the login concierge API group (i.e., the API group name with the + // injected suffix). + loginConciergeAPIGroup, ok := groupsuffix.Replace(loginv1alpha1.GroupName, *cfg.APIGroupSuffix) + if !ok { + return fmt.Errorf("cannot make api group from %s/%s", loginv1alpha1.GroupName, *cfg.APIGroupSuffix) + } + loginConciergeScheme := conciergescheme.New(loginConciergeAPIGroup, *cfg.APIGroupSuffix) + // Prepare to start the controllers, but defer actually starting them until the // post start hook of the aggregated API server. startControllersFunc, err := controllermanager.PrepareControllers( @@ -138,6 +145,7 @@ func (a *App) runServer(ctx context.Context) error { ServingCertDuration: time.Duration(*cfg.APIConfig.ServingCertificateConfig.DurationSeconds) * time.Second, ServingCertRenewBefore: time.Duration(*cfg.APIConfig.ServingCertificateConfig.RenewBeforeSeconds) * time.Second, AuthenticatorCache: authenticators, + LoginJSONDecoder: getLoginJSONDecoder(loginConciergeAPIGroup, loginConciergeScheme), }, ) if err != nil { @@ -150,7 +158,8 @@ func (a *App) runServer(ctx context.Context) error { authenticators, dynamiccertauthority.New(dynamicSigningCertProvider), startControllersFunc, - *cfg.APIGroupSuffix, + loginConciergeAPIGroup, + loginConciergeScheme, ) if err != nil { return fmt.Errorf("could not configure aggregated API server: %w", err) @@ -172,14 +181,10 @@ func getAggregatedAPIServerConfig( authenticator credentialrequest.TokenCredentialRequestAuthenticator, issuer credentialrequest.CertIssuer, startControllersPostStartHook func(context.Context), - apiGroupSuffix string, + loginConciergeAPIGroup string, + loginConciergeScheme *runtime.Scheme, ) (*apiserver.Config, error) { - loginConciergeAPIGroup, ok := groupsuffix.Replace(loginv1alpha1.GroupName, apiGroupSuffix) - if !ok { - return nil, fmt.Errorf("cannot make api group from %s/%s", loginv1alpha1.GroupName, apiGroupSuffix) - } - - scheme := getAggregatedAPIServerScheme(loginConciergeAPIGroup, apiGroupSuffix) + scheme := loginConciergeScheme codecs := serializer.NewCodecFactory(scheme) defaultEtcdPathPrefix := fmt.Sprintf("/registry/%s", loginConciergeAPIGroup) @@ -224,90 +229,15 @@ func getAggregatedAPIServerConfig( return apiServerConfig, nil } -func getAggregatedAPIServerScheme(loginConciergeAPIGroup, apiGroupSuffix string) *runtime.Scheme { - // standard set up of the server side scheme - scheme := runtime.NewScheme() - - // add the options to empty v1 - metav1.AddToGroupVersion(scheme, metav1.Unversioned) - - // nothing fancy is required if using the standard group - if loginConciergeAPIGroup == loginv1alpha1.GroupName { - utilruntime.Must(loginv1alpha1.AddToScheme(scheme)) - utilruntime.Must(loginapi.AddToScheme(scheme)) - return scheme +func getLoginJSONDecoder(loginConciergeAPIGroup string, loginConciergeScheme *runtime.Scheme) runtime.Decoder { + scheme := loginConciergeScheme + codecs := serializer.NewCodecFactory(scheme) + respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) + if !ok { + panic(fmt.Errorf("unknown content type: %s ", runtime.ContentTypeJSON)) // static input, programmer error } - - // we need a temporary place to register our types to avoid double registering them - tmpScheme := runtime.NewScheme() - utilruntime.Must(loginv1alpha1.AddToScheme(tmpScheme)) - utilruntime.Must(loginapi.AddToScheme(tmpScheme)) - - for gvk := range tmpScheme.AllKnownTypes() { - if gvk.GroupVersion() == metav1.Unversioned { - continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore - } - - if gvk.Group != loginv1alpha1.GroupName { - panic("tmp scheme has types not in the aggregated API group: " + gvk.Group) // programmer error - } - - obj, err := tmpScheme.New(gvk) - if err != nil { - panic(err) // programmer error, scheme internal code is broken - } - newGVK := schema.GroupVersionKind{ - Group: loginConciergeAPIGroup, - Version: gvk.Version, - Kind: gvk.Kind, - } - - // register the existing type but with the new group in the correct scheme - scheme.AddKnownTypeWithName(newGVK, obj) - } - - // manually register conversions and defaulting into the correct scheme since we cannot directly call loginv1alpha1.AddToScheme - utilruntime.Must(loginv1alpha1.RegisterConversions(scheme)) - utilruntime.Must(loginv1alpha1.RegisterDefaults(scheme)) - - // we do not want to return errors from the scheme and instead would prefer to defer - // to the REST storage layer for consistency. The simplest way to do this is to force - // a cache miss from the authenticator cache. Kube API groups are validated via the - // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never - // to be in the authenticator cache. Add a timestamp just to be extra sure. - const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" - authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() - - // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest - // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites - // any previously registered defaulting function. Thus to make sure that we catch - // a situation where we add a defaulting func, we attempt to call it here with a nil - // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no - // defaulting func registered, but it will almost certainly panic if one is added. - scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) - - // on incoming requests, restore the authenticator API group to the standard group - // note that we are responsible for duplicating this logic for every external API version - scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { - credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) - - if credentialRequest.Spec.Authenticator.APIGroup == nil { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - if !ok { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup + return codecs.DecoderToVersion(respInfo.Serializer, schema.GroupVersion{ + Group: loginConciergeAPIGroup, + Version: login.SchemeGroupVersion.Version, }) - - return scheme } diff --git a/internal/concierge/server/server_test.go b/internal/concierge/server/server_test.go index 258e914e..0825cf5b 100644 --- a/internal/concierge/server/server_test.go +++ b/internal/concierge/server/server_test.go @@ -6,21 +6,12 @@ package server import ( "bytes" "context" - "reflect" "strings" "testing" "github.com/google/go-cmp/cmp" "github.com/spf13/cobra" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - - loginapi "go.pinniped.dev/generated/1.20/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" - "go.pinniped.dev/internal/groupsuffix" ) const knownGoodUsage = ` @@ -97,167 +88,3 @@ func TestCommand(t *testing.T) { }) } } - -func Test_getAggregatedAPIServerScheme(t *testing.T) { - // the standard group - regularGV := schema.GroupVersion{ - Group: "login.concierge.pinniped.dev", - Version: "v1alpha1", - } - regularGVInternal := schema.GroupVersion{ - Group: "login.concierge.pinniped.dev", - Version: runtime.APIVersionInternal, - } - - // the canonical other group - otherGV := schema.GroupVersion{ - Group: "login.concierge.walrus.tld", - Version: "v1alpha1", - } - otherGVInternal := schema.GroupVersion{ - Group: "login.concierge.walrus.tld", - Version: runtime.APIVersionInternal, - } - - // kube's core internal - internalGV := schema.GroupVersion{ - Group: "", - Version: runtime.APIVersionInternal, - } - - tests := []struct { - name string - apiGroupSuffix string - want map[schema.GroupVersionKind]reflect.Type - }{ - { - name: "regular api group", - apiGroupSuffix: "pinniped.dev", - want: map[schema.GroupVersionKind]reflect.Type{ - // all the types that are in the aggregated API group - - regularGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), - regularGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), - - regularGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), - regularGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), - - regularGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - regularGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - regularGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - regularGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - regularGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - regularGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - regularGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - regularGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - - regularGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - // the types below this line do not really matter to us because they are in the core group - - internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), - metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), - metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), - metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), - metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), - metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - }, - }, - { - name: "other api group", - apiGroupSuffix: "walrus.tld", - want: map[schema.GroupVersionKind]reflect.Type{ - // all the types that are in the aggregated API group - - otherGV.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequest{}).Elem(), - otherGV.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginv1alpha1.TokenCredentialRequestList{}).Elem(), - - otherGVInternal.WithKind("TokenCredentialRequest"): reflect.TypeOf(&loginapi.TokenCredentialRequest{}).Elem(), - otherGVInternal.WithKind("TokenCredentialRequestList"): reflect.TypeOf(&loginapi.TokenCredentialRequestList{}).Elem(), - - otherGV.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - otherGV.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - otherGV.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - otherGV.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - otherGV.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - otherGV.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - otherGV.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - otherGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - - otherGVInternal.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - // the types below this line do not really matter to us because they are in the core group - - internalGV.WithKind("WatchEvent"): reflect.TypeOf(&metav1.InternalEvent{}).Elem(), - - metav1.Unversioned.WithKind("APIGroup"): reflect.TypeOf(&metav1.APIGroup{}).Elem(), - metav1.Unversioned.WithKind("APIGroupList"): reflect.TypeOf(&metav1.APIGroupList{}).Elem(), - metav1.Unversioned.WithKind("APIResourceList"): reflect.TypeOf(&metav1.APIResourceList{}).Elem(), - metav1.Unversioned.WithKind("APIVersions"): reflect.TypeOf(&metav1.APIVersions{}).Elem(), - metav1.Unversioned.WithKind("CreateOptions"): reflect.TypeOf(&metav1.CreateOptions{}).Elem(), - metav1.Unversioned.WithKind("DeleteOptions"): reflect.TypeOf(&metav1.DeleteOptions{}).Elem(), - metav1.Unversioned.WithKind("ExportOptions"): reflect.TypeOf(&metav1.ExportOptions{}).Elem(), - metav1.Unversioned.WithKind("GetOptions"): reflect.TypeOf(&metav1.GetOptions{}).Elem(), - metav1.Unversioned.WithKind("ListOptions"): reflect.TypeOf(&metav1.ListOptions{}).Elem(), - metav1.Unversioned.WithKind("PatchOptions"): reflect.TypeOf(&metav1.PatchOptions{}).Elem(), - metav1.Unversioned.WithKind("Status"): reflect.TypeOf(&metav1.Status{}).Elem(), - metav1.Unversioned.WithKind("UpdateOptions"): reflect.TypeOf(&metav1.UpdateOptions{}).Elem(), - metav1.Unversioned.WithKind("WatchEvent"): reflect.TypeOf(&metav1.WatchEvent{}).Elem(), - }, - }, - } - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - loginConciergeAPIGroup, ok := groupsuffix.Replace("login.concierge.pinniped.dev", tt.apiGroupSuffix) - require.True(t, ok) - - scheme := getAggregatedAPIServerScheme(loginConciergeAPIGroup, tt.apiGroupSuffix) - require.Equal(t, tt.want, scheme.AllKnownTypes()) - - // make a credential request like a client would send - authenticationConciergeAPIGroup := "authentication.concierge." + tt.apiGroupSuffix - credentialRequest := &loginv1alpha1.TokenCredentialRequest{ - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &authenticationConciergeAPIGroup, - }, - }, - } - - // run defaulting on it - scheme.Default(credentialRequest) - - // make sure the group is restored if needed - require.Equal(t, "authentication.concierge.pinniped.dev", *credentialRequest.Spec.Authenticator.APIGroup) - - // make a credential request in the standard group - defaultAuthenticationConciergeAPIGroup := "authentication.concierge.pinniped.dev" - defaultCredentialRequest := &loginv1alpha1.TokenCredentialRequest{ - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &defaultAuthenticationConciergeAPIGroup, - }, - }, - } - - // run defaulting on it - scheme.Default(defaultCredentialRequest) - - if tt.apiGroupSuffix == "pinniped.dev" { // when using the standard group, this should just work - require.Equal(t, "authentication.concierge.pinniped.dev", *defaultCredentialRequest.Spec.Authenticator.APIGroup) - } else { // when using any other group, this should always be a cache miss - require.True(t, strings.HasPrefix(*defaultCredentialRequest.Spec.Authenticator.APIGroup, "_INVALID_API_GROUP_2")) - } - }) - } -} diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 99b2012e..ed0bfc4a 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -12,6 +12,7 @@ import ( "net/http" "time" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/clock" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -82,6 +83,10 @@ type Config struct { // AuthenticatorCache is a cache of authenticators shared amongst various authenticated-related controllers. AuthenticatorCache *authncache.Cache + // LoginJSONDecoder can decode login.concierge.pinniped.dev types (e.g., TokenCredentialRequest) + // into their internal representation. + LoginJSONDecoder runtime.Decoder + // Labels are labels that should be added to any resources created by the controllers. Labels map[string]string } @@ -289,7 +294,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` tls.Listen, func() (http.Handler, error) { - impersonationProxyHandler, err := impersonator.New(c.AuthenticatorCache, klogr.New().WithName("impersonation-proxy")) + impersonationProxyHandler, err := impersonator.New(c.AuthenticatorCache, c.LoginJSONDecoder, klogr.New().WithName("impersonation-proxy")) if err != nil { return nil, fmt.Errorf("could not create impersonation proxy: %w", err) } diff --git a/internal/testutil/impersonationtoken/impersonationtoken.go b/internal/testutil/impersonationtoken/impersonationtoken.go new file mode 100644 index 00000000..0e67a68b --- /dev/null +++ b/internal/testutil/impersonationtoken/impersonationtoken.go @@ -0,0 +1,67 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +// Package impersonationtoken contains a test utility to generate a token to be used against our +// impersonation proxy. +// +// It is its own package to fix import cycles involving concierge/scheme, testutil, and groupsuffix. +package impersonationtoken + +import ( + "encoding/base64" + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" + + loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" + "go.pinniped.dev/internal/groupsuffix" +) + +func Make( + t *testing.T, + token string, + authenticator *corev1.TypedLocalObjectReference, + apiGroupSuffix string, +) string { + t.Helper() + + // The impersonation test token should be a base64-encoded TokenCredentialRequest object. The API + // group of the TokenCredentialRequest object, and its Spec.Authenticator, should match whatever + // is installed on the cluster. This API group is usually replaced by the kubeclient middleware, + // but this object is not touched by the middleware since it is in a HTTP header. Therefore, we + // need to make a manual edit here. + loginConciergeGroupName, ok := groupsuffix.Replace(loginv1alpha1.GroupName, apiGroupSuffix) + require.True(t, ok, "couldn't replace suffix of %q with %q", loginv1alpha1.GroupName, apiGroupSuffix) + tokenCredentialRequest := loginv1alpha1.TokenCredentialRequest{ + TypeMeta: metav1.TypeMeta{ + Kind: "TokenCredentialRequest", + APIVersion: loginConciergeGroupName + "/v1alpha1", + }, + Spec: loginv1alpha1.TokenCredentialRequestSpec{ + Token: token, + Authenticator: *authenticator.DeepCopy(), + }, + } + + // It is assumed that the provided authenticator uses the default pinniped.dev API group, since + // this is usually replaced by the kubeclient middleware. Since we are not going through the + // kubeclient middleware, we need to make this replacement ourselves. + require.NotNil(t, tokenCredentialRequest.Spec.Authenticator.APIGroup, "expected authenticator to have non-nil API group") + authenticatorAPIGroup, ok := groupsuffix.Replace(*tokenCredentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) + require.True(t, ok, "couldn't replace suffix of %q with %q", *tokenCredentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) + tokenCredentialRequest.Spec.Authenticator.APIGroup = &authenticatorAPIGroup + + scheme := conciergescheme.New(loginConciergeGroupName, apiGroupSuffix) + codecs := serializer.NewCodecFactory(scheme) + respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) + require.True(t, ok, "couldn't find serializer info for media type") + + reqJSON, err := runtime.Encode(respInfo.PrettySerializer, &tokenCredentialRequest) + require.NoError(t, err) + return base64.RawURLEncoding.EncodeToString(reqJSON) +} diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 979060ed..e5a64bfd 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -5,8 +5,6 @@ package integration import ( "context" - "encoding/base64" - "encoding/json" "fmt" "net/http" "net/url" @@ -20,8 +18,8 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/yaml" - loginv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/testutil/impersonationtoken" "go.pinniped.dev/test/library" ) @@ -49,7 +47,7 @@ func TestImpersonationProxy(t *testing.T) { kubeconfig := &rest.Config{ Host: proxyServiceURL, TLSClientConfig: rest.TLSClientConfig{Insecure: true}, - BearerToken: makeImpersonationTestToken(t, authenticator), + BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), Proxy: func(req *http.Request) (*url.URL, error) { proxyURL, err := url.Parse(env.Proxy) require.NoError(t, err) @@ -143,24 +141,3 @@ func hasLoadBalancerService(ctx context.Context, t *testing.T, client kubernetes } return false } - -func makeImpersonationTestToken(t *testing.T, authenticator corev1.TypedLocalObjectReference) string { - t.Helper() - - env := library.IntegrationEnv(t) - reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: env.ConciergeNamespace, - }, - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginv1alpha1.GroupName + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: env.TestUser.Token, - Authenticator: authenticator, - }, - }) - require.NoError(t, err) - return base64.RawURLEncoding.EncodeToString(reqJSON) -} From c7905c66386b229c1ac633fd4bae92e912cf76c1 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Tue, 16 Feb 2021 08:15:50 -0500 Subject: [PATCH 020/203] internal/concierge/impersonator: fail if impersonation headers set If someone has already set impersonation headers in their request, then we should fail loudly so the client knows that its existing impersonation headers will not work. Signed-off-by: Andrew Keesler --- .../concierge/impersonator/impersonator.go | 24 +++++++++++++++++++ .../impersonator/impersonator_test.go | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 710df348..2bd01574 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -88,6 +88,12 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { "method", r.Method, ) + if err := ensureNoImpersonationHeaders(r); err != nil { + log.Error(err, "impersonation header already exists") + http.Error(w, "impersonation header already exists", http.StatusBadRequest) + return + } + tokenCredentialReq, err := extractToken(r, p.jsonDecoder) if err != nil { log.Error(err, "invalid token encoding") @@ -120,6 +126,24 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { p.proxy.ServeHTTP(w, newR) } +func ensureNoImpersonationHeaders(r *http.Request) error { + if _, ok := r.Header[transport.ImpersonateUserHeader]; ok { + return fmt.Errorf("%q header already exists", transport.ImpersonateUserHeader) + } + + if _, ok := r.Header[transport.ImpersonateGroupHeader]; ok { + return fmt.Errorf("%q header already exists", transport.ImpersonateGroupHeader) + } + + for header := range r.Header { + if strings.HasPrefix(header, transport.ImpersonateUserExtraHeaderPrefix) { + return fmt.Errorf("%q header already exists", transport.ImpersonateUserExtraHeaderPrefix) + } + } + + return nil +} + func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header { newHeaders := http.Header{} newHeaders.Set("Impersonate-User", userInfo.GetName()) diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 94a46f28..e993538c 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -127,6 +127,30 @@ func TestImpersonator(t *testing.T) { }, wantCreationErr: "could not get in-cluster transport: using a custom transport with TLS certificate options or the insecure flag is not allowed", }, + { + name: "Impersonate-User header already in request", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}), + wantHTTPBody: "impersonation header already exists\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"\\\"Impersonate-User\\\" header already exists\" \"msg\"=\"impersonation header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "Impersonate-Group header already in request", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}), + wantHTTPBody: "impersonation header already exists\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"\\\"Impersonate-Group\\\" header already exists\" \"msg\"=\"impersonation header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "Impersonate-Extra header already in request", + getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}), + wantHTTPBody: "impersonation header already exists\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"error\"=\"\\\"Impersonate-Extra-\\\" header already exists\" \"msg\"=\"impersonation header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, { name: "missing authorization header", getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, From eb199801107cc55450fd982f4842e4d69b264a16 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Tue, 16 Feb 2021 09:09:54 -0500 Subject: [PATCH 021/203] internal/concierge/impersonator: set user extra impersonation headers Signed-off-by: Andrew Keesler --- .../concierge/impersonator/impersonator.go | 38 ++++++++++++++-- .../impersonator/impersonator_test.go | 45 ++++++++++++++++--- 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 2bd01574..9dda0189 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -4,6 +4,7 @@ package impersonator import ( + "context" "encoding/base64" "fmt" "net/http" @@ -144,18 +145,49 @@ func ensureNoImpersonationHeaders(r *http.Request) error { return nil } +type roundTripperFunc func(*http.Request) (*http.Response, error) + +func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header { newHeaders := http.Header{} - newHeaders.Set("Impersonate-User", userInfo.GetName()) - for _, group := range userInfo.GetGroups() { - newHeaders.Add("Impersonate-Group", group) + + // Leverage client-go's impersonation RoundTripper to set impersonation headers for us in the new + // request. The client-go RoundTripper not only sets all of the impersonation headers for us, but + // it also does some helpful escaping of characters that can't go into an HTTP header. To do this, + // we make a fake call to the impersonation RoundTripper with a fake HTTP request and a delegate + // RoundTripper that captures the impersonation headers set on the request. + impersonateConfig := transport.ImpersonationConfig{ + UserName: userInfo.GetName(), + Groups: userInfo.GetGroups(), + Extra: userInfo.GetExtra(), } + impersonateHeaderSpy := roundTripperFunc(func(r *http.Request) (*http.Response, error) { + newHeaders.Set(transport.ImpersonateUserHeader, r.Header.Get(transport.ImpersonateUserHeader)) + for _, groupHeaderValue := range r.Header.Values(transport.ImpersonateGroupHeader) { + newHeaders.Add(transport.ImpersonateGroupHeader, groupHeaderValue) + } + for headerKey, headerValues := range r.Header { + if strings.HasPrefix(headerKey, transport.ImpersonateUserExtraHeaderPrefix) { + for _, headerValue := range headerValues { + newHeaders.Add(headerKey, headerValue) + } + } + } + return nil, nil + }) + fakeReq, _ := http.NewRequestWithContext(context.Background(), "", "", nil) + //nolint:bodyclose // We return a nil http.Response above, so there is nothing to close. + _, _ = transport.NewImpersonatingRoundTripper(impersonateConfig, impersonateHeaderSpy).RoundTrip(fakeReq) + + // Copy over the allowed header values from the original request to the new request. for _, header := range allowedHeaders { values := requestHeaders.Values(header) for i := range values { newHeaders.Add(header, values[i]) } } + return newHeaders } diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index e993538c..8a279a7b 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "net/url" + "reflect" "testing" "github.com/golang/mock/gomock" @@ -21,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd/api" + "k8s.io/client-go/transport" authenticationv1alpha1 "go.pinniped.dev/generated/1.20/apis/concierge/authentication/v1alpha1" "go.pinniped.dev/generated/1.20/apis/concierge/login" @@ -37,8 +39,20 @@ func TestImpersonator(t *testing.T) { const ( defaultAPIGroup = "pinniped.dev" customAPIGroup = "walrus.tld" + + testUser = "test-user" ) + testGroups := []string{"test-group-1", "test-group-2"} + testExtra := map[string][]string{ + "extra-1": {"some", "extra", "stuff"}, + "extra-2": {"some", "more", "extra", "stuff"}, + } + testExtraHeaders := map[string]string{ + "extra-1": transport.ImpersonateUserExtraHeaderPrefix + "extra-1", + "extra-2": transport.ImpersonateUserExtraHeaderPrefix + "extra-2", + } + validURL, _ := url.Parse("http://pinniped.dev/blah") testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { // Expect that the request is authenticated based on the kubeconfig credential. @@ -56,6 +70,25 @@ func TestImpersonator(t *testing.T) { http.Error(w, "got unexpected user agent header", http.StatusBadRequest) return } + // Ensure impersonation headers are set. + if values := r.Header.Values(transport.ImpersonateUserHeader); len(values) != 1 || values[0] != testUser { + message := fmt.Sprintf("got unexpected %q header: %q", transport.ImpersonateUserHeader, values) + http.Error(w, message, http.StatusBadRequest) + return + } + if values := r.Header.Values(transport.ImpersonateGroupHeader); !reflect.DeepEqual(testGroups, values) { + message := fmt.Sprintf("got unexpected %q headers: %q", transport.ImpersonateGroupHeader, values) + http.Error(w, message, http.StatusBadRequest) + return + } + for testExtraKey, testExtraValues := range testExtra { + header := testExtraHeaders[testExtraKey] + if values := r.Header.Values(header); !reflect.DeepEqual(testExtraValues, values) { + message := fmt.Sprintf("got unexpected %q headers: %q", header, values) + http.Error(w, message, http.StatusBadRequest) + return + } + } _, _ = w.Write([]byte("successful proxied response")) }) testServerKubeconfig := rest.Config{ @@ -230,9 +263,10 @@ func TestImpersonator(t *testing.T) { }), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { userInfo := user.DefaultInfo{ - Name: "test-user", - Groups: []string{"test-group-1", "test-group-2"}, + Name: testUser, + Groups: testGroups, UID: "test-uid", + Extra: testExtra, } response := &authenticator.Response{User: &userInfo} recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) @@ -252,9 +286,10 @@ func TestImpersonator(t *testing.T) { }), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { userInfo := user.DefaultInfo{ - Name: "test-user", - Groups: []string{"test-group-1", "test-group-2"}, + Name: testUser, + Groups: testGroups, UID: "test-uid", + Extra: testExtra, } response := &authenticator.Response{User: &userInfo} recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) @@ -306,7 +341,7 @@ func TestImpersonator(t *testing.T) { proxy.ServeHTTP(w, tt.request) require.Equal(t, requestBeforeServe, tt.request, "ServeHTTP() mutated the request, and it should not per http.Handler docs") if tt.wantHTTPStatus != 0 { - require.Equal(t, tt.wantHTTPStatus, w.Code) + require.Equalf(t, tt.wantHTTPStatus, w.Code, "fyi, response body was %q", w.Body.String()) } if tt.wantHTTPBody != "" { require.Equal(t, tt.wantHTTPBody, w.Body.String()) From 67da840097d0554bf6eb2c5d4c54b580aac1506e Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 16 Feb 2021 15:57:02 -0800 Subject: [PATCH 022/203] Add loadbalancer for impersonation proxy when needed --- .../impersonatorconfig/impersonator_config.go | 95 +++++++++----- .../impersonator_config_test.go | 120 ++++++++++++++++-- .../controllermanager/prepare_controllers.go | 1 + 3 files changed, 171 insertions(+), 45 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index cfa4f5ca..655856ef 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -4,6 +4,7 @@ package impersonatorconfig import ( + "context" "crypto/tls" "crypto/x509/pkix" "errors" @@ -12,7 +13,10 @@ import ( "net/http" "time" + v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" @@ -34,10 +38,12 @@ type impersonatorConfigController struct { k8sClient kubernetes.Interface configMapsInformer corev1informers.ConfigMapInformer generatedLoadBalancerServiceName string + labels map[string]string startTLSListenerFunc StartTLSListenerFunc httpHandlerFactory func() (http.Handler, error) server *http.Server + loadBalancer *v1.Service hasControlPlaneNodes *bool } @@ -51,6 +57,7 @@ func NewImpersonatorConfigController( withInformer pinnipedcontroller.WithInformerOptionFunc, withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, generatedLoadBalancerServiceName string, + labels map[string]string, startTLSListenerFunc StartTLSListenerFunc, httpHandlerFactory func() (http.Handler, error), ) controllerlib.Controller { @@ -63,6 +70,7 @@ func NewImpersonatorConfigController( k8sClient: k8sClient, configMapsInformer: configMapsInformer, generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, + labels: labels, startTLSListenerFunc: startTLSListenerFunc, httpHandlerFactory: httpHandlerFactory, }, @@ -130,38 +138,19 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { } } - // TODO when the proxy is going to run, and the endpoint goes from being not specified to being specified, then the LoadBalancer is deleted - // TODO when the proxy is going to run, and when the endpoint goes from being specified to being not specified, then the LoadBalancer is created - // TODO when auto mode decides that the proxy should be disabled, then it also does not create the LoadBalancer (or it deletes it) - - // client, err := kubeclient.New() - // if err != nil { - // plog.WarningErr("could not create client", err) - // } else { - // appNameLabel := cfg.Labels["app"] - // loadBalancer := v1.Service{ - // Spec: v1.ServiceSpec{ - // Type: "LoadBalancer", - // Ports: []v1.ServicePort{ - // { - // TargetPort: intstr.FromInt(8444), - // Port: 443, - // Protocol: v1.ProtocolTCP, - // }, - // }, - // Selector: map[string]string{"app": appNameLabel}, - // }, - // ObjectMeta: metav1.ObjectMeta{ - // Name: "impersonation-proxy-load-balancer", - // Namespace: podInfo.Namespace, - // Labels: cfg.Labels, - // }, - // } - // _, err = client.Kubernetes.CoreV1().Services(podInfo.Namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) - // if err != nil { - // plog.WarningErr("could not create load balancer", err) - // } - // } + // start the load balancer only if: + // - the impersonator is running + // - the cluster is cloud hosted + // - there is no endpoint specified in the config + if c.server != nil && !*c.hasControlPlaneNodes && config.Endpoint == "" { + if err = c.startLoadBalancer(ctx.Context); err != nil { + return err + } + } else { + if err = c.stopLoadBalancer(ctx.Context); err != nil { + return err + } + } return nil } @@ -220,3 +209,45 @@ func (c *impersonatorConfigController) startImpersonator() error { }() return nil } + +func (c *impersonatorConfigController) stopLoadBalancer(ctx context.Context) error { + if c.loadBalancer != nil { + err := c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) + if err != nil { + return err + } + } + return nil +} + +func (c *impersonatorConfigController) startLoadBalancer(ctx context.Context) error { + if c.loadBalancer != nil { + return nil + } + + appNameLabel := c.labels["app"] // TODO what if this doesn't exist + loadBalancer := v1.Service{ + Spec: v1.ServiceSpec{ + Type: "LoadBalancer", + Ports: []v1.ServicePort{ + { + TargetPort: intstr.FromInt(8444), + Port: 443, + Protocol: v1.ProtocolTCP, + }, + }, + Selector: map[string]string{"app": appNameLabel}, + }, + ObjectMeta: metav1.ObjectMeta{ + Name: c.generatedLoadBalancerServiceName, + Namespace: c.namespace, + Labels: c.labels, + }, + } + createdLoadBalancer, err := c.k8sClient.CoreV1().Services(c.namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("could not create load balancer: %w", err) + } + c.loadBalancer = createdLoadBalancer + return nil +} diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 65d35546..c4037122 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -20,6 +20,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeinformers "k8s.io/client-go/informers" kubernetesfake "k8s.io/client-go/kubernetes/fake" @@ -79,6 +80,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { generatedLoadBalancerServiceName, nil, nil, + nil, ) configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) }) @@ -147,6 +149,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" const generatedLoadBalancerServiceName = "some-service-resource-name" + var labels = map[string]string{"app": "app-name", "other-key": "other-value"} var r *require.Assertions @@ -242,6 +245,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { controllerlib.WithInformer, controllerlib.WithInitialEvent, generatedLoadBalancerServiceName, + labels, startTLSListenerFunc, func() (http.Handler, error) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -351,13 +355,27 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes", func() { it.Before(func() { addNodeWithRoleToTracker("worker") + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) }) it("automatically starts the impersonator", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() }) + + it("starts the load balancer automatically", func() { + // action 0: list nodes + // action 1: create load balancer + // that should be all + createLoadBalancerAction := kubeAPIClient.Actions()[1].(coretesting.CreateAction) + r.Equal("create", createLoadBalancerAction.GetVerb()) + createdLoadBalancerService := createLoadBalancerAction.GetObject().(*corev1.Service) + r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) + r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) + r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) + r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) + r.Equal(labels, createdLoadBalancerService.Labels) + }) }) }) @@ -369,21 +387,20 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("only starts the impersonator once and only lists the cluster's nodes once", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(2, len(kubeAPIClient.Actions())) r.Equal( - []coretesting.Action{ - coretesting.NewListAction( - schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, - schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, - "", - metav1.ListOptions{}), - }, - kubeAPIClient.Actions(), + coretesting.NewListAction( + schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, + schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, + "", + metav1.ListOptions{}), + kubeAPIClient.Actions()[0], ) r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time requireTLSServerIsRunning() // still running - r.Equal(1, len(kubeAPIClient.Actions())) // no new API calls + r.Equal(2, len(kubeAPIClient.Actions())) // no new API calls }) }) @@ -451,6 +468,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() + r.Equal(1, len(kubeAPIClient.Actions())) }) }) }) @@ -485,25 +503,48 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") }) + + it("does not start the load balancer if there are control plane nodes", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + // action 0: list nodes + // that should be all + r.Equal(1, len(kubeAPIClient.Actions())) + }) }) when("the configuration switches from enabled to disabled mode", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") - addNodeWithRoleToTracker("control-plane") + addNodeWithRoleToTracker("worker") }) - it("starts the impersonator, then shuts it down, then starts it again", func() { + it("starts the impersonator and loadbalancer, then shuts it down, then starts it again", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() + // TODO extract this + // action 0: list nodes + // action 1: create load balancer + // that should be all + createLoadBalancerAction := kubeAPIClient.Actions()[1].(coretesting.CreateAction) + r.Equal("create", createLoadBalancerAction.GetVerb()) + createdLoadBalancerService := createLoadBalancerAction.GetObject().(*corev1.Service) + r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) + r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) + r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) + r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) + r.Equal(labels, createdLoadBalancerService.Labels) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsNoLongerRunning() + deleteLoadBalancerAction := kubeAPIClient.Actions()[2].(coretesting.DeleteAction) + r.Equal("delete", deleteLoadBalancerAction.GetVerb()) + r.Equal(generatedLoadBalancerServiceName, deleteLoadBalancerAction.GetName()) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") @@ -530,6 +571,59 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) }) + + when("the endpoint switches from not specified, to specified, to not specified", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` + mode: enabled + endpoint: https://proxy.example.com:8443/ + `)) + addNodeWithRoleToTracker("worker") + }) + + it("starts, stops, restarts the loadbalancer", func() { + startInformersAndController() + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + + loadBalancer, err := kubeAPIClient.CoreV1().Services(installedInNamespace).Get(context.Background(), generatedLoadBalancerServiceName, metav1.GetOptions{}) + r.Nil(loadBalancer) + r.EqualError(err, "services \"some-service-resource-name\" not found") + + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + loadBalancer, err = kubeAPIClient.CoreV1().Services(installedInNamespace).Get(context.Background(), generatedLoadBalancerServiceName, metav1.GetOptions{}) + r.NotNil(loadBalancer) + r.NoError(err, "services \"some-service-resource-name\" not found") + + updateImpersonatorConfigMapInTracker(configMapResourceName, here.Doc(` + mode: enabled + endpoint: https://proxy.example.com:8443/ + `), "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + loadBalancer, err = kubeAPIClient.CoreV1().Services(installedInNamespace).Get(context.Background(), generatedLoadBalancerServiceName, metav1.GetOptions{}) + r.Nil(loadBalancer) + r.EqualError(err, "services \"some-service-resource-name\" not found") + }) + }) + }) + + when("there is an error creating the load balancer", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + startInformersAndController() + kubeAPIClient.PrependReactor("create", "services", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on create") + }) + }) + + it("exits with an error", func() { + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not create load balancer: error on create") + }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) } diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index ed0bfc4a..a342b684 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -292,6 +292,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { controllerlib.WithInformer, controllerlib.WithInitialEvent, "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` + c.Labels, tls.Listen, func() (http.Handler, error) { impersonationProxyHandler, err := impersonator.New(c.AuthenticatorCache, c.LoginJSONDecoder, klogr.New().WithName("impersonation-proxy")) From 10b769c676da444c15e5da297caf3060281ee616 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 17 Feb 2021 10:32:29 -0800 Subject: [PATCH 023/203] Fixed integration tests for load balancer capabilities --- .../concierge_impersonation_proxy_test.go | 39 ++++++++++++++----- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index e5a64bfd..c8bbb8aa 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -59,8 +59,10 @@ func TestImpersonationProxy(t *testing.T) { impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") - // TODO if there is already a ConfigMap, remember its contents and delete it, which puts the proxy into its default settings - // TODO and in a t.Cleanup() if there was already a ConfigMap at the start of the test, then restore the original contents + oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.GetOptions{}) + if oldConfigMap.Data != nil { + adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.DeleteOptions{}) + } serviceUnavailableError := fmt.Sprintf(`Get "%s/api/v1/namespaces": Service Unavailable`, proxyServiceURL) if env.HasCapability(library.HasExternalLoadBalancerProvider) { @@ -88,7 +90,18 @@ func TestImpersonationProxy(t *testing.T) { require.NoError(t, err) t.Cleanup(func() { - // TODO clean up the ConfigMap at the end of the test, and make sure that it happens before the t.Cleanup() above which is trying to restore the original ConfigMap + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.DeleteOptions{}) + require.NoError(t, err) + + if len(oldConfigMap.Data) != 0 { + t.Log(oldConfigMap) + oldConfigMap.UID = "" // cant have a UID yet + oldConfigMap.ResourceVersion = "" + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, oldConfigMap, metav1.CreateOptions{}) + require.NoError(t, err) + } }) } @@ -106,23 +119,31 @@ func TestImpersonationProxy(t *testing.T) { // Update configuration to force the proxy to disabled mode configMap := configMapForConfig(t, impersonator.Config{Mode: impersonator.ModeDisabled}) - _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) - require.NoError(t, err) + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) + require.NoError(t, err) + } else { + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) + require.NoError(t, err) + } // Check that we can't use the impersonation proxy to execute kubectl commands again _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableError) - // if env.HasCapability(library.HasExternalLoadBalancerProvider) { - // TODO we started the test with a load balancer, so after forcing the proxy to disable, assert that the LoadBalancer was deleted - // } + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + // the load balancer should not exist after we disable the impersonation proxy + require.Eventually(t, func() bool { + return !hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) + }, 10*time.Second, 500*time.Millisecond) + } } func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigMap { configString, err := yaml.Marshal(config) require.NoError(t, err) configMap := corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "pinniped-concierge-impersonation-proxy-config"}, + ObjectMeta: metav1.ObjectMeta{Name: "pinniped-concierge-impersonation-proxy-config"}, // TODO don't hard code this Data: map[string]string{ "config.yaml": string(configString), }} From 0ad91c43f7173ea5532b8821e1a556cfed802e7d Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 17 Feb 2021 17:22:13 -0800 Subject: [PATCH 024/203] ImpersonationConfigController uses servicesinformer This is a more reliable way to determine whether the load balancer is already running. Also added more unit tests for the load balancer. Signed-off-by: Ryan Richard --- .../impersonatorconfig/impersonator_config.go | 92 +++-- .../impersonator_config_test.go | 364 +++++++++++++++--- .../controllermanager/prepare_controllers.go | 1 + 3 files changed, 368 insertions(+), 89 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 655856ef..dd3662c8 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -37,13 +37,13 @@ type impersonatorConfigController struct { configMapResourceName string k8sClient kubernetes.Interface configMapsInformer corev1informers.ConfigMapInformer + servicesInformer corev1informers.ServiceInformer generatedLoadBalancerServiceName string labels map[string]string startTLSListenerFunc StartTLSListenerFunc httpHandlerFactory func() (http.Handler, error) server *http.Server - loadBalancer *v1.Service hasControlPlaneNodes *bool } @@ -54,6 +54,7 @@ func NewImpersonatorConfigController( configMapResourceName string, k8sClient kubernetes.Interface, configMapsInformer corev1informers.ConfigMapInformer, + servicesInformer corev1informers.ServiceInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, generatedLoadBalancerServiceName string, @@ -69,6 +70,7 @@ func NewImpersonatorConfigController( configMapResourceName: configMapResourceName, k8sClient: k8sClient, configMapsInformer: configMapsInformer, + servicesInformer: servicesInformer, generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, labels: labels, startTLSListenerFunc: startTLSListenerFunc, @@ -80,6 +82,11 @@ func NewImpersonatorConfigController( pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(configMapResourceName, namespace), controllerlib.InformerOption{}, ), + withInformer( + servicesInformer, + pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(generatedLoadBalancerServiceName, namespace), + controllerlib.InformerOption{}, + ), // Be sure to run once even if the ConfigMap that the informer is watching doesn't exist. withInitialEvent(controllerlib.Key{ Namespace: namespace, @@ -128,26 +135,22 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { plog.Debug("Queried for control plane nodes", "foundControlPlaneNodes", hasControlPlaneNodes) } - if (config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes) || config.Mode == impersonator.ModeEnabled { - if err = c.startImpersonator(); err != nil { + if c.shouldHaveImpersonator(config) { + if err = c.ensureImpersonatorIsStarted(); err != nil { return err } } else { - if err = c.stopImpersonator(); err != nil { + if err = c.ensureImpersonatorIsStopped(); err != nil { return err } } - // start the load balancer only if: - // - the impersonator is running - // - the cluster is cloud hosted - // - there is no endpoint specified in the config - if c.server != nil && !*c.hasControlPlaneNodes && config.Endpoint == "" { - if err = c.startLoadBalancer(ctx.Context); err != nil { + if c.shouldHaveLoadBalancer(config) { + if err = c.ensureLoadBalancerIsStarted(ctx.Context); err != nil { return err } } else { - if err = c.stopLoadBalancer(ctx.Context); err != nil { + if err = c.ensureLoadBalancerIsStopped(ctx.Context); err != nil { return err } } @@ -155,7 +158,19 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { return nil } -func (c *impersonatorConfigController) stopImpersonator() error { +func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { + return (config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes) || config.Mode == impersonator.ModeEnabled +} + +func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonator.Config) bool { + // start the load balancer only if: + // - the impersonator is running + // - the cluster is cloud hosted + // - there is no endpoint specified in the config + return c.server != nil && !*c.hasControlPlaneNodes && config.Endpoint == "" +} + +func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { if c.server != nil { plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) err := c.server.Close() @@ -167,7 +182,7 @@ func (c *impersonatorConfigController) stopImpersonator() error { return nil } -func (c *impersonatorConfigController) startImpersonator() error { +func (c *impersonatorConfigController) ensureImpersonatorIsStarted() error { if c.server != nil { return nil } @@ -210,22 +225,47 @@ func (c *impersonatorConfigController) startImpersonator() error { return nil } -func (c *impersonatorConfigController) stopLoadBalancer(ctx context.Context) error { - if c.loadBalancer != nil { - err := c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) - if err != nil { - return err - } +func (c *impersonatorConfigController) isLoadBalancerRunning() (bool, error) { + _, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + return false, nil } - return nil + if err != nil { + return false, err + } + return true, nil } -func (c *impersonatorConfigController) startLoadBalancer(ctx context.Context) error { - if c.loadBalancer != nil { +func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { + running, err := c.isLoadBalancerRunning() + if err != nil { + return err + } + if !running { return nil } - appNameLabel := c.labels["app"] // TODO what if this doesn't exist + plog.Info("Deleting load balancer for impersonation proxy", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) + if err != nil { + return err + } + + return nil +} + +func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.Context) error { + running, err := c.isLoadBalancerRunning() + if err != nil { + return err + } + if running { + return nil + } + appNameLabel := c.labels["app"] loadBalancer := v1.Service{ Spec: v1.ServiceSpec{ Type: "LoadBalancer", @@ -244,10 +284,12 @@ func (c *impersonatorConfigController) startLoadBalancer(ctx context.Context) er Labels: c.labels, }, } - createdLoadBalancer, err := c.k8sClient.CoreV1().Services(c.namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) + plog.Info("creating load balancer for impersonation proxy", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + _, err = c.k8sClient.CoreV1().Services(c.namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create load balancer: %w", err) } - c.loadBalancer = createdLoadBalancer return nil } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index c4037122..d8163e29 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -19,10 +19,12 @@ import ( "github.com/sclevine/spec/report" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" kubeinformers "k8s.io/client-go/informers" + corev1informers "k8s.io/client-go/informers/core/v1" kubernetesfake "k8s.io/client-go/kubernetes/fake" coretesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" @@ -64,17 +66,22 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { var observableWithInformerOption *testutil.ObservableWithInformerOption var observableWithInitialEventOption *testutil.ObservableWithInitialEventOption var configMapsInformerFilter controllerlib.Filter + var servicesInformerFilter controllerlib.Filter it.Before(func() { r = require.New(t) observableWithInformerOption = testutil.NewObservableWithInformerOption() observableWithInitialEventOption = testutil.NewObservableWithInitialEventOption() - configMapsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().ConfigMaps() + sharedInformerFactory := kubeinformers.NewSharedInformerFactory(nil, 0) + configMapsInformer := sharedInformerFactory.Core().V1().ConfigMaps() + servicesInformer := sharedInformerFactory.Core().V1().Services() + _ = NewImpersonatorConfigController( installedInNamespace, configMapResourceName, nil, configMapsInformer, + servicesInformer, observableWithInformerOption.WithInformer, observableWithInitialEventOption.WithInitialEvent, generatedLoadBalancerServiceName, @@ -83,6 +90,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { nil, ) configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) + servicesInformerFilter = observableWithInformerOption.GetFilterForInformer(servicesInformer) }) when("watching ConfigMap objects", func() { @@ -133,6 +141,54 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { }) }) + when("watching Service objects", func() { + var subject controllerlib.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.Service + + it.Before(func() { + subject = servicesInformerFilter + target = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: generatedLoadBalancerServiceName, Namespace: installedInNamespace}} + wrongNamespace = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: generatedLoadBalancerServiceName, Namespace: "wrong-namespace"}} + wrongName = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target Service changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a Service from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a Service with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a Service with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + when("starting up", func() { it("asks for an initial event because the ConfigMap may not exist yet and it needs to run anyway", func() { r.Equal(&controllerlib.Key{ @@ -233,6 +289,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }, 10*time.Second, time.Millisecond) } + var waitForLoadBalancerToBeDeleted = func(informer corev1informers.ServiceInformer, name string) { + r.Eventually(func() bool { + _, err := informer.Lister().Services(installedInNamespace).Get(name) + return k8serrors.IsNotFound(err) + }, 10*time.Second, time.Millisecond) + } + // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. var startInformersAndController = func() { @@ -242,6 +305,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { configMapResourceName, kubeAPIClient, kubeInformers.Core().V1().ConfigMaps(), + kubeInformers.Core().V1().Services(), controllerlib.WithInformer, controllerlib.WithInitialEvent, generatedLoadBalancerServiceName, @@ -307,6 +371,31 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } + var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { + loadBalancerService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + } + r.NoError(client.Tracker().Add(loadBalancerService)) + } + + var deleteLoadBalancerServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Delete( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + installedInNamespace, + resourceName, + )) + } + var addNodeWithRoleToTracker = func(role string) { r.NoError(kubeAPIClient.Tracker().Add( &corev1.Node{ @@ -318,6 +407,34 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } + var requireNodesListed = func(action coretesting.Action) { + r.Equal( + coretesting.NewListAction( + schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, + schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, + "", + metav1.ListOptions{}), + action, + ) + } + + var requireLoadBalancerWasCreated = func(action coretesting.Action) { + createLoadBalancerAction := action.(coretesting.CreateAction) + r.Equal("create", createLoadBalancerAction.GetVerb()) + createdLoadBalancerService := createLoadBalancerAction.GetObject().(*corev1.Service) + r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) + r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) + r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) + r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) + r.Equal(labels, createdLoadBalancerService.Labels) + } + + var requireLoadBalancerDeleted = func(action coretesting.Action) { + deleteLoadBalancerAction := action.(coretesting.DeleteAction) + r.Equal("delete", deleteLoadBalancerAction.GetVerb()) + r.Equal(generatedLoadBalancerServiceName, deleteLoadBalancerAction.GetName()) + } + it.Before(func() { r = require.New(t) @@ -345,10 +462,29 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("control-plane") }) - it("does not start the impersonator", func() { + it("does not start the impersonator or load balancer", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() + r.Equal(1, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) + }) + + when("there are visible control plane nodes and a loadbalancer", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane") + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("does not start the impersonator, deletes the loadbalancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerWasNeverStarted() + r.Equal(2, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) }) }) @@ -364,17 +500,28 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("starts the load balancer automatically", func() { - // action 0: list nodes - // action 1: create load balancer - // that should be all - createLoadBalancerAction := kubeAPIClient.Actions()[1].(coretesting.CreateAction) - r.Equal("create", createLoadBalancerAction.GetVerb()) - createdLoadBalancerService := createLoadBalancerAction.GetObject().(*corev1.Service) - r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) - r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) - r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) - r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) - r.Equal(labels, createdLoadBalancerService.Labels) + r.Equal(2, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + }) + + it("automatically starts the impersonator", func() { + requireTLSServerIsRunning() + }) + + it("does not start the load balancer automatically", func() { + r.Equal(1, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) }) }) }) @@ -388,14 +535,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Equal(2, len(kubeAPIClient.Actions())) - r.Equal( - coretesting.NewListAction( - schema.GroupVersionResource{Version: "v1", Resource: "nodes"}, - schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"}, - "", - metav1.ListOptions{}), - kubeAPIClient.Actions()[0], - ) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time @@ -456,6 +601,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() + requireNodesListed(kubeAPIClient.Actions()[0]) + r.Equal(1, len(kubeAPIClient.Actions())) }) }) @@ -468,6 +615,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() + requireNodesListed(kubeAPIClient.Actions()[0]) r.Equal(1, len(kubeAPIClient.Actions())) }) }) @@ -483,33 +631,121 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() + requireNodesListed(kubeAPIClient.Actions()[0]) + r.Equal(1, len(kubeAPIClient.Actions())) }) }) when("the configuration is enabled mode", func() { - it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") - addNodeWithRoleToTracker("control-plane") + + when("there are control plane nodes", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("control-plane") + }) + + it("starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + }) + + it("does not start the load balancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) }) - it("starts the impersonator regardless of the visibility of control plane nodes", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() + when("there are no control plane nodes but a loadbalancer already exists", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + }) + + it("does not start the load balancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) }) - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + when("there are control plane nodes and a loadbalancer already exists", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("control-plane") + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + }) + + it("stops the load balancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(2, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) + }) }) - it("does not start the load balancer if there are control plane nodes", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - // action 0: list nodes - // that should be all - r.Equal(1, len(kubeAPIClient.Actions())) + when("there are no control plane nodes and there is no load balancer", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + }) + + it("starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunning() + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + }) + + it("starts the load balancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(2, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + }) }) }) @@ -524,33 +760,32 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() - // TODO extract this - // action 0: list nodes - // action 1: create load balancer - // that should be all - createLoadBalancerAction := kubeAPIClient.Actions()[1].(coretesting.CreateAction) - r.Equal("create", createLoadBalancerAction.GetVerb()) - createdLoadBalancerService := createLoadBalancerAction.GetObject().(*corev1.Service) - r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) - r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) - r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) - r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) - r.Equal(labels, createdLoadBalancerService.Labels) + r.Equal(2, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsNoLongerRunning() - deleteLoadBalancerAction := kubeAPIClient.Actions()[2].(coretesting.DeleteAction) - r.Equal("delete", deleteLoadBalancerAction.GetVerb()) - r.Equal(generatedLoadBalancerServiceName, deleteLoadBalancerAction.GetName()) + r.Equal(3, len(kubeAPIClient.Actions())) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) + + deleteLoadBalancerServiceFromTracker(generatedLoadBalancerServiceName, kubeInformerClient) + waitForLoadBalancerToBeDeleted(kubeInformers.Core().V1().Services(), generatedLoadBalancerServiceName) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() + r.Equal(4, len(kubeAPIClient.Actions())) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) }) when("there is an error while shutting down the server", func() { @@ -572,7 +807,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("the endpoint switches from not specified, to specified, to not specified", func() { + when("the endpoint switches from specified, to not specified, to specified", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: enabled @@ -581,22 +816,24 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("worker") }) - it("starts, stops, restarts the loadbalancer", func() { + it("doesn't start, then creates, then deletes the load balancer", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - loadBalancer, err := kubeAPIClient.CoreV1().Services(installedInNamespace).Get(context.Background(), generatedLoadBalancerServiceName, metav1.GetOptions{}) - r.Nil(loadBalancer) - r.EqualError(err, "services \"some-service-resource-name\" not found") + r.Equal(1, len(kubeAPIClient.Actions())) + requireNodesListed(kubeAPIClient.Actions()[0]) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - loadBalancer, err = kubeAPIClient.CoreV1().Services(installedInNamespace).Get(context.Background(), generatedLoadBalancerServiceName, metav1.GetOptions{}) - r.NotNil(loadBalancer) - r.NoError(err, "services \"some-service-resource-name\" not found") + r.Equal(2, len(kubeAPIClient.Actions())) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") updateImpersonatorConfigMapInTracker(configMapResourceName, here.Doc(` mode: enabled @@ -605,9 +842,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - loadBalancer, err = kubeAPIClient.CoreV1().Services(installedInNamespace).Get(context.Background(), generatedLoadBalancerServiceName, metav1.GetOptions{}) - r.Nil(loadBalancer) - r.EqualError(err, "services \"some-service-resource-name\" not found") + r.Equal(3, len(kubeAPIClient.Actions())) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) }) }) }) diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index a342b684..8d6e43ee 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -289,6 +289,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { "pinniped-concierge-impersonation-proxy-config", // TODO this string should come from `c.NamesConfig` client.Kubernetes, informers.installationNamespaceK8s.Core().V1().ConfigMaps(), + informers.installationNamespaceK8s.Core().V1().Services(), controllerlib.WithInformer, controllerlib.WithInitialEvent, "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` From 22a3e73baccf1fa9bd278acd26448cc31e25563a Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 17 Feb 2021 17:29:56 -0800 Subject: [PATCH 025/203] impersonator_config_test.go: use require.Len() when applicable Also fix a lint error in concierge_impersonation_proxy_test.go Signed-off-by: Ryan Richard --- .../impersonator_config_test.go | 39 +++++++++---------- .../concierge_impersonation_proxy_test.go | 2 +- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index d8163e29..d441eabb 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -466,7 +466,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) }) @@ -482,7 +482,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) }) @@ -500,7 +500,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("starts the load balancer automatically", func() { - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) }) @@ -520,7 +520,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("does not start the load balancer automatically", func() { - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) }) @@ -534,7 +534,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("only starts the impersonator once and only lists the cluster's nodes once", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) @@ -545,7 +545,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time requireTLSServerIsRunning() // still running - r.Equal(2, len(kubeAPIClient.Actions())) // no new API calls + r.Len(kubeAPIClient.Actions(), 2) // no new API calls }) }) @@ -602,7 +602,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() requireNodesListed(kubeAPIClient.Actions()[0]) - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) }) }) @@ -616,7 +616,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() requireNodesListed(kubeAPIClient.Actions()[0]) - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) }) }) }) @@ -632,12 +632,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() requireNodesListed(kubeAPIClient.Actions()[0]) - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) }) }) when("the configuration is enabled mode", func() { - when("there are control plane nodes", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") @@ -659,7 +658,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) }) @@ -687,7 +686,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) }) @@ -715,7 +714,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("stops the load balancer", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) }) @@ -742,7 +741,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the load balancer", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) }) @@ -760,7 +759,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) @@ -773,7 +772,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsNoLongerRunning() - r.Equal(3, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 3) requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) deleteLoadBalancerServiceFromTracker(generatedLoadBalancerServiceName, kubeInformerClient) @@ -784,7 +783,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunning() - r.Equal(4, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 4) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) }) @@ -821,14 +820,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(2, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 2) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) // update manually because the kubeAPIClient isn't connected to the informer in the tests @@ -842,7 +841,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(3, len(kubeAPIClient.Actions())) + r.Len(kubeAPIClient.Actions(), 3) requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) }) }) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index c8bbb8aa..3749a97f 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -61,7 +61,7 @@ func TestImpersonationProxy(t *testing.T) { oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.GetOptions{}) if oldConfigMap.Data != nil { - adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.DeleteOptions{}) + require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.DeleteOptions{})) } serviceUnavailableError := fmt.Sprintf(`Get "%s/api/v1/namespaces": Service Unavailable`, proxyServiceURL) From b3cdc438ce0511eeb8e39aa93a0fdf940c0b9d7d Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Feb 2021 10:13:24 -0500 Subject: [PATCH 026/203] internal/concierge/impersonator: reuse kube bearertoken.Authenticator Signed-off-by: Andrew Keesler --- .../concierge/impersonator/impersonator.go | 83 ++++++++++++------- .../impersonator/impersonator_test.go | 4 +- 2 files changed, 53 insertions(+), 34 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 9dda0189..48221265 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -14,11 +14,14 @@ import ( "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/request/bearertoken" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/rest" "k8s.io/client-go/transport" "go.pinniped.dev/generated/1.20/apis/concierge/login" + "go.pinniped.dev/internal/constable" "go.pinniped.dev/internal/controller/authenticator/authncache" "go.pinniped.dev/internal/kubeclient" ) @@ -95,38 +98,61 @@ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - tokenCredentialReq, err := extractToken(r, p.jsonDecoder) + // Never mutate request (see http.Handler docs). + newR := r.Clone(r.Context()) + + authentication, authenticated, err := bearertoken.New(authenticator.TokenFunc(func(ctx context.Context, token string) (*authenticator.Response, bool, error) { + tokenCredentialReq, err := extractToken(token, p.jsonDecoder) + if err != nil { + log.Error(err, "invalid token encoding") + return nil, false, &httpError{message: "invalid token encoding", code: http.StatusBadRequest} + } + + log = log.WithValues( + "authenticator", tokenCredentialReq.Spec.Authenticator, + ) + + userInfo, err := p.cache.AuthenticateTokenCredentialRequest(newR.Context(), tokenCredentialReq) + if err != nil { + log.Error(err, "received invalid token") + return nil, false, &httpError{message: "invalid token", code: http.StatusUnauthorized} + } + if userInfo == nil { + log.Info("received token that did not authenticate") + return nil, false, &httpError{message: "not authenticated", code: http.StatusUnauthorized} + } + log = log.WithValues("userID", userInfo.GetUID()) + + return &authenticator.Response{User: userInfo}, true, nil + })).AuthenticateRequest(newR) if err != nil { - log.Error(err, "invalid token encoding") + httpErr, ok := err.(*httpError) + if !ok { + log.Error(err, "unrecognized error") + http.Error(w, "unrecognized error", http.StatusInternalServerError) + } + http.Error(w, httpErr.message, httpErr.code) + return + } + if !authenticated { + log.Error(constable.Error("token authenticator did not find token"), "invalid token encoding") http.Error(w, "invalid token encoding", http.StatusBadRequest) return } - log = log.WithValues( - "authenticator", tokenCredentialReq.Spec.Authenticator, - ) - userInfo, err := p.cache.AuthenticateTokenCredentialRequest(r.Context(), tokenCredentialReq) - if err != nil { - log.Error(err, "received invalid token") - http.Error(w, "invalid token", http.StatusUnauthorized) - return - } - - if userInfo == nil { - log.Info("received token that did not authenticate") - http.Error(w, "not authenticated", http.StatusUnauthorized) - return - } - log = log.WithValues("userID", userInfo.GetUID()) - - // Never mutate request (see http.Handler docs). - newR := r.WithContext(r.Context()) - newR.Header = getProxyHeaders(userInfo, r.Header) + newR.Header = getProxyHeaders(authentication.User, r.Header) log.Info("proxying authenticated request") p.proxy.ServeHTTP(w, newR) } +type httpError struct { + message string + code int +} + +func (e *httpError) Error() string { return e.message } + func ensureNoImpersonationHeaders(r *http.Request) error { if _, ok := r.Header[transport.ImpersonateUserHeader]; ok { return fmt.Errorf("%q header already exists", transport.ImpersonateUserHeader) @@ -191,16 +217,8 @@ func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header return newHeaders } -func extractToken(req *http.Request, jsonDecoder runtime.Decoder) (*login.TokenCredentialRequest, error) { - authHeader := req.Header.Get("Authorization") - if authHeader == "" { - return nil, fmt.Errorf("missing authorization header") - } - if !strings.HasPrefix(authHeader, "Bearer ") { - return nil, fmt.Errorf("authorization header must be of type Bearer") - } - encoded := strings.TrimPrefix(authHeader, "Bearer ") - tokenCredentialRequestJSON, err := base64.RawURLEncoding.DecodeString(encoded) +func extractToken(token string, jsonDecoder runtime.Decoder) (*login.TokenCredentialRequest, error) { + tokenCredentialRequestJSON, err := base64.RawURLEncoding.DecodeString(token) if err != nil { return nil, fmt.Errorf("invalid base64 in encoded bearer token: %w", err) } @@ -209,6 +227,7 @@ func extractToken(req *http.Request, jsonDecoder runtime.Decoder) (*login.TokenC if err != nil { return nil, fmt.Errorf("invalid object encoded in bearer token: %w", err) } + tokenCredentialRequest, ok := obj.(*login.TokenCredentialRequest) if !ok { return nil, fmt.Errorf("invalid TokenCredentialRequest encoded in bearer token: got %T", obj) diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 8a279a7b..ea903734 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -190,7 +190,7 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"missing authorization header\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"error\"=\"token authenticator did not find token\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "authorization header missing bearer prefix", @@ -198,7 +198,7 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{"Authorization": {impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"authorization header must be of type Bearer\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"error\"=\"token authenticator did not find token\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token is not base64 encoded", From f5fedbb6b24a7fdea974e5dd5679bd6af76d7650 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 18 Feb 2021 10:59:58 -0800 Subject: [PATCH 027/203] Add Service resource "delete" permission to Concierge RBAC - Because the impersonation proxy config controller needs to be able to delete the load balancer which it created Signed-off-by: Margo Crawford --- deploy/concierge/rbac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/concierge/rbac.yaml b/deploy/concierge/rbac.yaml index 3f3796ec..449c6d5a 100644 --- a/deploy/concierge/rbac.yaml +++ b/deploy/concierge/rbac.yaml @@ -75,7 +75,7 @@ metadata: rules: - apiGroups: [ "" ] resources: [ services ] - verbs: [ create, get, list, patch, update, watch ] + verbs: [ create, get, list, patch, update, watch, delete ] - apiGroups: [ "" ] resources: [ secrets ] verbs: [ create, get, list, patch, update, watch, delete ] From 7a140bf63c2335df38b750b0e0d3daf920be1233 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Thu, 18 Feb 2021 11:08:13 -0800 Subject: [PATCH 028/203] concierge_impersonation_proxy_test.go: add an eventually loop Signed-off-by: Ryan Richard --- test/integration/concierge_impersonation_proxy_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 3749a97f..411f9dca 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -128,8 +128,10 @@ func TestImpersonationProxy(t *testing.T) { } // Check that we can't use the impersonation proxy to execute kubectl commands again - _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - require.EqualError(t, err, serviceUnavailableError) + require.Eventually(t, func() bool { + _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + return err.Error() == serviceUnavailableError + }, 10*time.Second, 500*time.Millisecond) if env.HasCapability(library.HasExternalLoadBalancerProvider) { // the load balancer should not exist after we disable the impersonation proxy From 126f9c0da3ddbb8daf4d9f636eca95a9d155654b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 18 Feb 2021 11:16:34 -0800 Subject: [PATCH 029/203] certs_manager.go: Rename some local variables Signed-off-by: Margo Crawford --- internal/controller/apicerts/certs_manager.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index c0be7873..22f0f6df 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -93,14 +93,14 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { } // Create a CA. - aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: c.generatedCACommonName}, c.certDuration) + ca, err := certauthority.New(pkix.Name{CommonName: c.generatedCACommonName}, c.certDuration) if err != nil { return fmt.Errorf("could not initialize CA: %w", err) } - // Using the CA from above, create a TLS server cert for the aggregated API server to use. + // Using the CA from above, create a TLS server cert. serviceEndpoint := c.serviceNameForGeneratedCertCommonName + "." + c.namespace + ".svc" - aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( + tlsCert, err := ca.Issue( pkix.Name{CommonName: serviceEndpoint}, []string{serviceEndpoint}, nil, @@ -111,7 +111,7 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { } // Write the CA's public key bundle and the serving certs to a secret. - tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(aggregatedAPIServerTLSCert) + tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(tlsCert) if err != nil { return fmt.Errorf("could not PEM encode serving certificate: %w", err) } @@ -123,7 +123,7 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { Labels: c.certsSecretLabels, }, StringData: map[string]string{ - caCertificateSecretKey: string(aggregatedAPIServerCA.Bundle()), + caCertificateSecretKey: string(ca.Bundle()), tlsPrivateKeySecretKey: string(tlsPrivateKeyPEM), tlsCertificateChainSecretKey: string(tlsCertChainPEM), }, From 19881e4d7f9cca59120d64e49c59776f3241efaf Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Thu, 18 Feb 2021 15:58:27 -0800 Subject: [PATCH 030/203] Increase how long we wait for loadbalancers to be deleted for int test Also add some log messages which might help us debug issues like this in the future. Signed-off-by: Ryan Richard --- .../impersonatorconfig/impersonator_config.go | 4 ++- .../concierge_impersonation_proxy_test.go | 25 +++++++++++++------ 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index dd3662c8..8f75b20c 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -96,7 +96,7 @@ func NewImpersonatorConfigController( } func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { - plog.Info("impersonatorConfigController Sync") + plog.Debug("Starting impersonatorConfigController Sync") configMap, err := c.configMapsInformer.Lister().ConfigMaps(c.namespace).Get(c.configMapResourceName) notFound := k8serrors.IsNotFound(err) @@ -155,6 +155,8 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { } } + plog.Debug("Successfully finished impersonatorConfigController Sync") + return nil } diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 411f9dca..ce275dfc 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -23,7 +23,9 @@ import ( "go.pinniped.dev/test/library" ) -// Smoke test to see if the kubeconfig works and the cluster is reachable. +// TODO don't hard code "pinniped-concierge-" in this string. It should be constructed from the env app name. +const impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" + func TestImpersonationProxy(t *testing.T) { env := library.IntegrationEnv(t) if env.Proxy == "" { @@ -59,9 +61,10 @@ func TestImpersonationProxy(t *testing.T) { impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") - oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.GetOptions{}) + oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName, metav1.GetOptions{}) if oldConfigMap.Data != nil { - require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.DeleteOptions{})) + t.Logf("stashing a pre-existing configmap %s", oldConfigMap.Name) + require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName, metav1.DeleteOptions{})) } serviceUnavailableError := fmt.Sprintf(`Get "%s/api/v1/namespaces": Service Unavailable`, proxyServiceURL) @@ -86,19 +89,22 @@ func TestImpersonationProxy(t *testing.T) { Endpoint: proxyServiceURL, TLS: nil, }) + t.Logf("creating configmap %s", configMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) require.NoError(t, err) t.Cleanup(func() { ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, "pinniped-concierge-impersonation-proxy-config", metav1.DeleteOptions{}) + t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName) + err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName, metav1.DeleteOptions{}) require.NoError(t, err) if len(oldConfigMap.Data) != 0 { t.Log(oldConfigMap) oldConfigMap.UID = "" // cant have a UID yet oldConfigMap.ResourceVersion = "" + t.Logf("restoring a pre-existing configmap %s", oldConfigMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, oldConfigMap, metav1.CreateOptions{}) require.NoError(t, err) } @@ -120,9 +126,11 @@ func TestImpersonationProxy(t *testing.T) { // Update configuration to force the proxy to disabled mode configMap := configMapForConfig(t, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { + t.Logf("creating configmap %s", configMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) require.NoError(t, err) } else { + t.Logf("updating configmap %s", configMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) require.NoError(t, err) } @@ -134,10 +142,11 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 500*time.Millisecond) if env.HasCapability(library.HasExternalLoadBalancerProvider) { - // the load balancer should not exist after we disable the impersonation proxy + // The load balancer should not exist after we disable the impersonation proxy. + // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). require.Eventually(t, func() bool { return !hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) - }, 10*time.Second, 500*time.Millisecond) + }, time.Minute, 500*time.Millisecond) } } @@ -145,7 +154,9 @@ func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigM configString, err := yaml.Marshal(config) require.NoError(t, err) configMap := corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "pinniped-concierge-impersonation-proxy-config"}, // TODO don't hard code this + ObjectMeta: metav1.ObjectMeta{ + Name: impersonationProxyConfigMapName, + }, Data: map[string]string{ "config.yaml": string(configString), }} From b8592a361caef342f5534a53e80713d3d033c8b1 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 18 Feb 2021 16:27:03 -0800 Subject: [PATCH 031/203] Add some comments to concierge_impersonation_proxy_test.go Signed-off-by: Margo Crawford --- .../concierge_impersonation_proxy_test.go | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index ce275dfc..637f2002 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -123,6 +123,23 @@ func TestImpersonationProxy(t *testing.T) { ) } + t.Run("watching all the verbs", func(t *testing.T) { + // Start a watch in a informer. + // Create an RBAC rule to allow this user to read/write everything. + // t.Cleanup Delete the RBAC rule. + // Create a namespace, because it will be easier to deletecollection if we have a namespace. + // t.Cleanup Delete the namespace. + // Then "create" several Secrets. + // "get" one them. + // "list" them all. + // "update" one of them. + // "patch" one of them. + // "delete" one of them. + // "deletecollection" all of them. + // Make sure the watch sees all of those actions. + // Close the informer. + }) + // Update configuration to force the proxy to disabled mode configMap := configMapForConfig(t, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { @@ -135,8 +152,10 @@ func TestImpersonationProxy(t *testing.T) { require.NoError(t, err) } - // Check that we can't use the impersonation proxy to execute kubectl commands again + // Check that the impersonation proxy has shut down require.Eventually(t, func() bool { + // It's okay if this returns RBAC errors because this user has no role bindings. + // What we want to see is that the proxy eventually shuts down entirely. _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return err.Error() == serviceUnavailableError }, 10*time.Second, 500*time.Millisecond) From 80ff5c1f17a4d6bb0572431826fd5274bc425e31 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 22 Feb 2021 17:23:11 -0800 Subject: [PATCH 032/203] Fix bug which prevented watches from working through impersonator Also: - Changed base64 encoding of impersonator bearer tokens to use `base64.StdEncoding` to make it easier for users to manually create a token using the unix `base64` command - Test the headers which are and are not passed through to the Kube API by the impersonator more carefully in the unit tests - More WIP on concierge_impersonation_proxy_test.go Signed-off-by: Margo Crawford --- cmd/pinniped/cmd/login_oidc.go | 2 +- cmd/pinniped/cmd/login_oidc_test.go | 2 +- .../concierge/impersonator/impersonator.go | 7 +- .../impersonator/impersonator_test.go | 186 ++++++++++-------- .../impersonationtoken/impersonationtoken.go | 2 +- .../concierge_impersonation_proxy_test.go | 95 ++++++++- test/integration/e2e_test.go | 7 + test/library/access.go | 15 +- test/library/client.go | 22 +++ 9 files changed, 250 insertions(+), 88 deletions(-) diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 8b18805a..1f101950 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -294,7 +294,7 @@ func execCredentialForImpersonationProxy( if err != nil { return nil, fmt.Errorf("Error creating TokenCredentialRequest for impersonation proxy: %w", err) } - encodedToken := base64.RawURLEncoding.EncodeToString(reqJSON) + encodedToken := base64.StdEncoding.EncodeToString(reqJSON) cred := &clientauthv1beta1.ExecCredential{ TypeMeta: metav1.TypeMeta{ Kind: "ExecCredential", diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index e9a47e8f..198b7519 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -288,5 +288,5 @@ func impersonationProxyTestToken(token string) string { }, }, }) - return base64.RawURLEncoding.EncodeToString(reqJSON) + return base64.StdEncoding.EncodeToString(reqJSON) } diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index d9275673..e4be50c4 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -11,6 +11,7 @@ import ( "net/http/httputil" "net/url" "strings" + "time" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -34,6 +35,7 @@ var allowedHeaders = []string{ "User-Agent", "Connection", "Upgrade", + "Content-Type", } type proxy struct { @@ -68,7 +70,7 @@ func newInternal(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr. if err != nil { return nil, fmt.Errorf("could not get in-cluster transport config: %w", err) } - kubeTransportConfig.TLS.NextProtos = []string{"http/1.1"} + kubeTransportConfig.TLS.NextProtos = []string{"http/1.1"} // TODO huh? kubeRoundTripper, err := transport.New(kubeTransportConfig) if err != nil { @@ -77,6 +79,7 @@ func newInternal(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr. reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) reverseProxy.Transport = kubeRoundTripper + reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line return &proxy{ cache: cache, @@ -218,7 +221,7 @@ func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header } func extractToken(token string, jsonDecoder runtime.Decoder) (*login.TokenCredentialRequest, error) { - tokenCredentialRequestJSON, err := base64.RawURLEncoding.DecodeString(token) + tokenCredentialRequestJSON, err := base64.StdEncoding.DecodeString(token) if err != nil { return nil, fmt.Errorf("invalid base64 in encoded bearer token: %w", err) } diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index fd86cd18..91136b49 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -9,7 +9,6 @@ import ( "net/http" "net/http/httptest" "net/url" - "reflect" "testing" "github.com/golang/mock/gomock" @@ -22,7 +21,6 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd/api" - "k8s.io/client-go/transport" authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" "go.pinniped.dev/generated/latest/apis/concierge/login" @@ -48,54 +46,8 @@ func TestImpersonator(t *testing.T) { "extra-1": {"some", "extra", "stuff"}, "extra-2": {"some", "more", "extra", "stuff"}, } - testExtraHeaders := map[string]string{ - "extra-1": transport.ImpersonateUserExtraHeaderPrefix + "extra-1", - "extra-2": transport.ImpersonateUserExtraHeaderPrefix + "extra-2", - } validURL, _ := url.Parse("http://pinniped.dev/blah") - testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { - // Expect that the request is authenticated based on the kubeconfig credential. - if r.Header.Get("Authorization") != "Bearer some-service-account-token" { - http.Error(w, "expected to see service account token", http.StatusForbidden) - return - } - // Fail if we see the malicious header passed through the proxy (it's not on the allowlist). - if r.Header.Get("Malicious-Header") != "" { - http.Error(w, "didn't expect to see malicious header", http.StatusForbidden) - return - } - // Expect to see the user agent header passed through. - if r.Header.Get("User-Agent") != "test-user-agent" { - http.Error(w, "got unexpected user agent header", http.StatusBadRequest) - return - } - // Ensure impersonation headers are set. - if values := r.Header.Values(transport.ImpersonateUserHeader); len(values) != 1 || values[0] != testUser { - message := fmt.Sprintf("got unexpected %q header: %q", transport.ImpersonateUserHeader, values) - http.Error(w, message, http.StatusBadRequest) - return - } - if values := r.Header.Values(transport.ImpersonateGroupHeader); !reflect.DeepEqual(testGroups, values) { - message := fmt.Sprintf("got unexpected %q headers: %q", transport.ImpersonateGroupHeader, values) - http.Error(w, message, http.StatusBadRequest) - return - } - for testExtraKey, testExtraValues := range testExtra { - header := testExtraHeaders[testExtraKey] - if values := r.Header.Values(header); !reflect.DeepEqual(testExtraValues, values) { - message := fmt.Sprintf("got unexpected %q headers: %q", header, values) - http.Error(w, message, http.StatusBadRequest) - return - } - } - _, _ = w.Write([]byte("successful proxied response")) - }) - testServerKubeconfig := rest.Config{ - Host: testServerURL, - BearerToken: "some-service-account-token", - TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testServerCA)}, - } newRequest := func(h http.Header) *http.Request { r, err := http.NewRequestWithContext(context.Background(), http.MethodGet, validURL.String(), nil) require.NoError(t, err) @@ -113,15 +65,17 @@ func TestImpersonator(t *testing.T) { } tests := []struct { - name string - apiGroupOverride string - getKubeconfig func() (*rest.Config, error) - wantCreationErr string - request *http.Request - wantHTTPBody string - wantHTTPStatus int - wantLogs []string - expectMockToken func(*testing.T, *mocktokenauthenticator.MockTokenMockRecorder) + name string + apiGroupOverride string + getKubeconfig func() (*rest.Config, error) + wantCreationErr string + request *http.Request + wantHTTPBody string + wantHTTPStatus int + wantLogs []string + wantKubeAPIServerRequestHeaders http.Header + wantKubeAPIServerStatusCode int + expectMockToken func(*testing.T, *mocktokenauthenticator.MockTokenMockRecorder) }{ { name: "fail to get in-cluster config", @@ -162,7 +116,6 @@ func TestImpersonator(t *testing.T) { }, { name: "Impersonate-User header already in request", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, @@ -170,7 +123,6 @@ func TestImpersonator(t *testing.T) { }, { name: "Impersonate-Group header already in request", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, @@ -178,7 +130,6 @@ func TestImpersonator(t *testing.T) { }, { name: "Impersonate-Extra header already in request", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, @@ -186,7 +137,6 @@ func TestImpersonator(t *testing.T) { }, { name: "missing authorization header", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, @@ -194,7 +144,6 @@ func TestImpersonator(t *testing.T) { }, { name: "authorization header missing bearer prefix", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Authorization": {impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, @@ -202,7 +151,6 @@ func TestImpersonator(t *testing.T) { }, { name: "token is not base64 encoded", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Authorization": {"Bearer !!!"}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, @@ -210,16 +158,14 @@ func TestImpersonator(t *testing.T) { }, { name: "base64 encoded token is not valid json", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: newRequest(map[string][]string{"Authorization": {"Bearer abc"}}), + request: newRequest(map[string][]string{"Authorization": {"Bearer aGVsbG8gd29ybGQK"}}), // aGVsbG8gd29ybGQK is "hello world" base64 encoded wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: couldn't get version/kind; json parse error: invalid character 'i' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: couldn't get version/kind; json parse error: invalid character 'h' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "base64 encoded token is encoded with default api group but we are expecting custom api group", apiGroupOverride: customAPIGroup, - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, @@ -227,7 +173,6 @@ func TestImpersonator(t *testing.T) { }, { name: "base64 encoded token is encoded with custom api group but we are expecting default api group", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, @@ -235,16 +180,14 @@ func TestImpersonator(t *testing.T) { }, { name: "token could not be authenticated", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "", &badAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token\n", wantHTTPStatus: http.StatusUnauthorized, wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { - name: "token authenticates as nil", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, - request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), + name: "token authenticates as nil", + request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { recorder.AuthenticateToken(gomock.Any(), "test-token").Return(nil, false, nil) }, @@ -254,8 +197,47 @@ func TestImpersonator(t *testing.T) { }, // happy path { - name: "token validates", - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, + name: "token validates", + request: newRequest(map[string][]string{ + "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Malicious-Header": {"test-header-value-1"}, // this header should not be forwarded to the Kube API server + }), + expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { + userInfo := user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + UID: "test-uid", + Extra: testExtra, + } + response := &authenticator.Response{User: &userInfo} + recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) + }, + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, + }, + { + name: "token validates and the kube API request returns an error", request: newRequest(map[string][]string{ "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, "Malicious-Header": {"test-header-value-1"}, @@ -271,14 +253,22 @@ func TestImpersonator(t *testing.T) { response := &authenticator.Response{User: &userInfo} recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) }, - wantHTTPBody: "successful proxied response", - wantHTTPStatus: http.StatusOK, + wantKubeAPIServerStatusCode: http.StatusNotFound, + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Accept-Encoding": {"gzip"}, // because the rest client used in this test does not disable compression + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + }, + wantHTTPStatus: http.StatusNotFound, wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, }, { name: "token validates with custom api group", apiGroupOverride: customAPIGroup, - getKubeconfig: func() (*rest.Config, error) { return &testServerKubeconfig, nil }, request: newRequest(map[string][]string{ "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}, "Malicious-Header": {"test-header-value-1"}, @@ -294,6 +284,15 @@ func TestImpersonator(t *testing.T) { response := &authenticator.Response{User: &userInfo} recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) }, + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Accept-Encoding": {"gzip"}, // because the rest client used in this test does not disable compression + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, @@ -312,6 +311,32 @@ func TestImpersonator(t *testing.T) { } }() + if tt.wantKubeAPIServerStatusCode == 0 { + tt.wantKubeAPIServerStatusCode = http.StatusOK + } + + serverWasCalled := false + serverSawHeaders := http.Header{} + testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + serverWasCalled = true + serverSawHeaders = r.Header + if tt.wantKubeAPIServerStatusCode != http.StatusOK { + w.WriteHeader(tt.wantKubeAPIServerStatusCode) + } else { + _, _ = w.Write([]byte("successful proxied response")) + } + }) + testServerKubeconfig := rest.Config{ + Host: testServerURL, + BearerToken: "some-service-account-token", + TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testServerCA)}, + } + if tt.getKubeconfig == nil { + tt.getKubeconfig = func() (*rest.Config, error) { + return &testServerKubeconfig, nil + } + } + // stole this from cache_test, hopefully it is sufficient cacheWithMockAuthenticator := authncache.New() ctrl := gomock.NewController(t) @@ -349,6 +374,13 @@ func TestImpersonator(t *testing.T) { if tt.wantLogs != nil { require.Equal(t, tt.wantLogs, testLog.Lines()) } + + if tt.wantHTTPStatus == http.StatusOK || tt.wantKubeAPIServerStatusCode != http.StatusOK { + require.True(t, serverWasCalled, "Should have proxied the request to the Kube API server, but didn't") + require.Equal(t, tt.wantKubeAPIServerRequestHeaders, serverSawHeaders) + } else { + require.False(t, serverWasCalled, "Should not have proxied the request to the Kube API server, but did") + } }) } } diff --git a/internal/testutil/impersonationtoken/impersonationtoken.go b/internal/testutil/impersonationtoken/impersonationtoken.go index e2c47dd8..d9bcc69e 100644 --- a/internal/testutil/impersonationtoken/impersonationtoken.go +++ b/internal/testutil/impersonationtoken/impersonationtoken.go @@ -63,5 +63,5 @@ func Make( reqJSON, err := runtime.Encode(respInfo.PrettySerializer, &tokenCredentialRequest) require.NoError(t, err) - return base64.RawURLEncoding.EncodeToString(reqJSON) + return base64.StdEncoding.EncodeToString(reqJSON) } diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 637f2002..935a1b5e 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -12,8 +12,12 @@ import ( "time" "github.com/stretchr/testify/require" + v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "sigs.k8s.io/yaml" @@ -124,12 +128,94 @@ func TestImpersonationProxy(t *testing.T) { } t.Run("watching all the verbs", func(t *testing.T) { - // Start a watch in a informer. - // Create an RBAC rule to allow this user to read/write everything. - // t.Cleanup Delete the RBAC rule. // Create a namespace, because it will be easier to deletecollection if we have a namespace. // t.Cleanup Delete the namespace. - // Then "create" several Secrets. + namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + t.Logf("cleaning up test namespace %s", namespace.Name) + err = adminClient.CoreV1().Namespaces().Delete(context.Background(), namespace.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + }) + + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding( + t, + rbacv1.Subject{ + Kind: rbacv1.UserKind, + APIGroup: rbacv1.GroupName, + Name: env.TestUser.ExpectedUsername, + }, + rbacv1.RoleRef{ + Kind: "ClusterRole", + APIGroup: rbacv1.GroupName, + Name: "cluster-admin", + }, + ) + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespace.Name, + Verb: "create", + Group: "", + Version: "v1", + Resource: "configmaps", + }) + + // Create and start informer. + informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( + impersonationProxyClient, + 0, + k8sinformers.WithNamespace(namespace.Name)) + informer := informerFactory.Core().V1().ConfigMaps() + informer.Informer() // makes sure that the informer will cache + stopChannel := make(chan struct{}) + informerFactory.Start(stopChannel) + t.Cleanup(func() { + stopChannel <- struct{}{} + }) + informerFactory.WaitForCacheSync(ctx.Done()) + + // Test "create" verb. + _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create( + ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1"}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, err = informer.Lister().ConfigMaps(namespace.Name).Get("configmap-1") + return err == nil + }, 10*time.Second, 500*time.Millisecond) + + _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create( + ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2"}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, err = informer.Lister().ConfigMaps(namespace.Name).Get("configmap-2") + return err == nil + }, 10*time.Second, 500*time.Millisecond) + + _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create( + ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3"}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + require.Eventually(t, func() bool { + _, err = informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + return err == nil + }, 10*time.Second, 500*time.Millisecond) + + require.Eventually(t, func() bool { + configmaps, err := informer.Lister().ConfigMaps(namespace.Name).List(labels.Everything()) + return err == nil && len(configmaps) == 3 + }, 10*time.Second, 500*time.Millisecond) + + // TODO, test more verbs // "get" one them. // "list" them all. // "update" one of them. @@ -137,7 +223,6 @@ func TestImpersonationProxy(t *testing.T) { // "delete" one of them. // "deletecollection" all of them. // Make sure the watch sees all of those actions. - // Close the informer. }) // Update configuration to force the proxy to disabled mode diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index e0b2cc75..092d46a7 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -23,6 +23,7 @@ import ( coreosoidc "github.com/coreos/go-oidc/v3/oidc" "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -135,6 +136,12 @@ func TestE2EFullIntegration(t *testing.T) { rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.SupervisorTestUpstream.Username}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "view"}, ) + library.WaitForUserToHaveAccess(t, env.SupervisorTestUpstream.Username, []string{}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) // Use a specific session cache for this test. sessionCachePath := tempDir + "/sessions.yaml" diff --git a/test/library/access.go b/test/library/access.go index 7aacf6b1..e953cff8 100644 --- a/test/library/access.go +++ b/test/library/access.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package library @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" v1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -137,6 +138,12 @@ func addTestClusterUserCanViewEverythingRoleBinding(t *testing.T, testUsername s Name: "view", }, ) + WaitForUserToHaveAccess(t, testUsername, []string{}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) } func addTestClusterGroupCanViewEverythingRoleBinding(t *testing.T, testGroup string) { @@ -154,6 +161,12 @@ func addTestClusterGroupCanViewEverythingRoleBinding(t *testing.T, testGroup str Name: "view", }, ) + WaitForUserToHaveAccess(t, "", []string{testGroup}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) } func runKubectlGetNamespaces(t *testing.T, kubeConfigYAML string) (string, error) { diff --git a/test/library/client.go b/test/library/client.go index 6d7facb6..fb5a4b63 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -434,6 +435,27 @@ func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef return created } +func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldHaveAccessTo *authorizationv1.ResourceAttributes) { + t.Helper() + client := NewKubernetesClientset(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + RequireEventuallyWithoutError(t, func() (bool, error) { + subjectAccessReview, err := client.AuthorizationV1().SubjectAccessReviews().Create(ctx, + &authorizationv1.SubjectAccessReview{ + Spec: authorizationv1.SubjectAccessReviewSpec{ + ResourceAttributes: shouldHaveAccessTo, + User: user, + Groups: groups, + }}, metav1.CreateOptions{}) + if err != nil { + return false, err + } + return subjectAccessReview.Status.Allowed, nil + }, 10*time.Second, 500*time.Millisecond) +} + func testObjectMeta(t *testing.T, baseName string) metav1.ObjectMeta { return metav1.ObjectMeta{ GenerateName: fmt.Sprintf("test-%s-", baseName), From dac1c9939e01e3fdcced0fba237c2749a1856f3d Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 23 Feb 2021 10:38:02 -0800 Subject: [PATCH 033/203] concierge_impersonation_proxy_test.go: Test all the verbs Also: - Shut down the informer correctly in concierge_impersonation_proxy_test.go - Remove the t.Failed() checks which avoid cleaning up after failed tests. This was inconsistent with how most of the tests work, and left cruft on clusters when a test failed. Signed-off-by: Ryan Richard --- .../concierge_impersonation_proxy_test.go | 114 +++++++++++++----- test/integration/kubeclient_test.go | 15 +-- test/library/client.go | 12 -- 3 files changed, 89 insertions(+), 52 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 935a1b5e..08101533 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -15,8 +15,10 @@ import ( v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/types" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -172,57 +174,103 @@ func TestImpersonationProxy(t *testing.T) { stopChannel := make(chan struct{}) informerFactory.Start(stopChannel) t.Cleanup(func() { - stopChannel <- struct{}{} + // Shut down the informer. + close(stopChannel) }) informerFactory.WaitForCacheSync(ctx.Done()) - // Test "create" verb. - _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create( - ctx, + // Test "create" verb through the impersonation proxy. + _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1"}}, metav1.CreateOptions{}, ) require.NoError(t, err) - require.Eventually(t, func() bool { - _, err = informer.Lister().ConfigMaps(namespace.Name).Get("configmap-1") - return err == nil - }, 10*time.Second, 500*time.Millisecond) - - _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create( - ctx, + _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2"}}, metav1.CreateOptions{}, ) require.NoError(t, err) - require.Eventually(t, func() bool { - _, err = informer.Lister().ConfigMaps(namespace.Name).Get("configmap-2") - return err == nil - }, 10*time.Second, 500*time.Millisecond) - - _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create( - ctx, + _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3"}}, metav1.CreateOptions{}, ) require.NoError(t, err) - require.Eventually(t, func() bool { - _, err = informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") - return err == nil - }, 10*time.Second, 500*time.Millisecond) + // Make sure that all of the created ConfigMaps show up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. require.Eventually(t, func() bool { - configmaps, err := informer.Lister().ConfigMaps(namespace.Name).List(labels.Everything()) - return err == nil && len(configmaps) == 3 - }, 10*time.Second, 500*time.Millisecond) + _, err1 := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-1") + _, err2 := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-2") + _, err3 := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + return err1 == nil && err2 == nil && err3 == nil + }, 10*time.Second, 50*time.Millisecond) - // TODO, test more verbs - // "get" one them. - // "list" them all. - // "update" one of them. - // "patch" one of them. - // "delete" one of them. - // "deletecollection" all of them. - // Make sure the watch sees all of those actions. + // Test "get" verb through the impersonation proxy. + configMap3, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Get(ctx, "configmap-3", metav1.GetOptions{}) + require.NoError(t, err) + + // Test "list" verb through the impersonation proxy. + listResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, listResult.Items, 3) + + // Test "update" verb through the impersonation proxy. + configMap3.Data = map[string]string{"foo": "bar"} + updateResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Update(ctx, configMap3, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Equal(t, "bar", updateResult.Data["foo"]) + + // Make sure that the updated ConfigMap shows up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + require.Eventually(t, func() bool { + configMap, err := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + return err == nil && configMap.Data["foo"] == "bar" + }, 10*time.Second, 50*time.Millisecond) + + // Test "patch" verb through the impersonation proxy. + patchResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Patch(ctx, + "configmap-3", + types.MergePatchType, + []byte(`{"data":{"baz":"42"}}`), + metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Equal(t, "bar", patchResult.Data["foo"]) + require.Equal(t, "42", patchResult.Data["baz"]) + + // Make sure that the patched ConfigMap shows up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + require.Eventually(t, func() bool { + configMap, err := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + return err == nil && configMap.Data["foo"] == "bar" && configMap.Data["baz"] == "42" + }, 10*time.Second, 50*time.Millisecond) + + // Test "delete" verb through the impersonation proxy. + err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) + require.NoError(t, err) + + // Make sure that the deleted ConfigMap shows up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + require.Eventually(t, func() bool { + _, getErr := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(labels.Everything()) + return k8serrors.IsNotFound(getErr) && listErr == nil && len(list) == 2 + }, 10*time.Second, 50*time.Millisecond) + + // Test "deletecollection" verb through the impersonation proxy. + err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + require.NoError(t, err) + + // Make sure that the deleted ConfigMaps shows up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + require.Eventually(t, func() bool { + list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(labels.Everything()) + return listErr == nil && len(list) == 0 + }, 10*time.Second, 50*time.Millisecond) + + listResult, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, listResult.Items, 0) }) // Update configuration to force the proxy to disabled mode diff --git a/test/integration/kubeclient_test.go b/test/integration/kubeclient_test.go index 6cdaa56d..53826d6e 100644 --- a/test/integration/kubeclient_test.go +++ b/test/integration/kubeclient_test.go @@ -42,13 +42,12 @@ func TestKubeClientOwnerRef(t *testing.T) { ) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() err := namespaces.Delete(ctx, namespace.Name, metav1.DeleteOptions{}) require.NoError(t, err) - }() + }) // create something that we can point to parentSecret, err := regularClient.CoreV1().Secrets(namespace.Name).Create( @@ -91,13 +90,15 @@ func TestKubeClientOwnerRef(t *testing.T) { metav1.CreateOptions{}, ) require.NoError(t, err) - defer func() { + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() err := regularAggregationClient.ApiregistrationV1().APIServices().Delete(ctx, parentAPIService.Name, metav1.DeleteOptions{}) if errors.IsNotFound(err) { return } require.NoError(t, err) - }() + }) // work around stupid behavior of WithoutVersionDecoder.Decode parentAPIService.APIVersion, parentAPIService.Kind = apiregistrationv1.SchemeGroupVersion.WithKind("APIService").ToAPIVersionAndKind() diff --git a/test/library/client.go b/test/library/client.go index da1653e5..e4fb40a9 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -168,12 +168,6 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty t.Cleanup(func() { t.Helper() - - if t.Failed() { - t.Logf("skipping deletion of test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) - return - } - t.Logf("cleaning up test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() @@ -237,12 +231,6 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp t.Cleanup(func() { t.Helper() - - if t.Failed() { - t.Logf("skipping deletion of test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) - return - } - t.Logf("cleaning up test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() From 96d7743eabdb286ccb5870017ee7f1815cc0703e Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 24 Feb 2021 10:45:25 -0600 Subject: [PATCH 034/203] Add CredentialIssuer API fields for impersonation proxy. Adds a new optional `spec.impersonationProxyInfo` field to hold the URL and CA data for the impersonation proxy, as well as some additional status condition constants for describing the current status of the impersonation proxy. Signed-off-by: Matt Moyer --- .../v1alpha1/types_credentialissuer.go.tmpl | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index 63d59446..5af75b73 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -16,12 +16,15 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") FetchedKeyStrategyReason = StrategyReason("FetchedKey") + ListeningStrategyReason = StrategyReason("Listening") + DisabledStrategyReason = StrategyReason("Disabled") ) // Status of a credential issuer. @@ -29,19 +32,35 @@ type CredentialIssuerStatus struct { // List of integration strategies that were attempted by Pinniped. Strategies []CredentialIssuerStrategy `json:"strategies"` - // Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. + // Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. // +optional KubeConfigInfo *CredentialIssuerKubeConfigInfo `json:"kubeConfigInfo,omitempty"` + + // Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. + // +optional + ImpersonationProxyInfo *CredentialIssuerImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } -// Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +// Information needed to connect to the TokenCredentialRequest API on this cluster. type CredentialIssuerKubeConfigInfo struct { - // The K8s API server URL. + // The Kubernetes API server URL. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // The K8s API server CA bundle. + // The Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// Information needed to connect to the TokenCredentialRequest API on this cluster. +type CredentialIssuerImpersonationProxyInfo struct { + // The HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // The CA bundle to validate connections to the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } From 7be8927d5ee1faad016bfe78cc71e2d0682db07d Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 24 Feb 2021 10:47:06 -0600 Subject: [PATCH 035/203] Add generated code for new CredentialIssuer API fields. Signed-off-by: Matt Moyer --- ...cierge.pinniped.dev_credentialissuers.yaml | 26 +++++++++++++++--- generated/1.17/README.adoc | 25 ++++++++++++++--- .../config/v1alpha1/types_credentialissuer.go | 27 ++++++++++++++++--- .../config/v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++ ...cierge.pinniped.dev_credentialissuers.yaml | 26 +++++++++++++++--- generated/1.18/README.adoc | 25 ++++++++++++++--- .../config/v1alpha1/types_credentialissuer.go | 27 ++++++++++++++++--- .../config/v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++ ...cierge.pinniped.dev_credentialissuers.yaml | 26 +++++++++++++++--- generated/1.19/README.adoc | 25 ++++++++++++++--- .../config/v1alpha1/types_credentialissuer.go | 27 ++++++++++++++++--- .../config/v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++ ...cierge.pinniped.dev_credentialissuers.yaml | 26 +++++++++++++++--- generated/1.20/README.adoc | 25 ++++++++++++++--- .../config/v1alpha1/types_credentialissuer.go | 27 ++++++++++++++++--- .../config/v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++ ...cierge.pinniped.dev_credentialissuers.yaml | 26 +++++++++++++++--- .../config/v1alpha1/types_credentialissuer.go | 27 ++++++++++++++++--- .../config/v1alpha1/zz_generated.deepcopy.go | 21 +++++++++++++++ 19 files changed, 418 insertions(+), 52 deletions(-) diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index 900db6cb..72152f15 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -38,16 +38,34 @@ spec: status: description: Status of the credential issuer. properties: - kubeConfigInfo: + impersonationProxyInfo: description: Information needed to form a valid Pinniped-based kubeconfig - using this credential issuer. + using the impersonation proxy. properties: certificateAuthorityData: - description: The K8s API server CA bundle. + description: The CA bundle to validate connections to the impersonation + proxy. + minLength: 1 + type: string + endpoint: + description: The HTTPS endpoint of the impersonation proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object + kubeConfigInfo: + description: Information needed to form a valid Pinniped-based kubeconfig + using the TokenCredentialRequest API. + properties: + certificateAuthorityData: + description: The Kubernetes API server CA bundle. minLength: 1 type: string server: - description: The K8s API server URL. + description: The Kubernetes API server URL. minLength: 1 pattern: ^https://|^http:// type: string diff --git a/generated/1.17/README.adoc b/generated/1.17/README.adoc index d909c9de..7f5b6184 100644 --- a/generated/1.17/README.adoc +++ b/generated/1.17/README.adoc @@ -236,6 +236,24 @@ Describes the configuration status of a Pinniped credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo"] +==== CredentialIssuerImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | The HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | The CA bundle to validate connections to the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo"] ==== CredentialIssuerKubeConfigInfo @@ -249,8 +267,8 @@ Describes the configuration status of a Pinniped credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | The K8s API server URL. -| *`certificateAuthorityData`* __string__ | The K8s API server CA bundle. +| *`server`* __string__ | The Kubernetes API server URL. +| *`certificateAuthorityData`* __string__ | The Kubernetes API server CA bundle. |=== @@ -270,7 +288,8 @@ Status of a credential issuer. |=== | Field | Description | *`strategies`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerstrategy[$$CredentialIssuerStrategy$$] array__ | List of integration strategies that were attempted by Pinniped. -| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-17-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo[$$CredentialIssuerImpersonationProxyInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. |=== diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index 63d59446..5af75b73 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -16,12 +16,15 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") FetchedKeyStrategyReason = StrategyReason("FetchedKey") + ListeningStrategyReason = StrategyReason("Listening") + DisabledStrategyReason = StrategyReason("Disabled") ) // Status of a credential issuer. @@ -29,19 +32,35 @@ type CredentialIssuerStatus struct { // List of integration strategies that were attempted by Pinniped. Strategies []CredentialIssuerStrategy `json:"strategies"` - // Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. + // Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. // +optional KubeConfigInfo *CredentialIssuerKubeConfigInfo `json:"kubeConfigInfo,omitempty"` + + // Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. + // +optional + ImpersonationProxyInfo *CredentialIssuerImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } -// Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +// Information needed to connect to the TokenCredentialRequest API on this cluster. type CredentialIssuerKubeConfigInfo struct { - // The K8s API server URL. + // The Kubernetes API server URL. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // The K8s API server CA bundle. + // The Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// Information needed to connect to the TokenCredentialRequest API on this cluster. +type CredentialIssuerImpersonationProxyInfo struct { + // The HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // The CA bundle to validate connections to the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index eebbe7af..ffbbbc50 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -38,6 +38,22 @@ func (in *CredentialIssuer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopyInto(out *CredentialIssuerImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerImpersonationProxyInfo. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopy() *CredentialIssuerImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(CredentialIssuerImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerKubeConfigInfo) DeepCopyInto(out *CredentialIssuerKubeConfigInfo) { *out = *in @@ -102,6 +118,11 @@ func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = new(CredentialIssuerKubeConfigInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(CredentialIssuerImpersonationProxyInfo) + **out = **in + } return } diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 900db6cb..72152f15 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -38,16 +38,34 @@ spec: status: description: Status of the credential issuer. properties: - kubeConfigInfo: + impersonationProxyInfo: description: Information needed to form a valid Pinniped-based kubeconfig - using this credential issuer. + using the impersonation proxy. properties: certificateAuthorityData: - description: The K8s API server CA bundle. + description: The CA bundle to validate connections to the impersonation + proxy. + minLength: 1 + type: string + endpoint: + description: The HTTPS endpoint of the impersonation proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object + kubeConfigInfo: + description: Information needed to form a valid Pinniped-based kubeconfig + using the TokenCredentialRequest API. + properties: + certificateAuthorityData: + description: The Kubernetes API server CA bundle. minLength: 1 type: string server: - description: The K8s API server URL. + description: The Kubernetes API server URL. minLength: 1 pattern: ^https://|^http:// type: string diff --git a/generated/1.18/README.adoc b/generated/1.18/README.adoc index a40e3568..044cb262 100644 --- a/generated/1.18/README.adoc +++ b/generated/1.18/README.adoc @@ -236,6 +236,24 @@ Describes the configuration status of a Pinniped credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo"] +==== CredentialIssuerImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | The HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | The CA bundle to validate connections to the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo"] ==== CredentialIssuerKubeConfigInfo @@ -249,8 +267,8 @@ Describes the configuration status of a Pinniped credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | The K8s API server URL. -| *`certificateAuthorityData`* __string__ | The K8s API server CA bundle. +| *`server`* __string__ | The Kubernetes API server URL. +| *`certificateAuthorityData`* __string__ | The Kubernetes API server CA bundle. |=== @@ -270,7 +288,8 @@ Status of a credential issuer. |=== | Field | Description | *`strategies`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerstrategy[$$CredentialIssuerStrategy$$] array__ | List of integration strategies that were attempted by Pinniped. -| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-18-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo[$$CredentialIssuerImpersonationProxyInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. |=== diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index 63d59446..5af75b73 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -16,12 +16,15 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") FetchedKeyStrategyReason = StrategyReason("FetchedKey") + ListeningStrategyReason = StrategyReason("Listening") + DisabledStrategyReason = StrategyReason("Disabled") ) // Status of a credential issuer. @@ -29,19 +32,35 @@ type CredentialIssuerStatus struct { // List of integration strategies that were attempted by Pinniped. Strategies []CredentialIssuerStrategy `json:"strategies"` - // Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. + // Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. // +optional KubeConfigInfo *CredentialIssuerKubeConfigInfo `json:"kubeConfigInfo,omitempty"` + + // Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. + // +optional + ImpersonationProxyInfo *CredentialIssuerImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } -// Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +// Information needed to connect to the TokenCredentialRequest API on this cluster. type CredentialIssuerKubeConfigInfo struct { - // The K8s API server URL. + // The Kubernetes API server URL. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // The K8s API server CA bundle. + // The Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// Information needed to connect to the TokenCredentialRequest API on this cluster. +type CredentialIssuerImpersonationProxyInfo struct { + // The HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // The CA bundle to validate connections to the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index eebbe7af..ffbbbc50 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -38,6 +38,22 @@ func (in *CredentialIssuer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopyInto(out *CredentialIssuerImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerImpersonationProxyInfo. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopy() *CredentialIssuerImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(CredentialIssuerImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerKubeConfigInfo) DeepCopyInto(out *CredentialIssuerKubeConfigInfo) { *out = *in @@ -102,6 +118,11 @@ func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = new(CredentialIssuerKubeConfigInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(CredentialIssuerImpersonationProxyInfo) + **out = **in + } return } diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 900db6cb..72152f15 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -38,16 +38,34 @@ spec: status: description: Status of the credential issuer. properties: - kubeConfigInfo: + impersonationProxyInfo: description: Information needed to form a valid Pinniped-based kubeconfig - using this credential issuer. + using the impersonation proxy. properties: certificateAuthorityData: - description: The K8s API server CA bundle. + description: The CA bundle to validate connections to the impersonation + proxy. + minLength: 1 + type: string + endpoint: + description: The HTTPS endpoint of the impersonation proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object + kubeConfigInfo: + description: Information needed to form a valid Pinniped-based kubeconfig + using the TokenCredentialRequest API. + properties: + certificateAuthorityData: + description: The Kubernetes API server CA bundle. minLength: 1 type: string server: - description: The K8s API server URL. + description: The Kubernetes API server URL. minLength: 1 pattern: ^https://|^http:// type: string diff --git a/generated/1.19/README.adoc b/generated/1.19/README.adoc index 0b74cef3..5365664f 100644 --- a/generated/1.19/README.adoc +++ b/generated/1.19/README.adoc @@ -236,6 +236,24 @@ Describes the configuration status of a Pinniped credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo"] +==== CredentialIssuerImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | The HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | The CA bundle to validate connections to the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo"] ==== CredentialIssuerKubeConfigInfo @@ -249,8 +267,8 @@ Describes the configuration status of a Pinniped credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | The K8s API server URL. -| *`certificateAuthorityData`* __string__ | The K8s API server CA bundle. +| *`server`* __string__ | The Kubernetes API server URL. +| *`certificateAuthorityData`* __string__ | The Kubernetes API server CA bundle. |=== @@ -270,7 +288,8 @@ Status of a credential issuer. |=== | Field | Description | *`strategies`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerstrategy[$$CredentialIssuerStrategy$$] array__ | List of integration strategies that were attempted by Pinniped. -| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-19-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo[$$CredentialIssuerImpersonationProxyInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. |=== diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index 63d59446..5af75b73 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -16,12 +16,15 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") FetchedKeyStrategyReason = StrategyReason("FetchedKey") + ListeningStrategyReason = StrategyReason("Listening") + DisabledStrategyReason = StrategyReason("Disabled") ) // Status of a credential issuer. @@ -29,19 +32,35 @@ type CredentialIssuerStatus struct { // List of integration strategies that were attempted by Pinniped. Strategies []CredentialIssuerStrategy `json:"strategies"` - // Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. + // Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. // +optional KubeConfigInfo *CredentialIssuerKubeConfigInfo `json:"kubeConfigInfo,omitempty"` + + // Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. + // +optional + ImpersonationProxyInfo *CredentialIssuerImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } -// Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +// Information needed to connect to the TokenCredentialRequest API on this cluster. type CredentialIssuerKubeConfigInfo struct { - // The K8s API server URL. + // The Kubernetes API server URL. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // The K8s API server CA bundle. + // The Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// Information needed to connect to the TokenCredentialRequest API on this cluster. +type CredentialIssuerImpersonationProxyInfo struct { + // The HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // The CA bundle to validate connections to the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index eebbe7af..ffbbbc50 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -38,6 +38,22 @@ func (in *CredentialIssuer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopyInto(out *CredentialIssuerImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerImpersonationProxyInfo. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopy() *CredentialIssuerImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(CredentialIssuerImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerKubeConfigInfo) DeepCopyInto(out *CredentialIssuerKubeConfigInfo) { *out = *in @@ -102,6 +118,11 @@ func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = new(CredentialIssuerKubeConfigInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(CredentialIssuerImpersonationProxyInfo) + **out = **in + } return } diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 900db6cb..72152f15 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -38,16 +38,34 @@ spec: status: description: Status of the credential issuer. properties: - kubeConfigInfo: + impersonationProxyInfo: description: Information needed to form a valid Pinniped-based kubeconfig - using this credential issuer. + using the impersonation proxy. properties: certificateAuthorityData: - description: The K8s API server CA bundle. + description: The CA bundle to validate connections to the impersonation + proxy. + minLength: 1 + type: string + endpoint: + description: The HTTPS endpoint of the impersonation proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object + kubeConfigInfo: + description: Information needed to form a valid Pinniped-based kubeconfig + using the TokenCredentialRequest API. + properties: + certificateAuthorityData: + description: The Kubernetes API server CA bundle. minLength: 1 type: string server: - description: The K8s API server URL. + description: The Kubernetes API server URL. minLength: 1 pattern: ^https://|^http:// type: string diff --git a/generated/1.20/README.adoc b/generated/1.20/README.adoc index 76a678f4..5e16c419 100644 --- a/generated/1.20/README.adoc +++ b/generated/1.20/README.adoc @@ -236,6 +236,24 @@ Describes the configuration status of a Pinniped credential issuer. |=== +[id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo"] +==== CredentialIssuerImpersonationProxyInfo + + + +.Appears In: +**** +- xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerstatus[$$CredentialIssuerStatus$$] +**** + +[cols="25a,75a", options="header"] +|=== +| Field | Description +| *`endpoint`* __string__ | The HTTPS endpoint of the impersonation proxy. +| *`certificateAuthorityData`* __string__ | The CA bundle to validate connections to the impersonation proxy. +|=== + + [id="{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo"] ==== CredentialIssuerKubeConfigInfo @@ -249,8 +267,8 @@ Describes the configuration status of a Pinniped credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | The K8s API server URL. -| *`certificateAuthorityData`* __string__ | The K8s API server CA bundle. +| *`server`* __string__ | The Kubernetes API server URL. +| *`certificateAuthorityData`* __string__ | The Kubernetes API server CA bundle. |=== @@ -270,7 +288,8 @@ Status of a credential issuer. |=== | Field | Description | *`strategies`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerstrategy[$$CredentialIssuerStrategy$$] array__ | List of integration strategies that were attempted by Pinniped. -| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +| *`kubeConfigInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerkubeconfiginfo[$$CredentialIssuerKubeConfigInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. +| *`impersonationProxyInfo`* __xref:{anchor_prefix}-go-pinniped-dev-generated-1-20-apis-concierge-config-v1alpha1-credentialissuerimpersonationproxyinfo[$$CredentialIssuerImpersonationProxyInfo$$]__ | Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. |=== diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index 63d59446..5af75b73 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -16,12 +16,15 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") FetchedKeyStrategyReason = StrategyReason("FetchedKey") + ListeningStrategyReason = StrategyReason("Listening") + DisabledStrategyReason = StrategyReason("Disabled") ) // Status of a credential issuer. @@ -29,19 +32,35 @@ type CredentialIssuerStatus struct { // List of integration strategies that were attempted by Pinniped. Strategies []CredentialIssuerStrategy `json:"strategies"` - // Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. + // Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. // +optional KubeConfigInfo *CredentialIssuerKubeConfigInfo `json:"kubeConfigInfo,omitempty"` + + // Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. + // +optional + ImpersonationProxyInfo *CredentialIssuerImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } -// Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +// Information needed to connect to the TokenCredentialRequest API on this cluster. type CredentialIssuerKubeConfigInfo struct { - // The K8s API server URL. + // The Kubernetes API server URL. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // The K8s API server CA bundle. + // The Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// Information needed to connect to the TokenCredentialRequest API on this cluster. +type CredentialIssuerImpersonationProxyInfo struct { + // The HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // The CA bundle to validate connections to the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index eebbe7af..ffbbbc50 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -38,6 +38,22 @@ func (in *CredentialIssuer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopyInto(out *CredentialIssuerImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerImpersonationProxyInfo. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopy() *CredentialIssuerImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(CredentialIssuerImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerKubeConfigInfo) DeepCopyInto(out *CredentialIssuerKubeConfigInfo) { *out = *in @@ -102,6 +118,11 @@ func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = new(CredentialIssuerKubeConfigInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(CredentialIssuerImpersonationProxyInfo) + **out = **in + } return } diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 900db6cb..72152f15 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -38,16 +38,34 @@ spec: status: description: Status of the credential issuer. properties: - kubeConfigInfo: + impersonationProxyInfo: description: Information needed to form a valid Pinniped-based kubeconfig - using this credential issuer. + using the impersonation proxy. properties: certificateAuthorityData: - description: The K8s API server CA bundle. + description: The CA bundle to validate connections to the impersonation + proxy. + minLength: 1 + type: string + endpoint: + description: The HTTPS endpoint of the impersonation proxy. + minLength: 1 + pattern: ^https:// + type: string + required: + - certificateAuthorityData + - endpoint + type: object + kubeConfigInfo: + description: Information needed to form a valid Pinniped-based kubeconfig + using the TokenCredentialRequest API. + properties: + certificateAuthorityData: + description: The Kubernetes API server CA bundle. minLength: 1 type: string server: - description: The K8s API server URL. + description: The Kubernetes API server URL. minLength: 1 pattern: ^https://|^http:// type: string diff --git a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go index 63d59446..5af75b73 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -16,12 +16,15 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") FetchedKeyStrategyReason = StrategyReason("FetchedKey") + ListeningStrategyReason = StrategyReason("Listening") + DisabledStrategyReason = StrategyReason("Disabled") ) // Status of a credential issuer. @@ -29,19 +32,35 @@ type CredentialIssuerStatus struct { // List of integration strategies that were attempted by Pinniped. Strategies []CredentialIssuerStrategy `json:"strategies"` - // Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. + // Information needed to form a valid Pinniped-based kubeconfig using the TokenCredentialRequest API. // +optional KubeConfigInfo *CredentialIssuerKubeConfigInfo `json:"kubeConfigInfo,omitempty"` + + // Information needed to form a valid Pinniped-based kubeconfig using the impersonation proxy. + // +optional + ImpersonationProxyInfo *CredentialIssuerImpersonationProxyInfo `json:"impersonationProxyInfo,omitempty"` } -// Information needed to form a valid Pinniped-based kubeconfig using this credential issuer. +// Information needed to connect to the TokenCredentialRequest API on this cluster. type CredentialIssuerKubeConfigInfo struct { - // The K8s API server URL. + // The Kubernetes API server URL. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // The K8s API server CA bundle. + // The Kubernetes API server CA bundle. + // +kubebuilder:validation:MinLength=1 + CertificateAuthorityData string `json:"certificateAuthorityData"` +} + +// Information needed to connect to the TokenCredentialRequest API on this cluster. +type CredentialIssuerImpersonationProxyInfo struct { + // The HTTPS endpoint of the impersonation proxy. + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:Pattern=`^https://` + Endpoint string `json:"endpoint"` + + // The CA bundle to validate connections to the impersonation proxy. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go b/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go index eebbe7af..ffbbbc50 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go +++ b/generated/latest/apis/concierge/config/v1alpha1/zz_generated.deepcopy.go @@ -38,6 +38,22 @@ func (in *CredentialIssuer) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopyInto(out *CredentialIssuerImpersonationProxyInfo) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CredentialIssuerImpersonationProxyInfo. +func (in *CredentialIssuerImpersonationProxyInfo) DeepCopy() *CredentialIssuerImpersonationProxyInfo { + if in == nil { + return nil + } + out := new(CredentialIssuerImpersonationProxyInfo) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CredentialIssuerKubeConfigInfo) DeepCopyInto(out *CredentialIssuerKubeConfigInfo) { *out = *in @@ -102,6 +118,11 @@ func (in *CredentialIssuerStatus) DeepCopyInto(out *CredentialIssuerStatus) { *out = new(CredentialIssuerKubeConfigInfo) **out = **in } + if in.ImpersonationProxyInfo != nil { + in, out := &in.ImpersonationProxyInfo, &out.ImpersonationProxyInfo + *out = new(CredentialIssuerImpersonationProxyInfo) + **out = **in + } return } From 4dbde4cf7f62f5f4e77c3b8318881326457abbbc Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 24 Feb 2021 12:08:41 -0600 Subject: [PATCH 036/203] Fix TestImpersonationProxy on Kubernetes 1.20 with RootCAConfigMap. There is a new feature in 1.20 that creates a ConfigMap by default in each namespace: https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.20.md#introducing-rootcaconfigmap This broke this test because it assumed that all the ConfigMaps in the ephemeral test namespace were those created by the test code. The fix is to add a test label and rewrite our assertions to filter with it. Signed-off-by: Matt Moyer --- .../concierge_impersonation_proxy_test.go | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 08101533..1ec2fc08 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -180,18 +180,21 @@ func TestImpersonationProxy(t *testing.T) { informerFactory.WaitForCacheSync(ctx.Done()) // Test "create" verb through the impersonation proxy. + configMapLabels := labels.Set{ + "pinniped.dev/testConfigMap": library.RandHex(t, 8), + } _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) @@ -210,7 +213,9 @@ func TestImpersonationProxy(t *testing.T) { require.NoError(t, err) // Test "list" verb through the impersonation proxy. - listResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{}) + listResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: configMapLabels.String(), + }) require.NoError(t, err) require.Len(t, listResult.Items, 3) @@ -253,7 +258,7 @@ func TestImpersonationProxy(t *testing.T) { // demonstrate that the informer's "watch" verb is working through the impersonation proxy. require.Eventually(t, func() bool { _, getErr := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") - list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(labels.Everything()) + list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(configMapLabels.AsSelector()) return k8serrors.IsNotFound(getErr) && listErr == nil && len(list) == 2 }, 10*time.Second, 50*time.Millisecond) @@ -264,11 +269,13 @@ func TestImpersonationProxy(t *testing.T) { // Make sure that the deleted ConfigMaps shows up in the informer's cache to // demonstrate that the informer's "watch" verb is working through the impersonation proxy. require.Eventually(t, func() bool { - list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(labels.Everything()) + list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(configMapLabels.AsSelector()) return listErr == nil && len(list) == 0 }, 10*time.Second, 50*time.Millisecond) - listResult, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{}) + listResult, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + LabelSelector: configMapLabels.String(), + }) require.NoError(t, err) require.Len(t, listResult.Items, 0) }) From d42c533fbbcfde890d8ffe71b335781de62ddf1a Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 24 Feb 2021 10:56:24 -0800 Subject: [PATCH 037/203] WIP managing TLS secrets from the impersonation config controller Signed-off-by: Margo Crawford --- cmd/debug-make-impersonation-token/main.go | 2 +- .../impersonatorconfig/impersonator_config.go | 153 ++++++- .../impersonator_config_test.go | 422 +++++++++++++----- .../controllermanager/prepare_controllers.go | 4 +- 4 files changed, 452 insertions(+), 129 deletions(-) diff --git a/cmd/debug-make-impersonation-token/main.go b/cmd/debug-make-impersonation-token/main.go index 07dddffe..972ab27e 100644 --- a/cmd/debug-make-impersonation-token/main.go +++ b/cmd/debug-make-impersonation-token/main.go @@ -37,5 +37,5 @@ func main() { if err != nil { panic(err) } - fmt.Println(base64.RawURLEncoding.EncodeToString(reqJSON)) + fmt.Println(base64.StdEncoding.EncodeToString(reqJSON)) } diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 8f75b20c..508d17a4 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -11,6 +11,7 @@ import ( "fmt" "net" "net/http" + "sync" "time" v1 "k8s.io/api/core/v1" @@ -38,13 +39,17 @@ type impersonatorConfigController struct { k8sClient kubernetes.Interface configMapsInformer corev1informers.ConfigMapInformer servicesInformer corev1informers.ServiceInformer + secretsInformer corev1informers.SecretInformer generatedLoadBalancerServiceName string + tlsSecretName string labels map[string]string startTLSListenerFunc StartTLSListenerFunc httpHandlerFactory func() (http.Handler, error) server *http.Server hasControlPlaneNodes *bool + tlsCert *tls.Certificate + tlsCertMutex sync.RWMutex } type StartTLSListenerFunc func(network, listenAddress string, config *tls.Config) (net.Listener, error) @@ -55,9 +60,11 @@ func NewImpersonatorConfigController( k8sClient kubernetes.Interface, configMapsInformer corev1informers.ConfigMapInformer, servicesInformer corev1informers.ServiceInformer, + secretsInformer corev1informers.SecretInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, generatedLoadBalancerServiceName string, + tlsSecretName string, labels map[string]string, startTLSListenerFunc StartTLSListenerFunc, httpHandlerFactory func() (http.Handler, error), @@ -71,7 +78,9 @@ func NewImpersonatorConfigController( k8sClient: k8sClient, configMapsInformer: configMapsInformer, servicesInformer: servicesInformer, + secretsInformer: secretsInformer, generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, + tlsSecretName: tlsSecretName, labels: labels, startTLSListenerFunc: startTLSListenerFunc, httpHandlerFactory: httpHandlerFactory, @@ -87,6 +96,11 @@ func NewImpersonatorConfigController( pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(generatedLoadBalancerServiceName, namespace), controllerlib.InformerOption{}, ), + withInformer( + secretsInformer, + pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(tlsSecretName, namespace), + controllerlib.InformerOption{}, + ), // Be sure to run once even if the ConfigMap that the informer is watching doesn't exist. withInitialEvent(controllerlib.Key{ Namespace: namespace, @@ -155,21 +169,31 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { } } + if c.shouldHaveTLSSecret(config) { + if err = c.ensureTLSSecretIsCreated(ctx.Context, config); err != nil { + // return err // TODO + } + } else { + if err = c.ensureTLSSecretIsRemoved(ctx.Context); err != nil { + return err + } + } + plog.Debug("Successfully finished impersonatorConfigController Sync") return nil } +func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { + return c.shouldHaveImpersonator(config) +} + func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { return (config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes) || config.Mode == impersonator.ModeEnabled } func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonator.Config) bool { - // start the load balancer only if: - // - the impersonator is running - // - the cluster is cloud hosted - // - there is no endpoint specified in the config - return c.server != nil && !*c.hasControlPlaneNodes && config.Endpoint == "" + return c.shouldHaveImpersonator(config) && config.Endpoint == "" } func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { @@ -189,15 +213,6 @@ func (c *impersonatorConfigController) ensureImpersonatorIsStarted() error { return nil } - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) - if err != nil { - return fmt.Errorf("could not create impersonation CA: %w", err) - } - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, []string{"impersonation-proxy"}, nil, 24*time.Hour) - if err != nil { - return fmt.Errorf("could not create impersonation cert: %w", err) - } - handler, err := c.httpHandlerFactory() if err != nil { return err @@ -206,7 +221,7 @@ func (c *impersonatorConfigController) ensureImpersonatorIsStarted() error { listener, err := c.startTLSListenerFunc("tcp", impersonationProxyPort, &tls.Config{ MinVersion: tls.VersionTLS12, // Allow v1.2 because clients like the default `curl` on MacOS don't support 1.3 yet. GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - return impersonationCert, nil + return c.getTLSCert(), nil }, }) if err != nil { @@ -239,6 +254,18 @@ func (c *impersonatorConfigController) isLoadBalancerRunning() (bool, error) { return true, nil } +func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, error) { + secret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) + notFound := k8serrors.IsNotFound(err) + if notFound { + return false, nil, nil + } + if err != nil { + return false, nil, err + } + return true, secret, nil +} + func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { running, err := c.isLoadBalancerRunning() if err != nil { @@ -295,3 +322,99 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C } return nil } + +func (c *impersonatorConfigController) ensureTLSSecretIsCreated(ctx context.Context, config *impersonator.Config) error { + tlsSecretExists, tlsSecret, err := c.tlsSecretExists() + if err != nil { + return err + } + if tlsSecretExists { + certPEM := tlsSecret.Data[v1.TLSCertKey] + keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] + tlsCert, _ := tls.X509KeyPair(certPEM, keyPEM) + // TODO handle err, like when the Secret did not contain the fields that we expected + c.setTLSCert(&tlsCert) + return nil + } + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) // TODO change the length of this + if err != nil { + return fmt.Errorf("could not create impersonation CA: %w", err) + } + var ips []net.IP + if config.Endpoint == "" { // TODO are there other cases where we need to do this? + lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + return nil + } + if err != nil { + return err + } + ingresses := lb.Status.LoadBalancer.Ingress + if len(ingresses) == 0 { + return nil + } + ip := ingresses[0].IP // TODO multiple ips + ips = []net.IP{net.ParseIP(ip)} + // check with informer to get the ip address of the load balancer if its available + // if not, return + } else { + ips = []net.IP{net.ParseIP(config.Endpoint)} + } + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, ips, 24*time.Hour) // TODO change the length of this + if err != nil { + return fmt.Errorf("could not create impersonation cert: %w", err) + } + + c.setTLSCert(impersonationCert) + + certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + // TODO error handling? + + // TODO handle error on create + c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, &v1.Secret{ + Type: v1.SecretTypeTLS, + ObjectMeta: metav1.ObjectMeta{ + Name: c.tlsSecretName, + Namespace: c.namespace, + Labels: c.labels, + }, + Data: map[string][]byte{ + "ca.crt": impersonationCA.Bundle(), + v1.TLSPrivateKeyKey: keyPEM, + v1.TLSCertKey: certPEM, + }, + }, metav1.CreateOptions{}) + return nil +} + +func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Context) error { + tlsSecretExists, _, err := c.tlsSecretExists() + if err != nil { + return err + } + if !tlsSecretExists { + return nil + } + plog.Info("Deleting TLS certificates for impersonation proxy", + "service", c.tlsSecretName, + "namespace", c.namespace) + err = c.k8sClient.CoreV1().Secrets(c.namespace).Delete(ctx, c.tlsSecretName, metav1.DeleteOptions{}) + if err != nil { + return err + } + + return nil +} + +func (c *impersonatorConfigController) setTLSCert(cert *tls.Certificate) { + c.tlsCertMutex.Lock() + defer c.tlsCertMutex.Unlock() + c.tlsCert = cert +} + +func (c *impersonatorConfigController) getTLSCert() *tls.Certificate { + c.tlsCertMutex.RLock() + defer c.tlsCertMutex.RUnlock() + return c.tlsCert +} diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index d441eabb..b6fee18e 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -6,6 +6,8 @@ package impersonatorconfig import ( "context" "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" "errors" "fmt" "io/ioutil" @@ -29,6 +31,7 @@ import ( coretesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" + "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/controllerlib" "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/testutil" @@ -61,12 +64,14 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" const generatedLoadBalancerServiceName = "some-service-resource-name" + const tlsSecretName = "some-secret-name" var r *require.Assertions var observableWithInformerOption *testutil.ObservableWithInformerOption var observableWithInitialEventOption *testutil.ObservableWithInitialEventOption var configMapsInformerFilter controllerlib.Filter var servicesInformerFilter controllerlib.Filter + var secretsInformerFilter controllerlib.Filter it.Before(func() { r = require.New(t) @@ -75,6 +80,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { sharedInformerFactory := kubeinformers.NewSharedInformerFactory(nil, 0) configMapsInformer := sharedInformerFactory.Core().V1().ConfigMaps() servicesInformer := sharedInformerFactory.Core().V1().Services() + secretsInformer := sharedInformerFactory.Core().V1().Secrets() _ = NewImpersonatorConfigController( installedInNamespace, @@ -82,15 +88,18 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { nil, configMapsInformer, servicesInformer, + secretsInformer, observableWithInformerOption.WithInformer, observableWithInitialEventOption.WithInitialEvent, generatedLoadBalancerServiceName, + tlsSecretName, nil, nil, nil, ) configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) servicesInformerFilter = observableWithInformerOption.GetFilterForInformer(servicesInformer) + secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) }) when("watching ConfigMap objects", func() { @@ -189,6 +198,54 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { }) }) + when("watching Secret objects", func() { + var subject controllerlib.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.Secret + + it.Before(func() { + subject = secretsInformerFilter + target = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: installedInNamespace}} + wrongNamespace = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: "wrong-namespace"}} + wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target Secret changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a Secret from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a Secret with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a Secret with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + when("starting up", func() { it("asks for an initial event because the ConfigMap may not exist yet and it needs to run anyway", func() { r.Equal(&controllerlib.Key{ @@ -205,6 +262,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" const generatedLoadBalancerServiceName = "some-service-resource-name" + const tlsSecretName = "some-secret-name" var labels = map[string]string{"app": "app-name", "other-key": "other-value"} var r *require.Assertions @@ -232,7 +290,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var err error //nolint: gosec // Intentionally binding to all network interfaces. - startedTLSListener, err = tls.Listen(network, ":0", config) // automatically choose the port for unit tests + startedTLSListener, err = tls.Listen(network, "127.0.0.1:0", config) // automatically choose the port for unit tests r.NoError(err) return &tlsListenerWrapper{listener: startedTLSListener, closeError: startTLSListenerUponCloseError}, nil } @@ -248,11 +306,21 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var requireTLSServerIsRunning = func() { + var requireTLSServerIsRunning = func(caCrt []byte) { r.Greater(startTLSListenerFuncWasCalled, 0) - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // TODO once we're using certs, do not skip verify + var tr *http.Transport + if caCrt == nil { + tr = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + } + } else { + rootCAs := x509.NewCertPool() + rootCAs.AppendCertsFromPEM(caCrt) + + tr = &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: rootCAs}, + } } client := &http.Client{Transport: tr} url := "https://" + startedTLSListener.Addr().String() @@ -268,6 +336,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal("hello world", string(body)) } + var requireTLSServerIsRunningWithoutCerts = func() { + r.Greater(startTLSListenerFuncWasCalled, 0) + + tr := &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + } + + client := &http.Client{Transport: tr} + url := "https://" + startedTLSListener.Addr().String() + req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) + r.NoError(err) + _, err = client.Do(req) + r.Error(err) + r.Regexp("Get .*: remote error: tls: unrecognized name", err.Error()) + } + var requireTLSServerIsNoLongerRunning = func() { r.Greater(startTLSListenerFuncWasCalled, 0) _, err := tls.Dial( @@ -276,7 +360,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // TODO once we're using certs, do not skip verify ) r.Error(err) - r.Regexp(`dial tcp \[::\]:[0-9]+: connect: connection refused`, err.Error()) + r.Regexp(`dial tcp .*: connect: connection refused`, err.Error()) } var requireTLSServerWasNeverStarted = func() { @@ -306,9 +390,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { kubeAPIClient, kubeInformers.Core().V1().ConfigMaps(), kubeInformers.Core().V1().Services(), + kubeInformers.Core().V1().Secrets(), controllerlib.WithInformer, controllerlib.WithInitialEvent, generatedLoadBalancerServiceName, + tlsSecretName, labels, startTLSListenerFunc, func() (http.Handler, error) { @@ -371,6 +457,44 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } + var createStubTLSSecret = func(resourceName string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Data: map[string][]byte{}, + } + } + + var createActualTLSSecret = func(resourceName string) *corev1.Secret { + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) + r.NoError(err) + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, []net.IP{net.ParseIP("127.0.0.1")}, 24*time.Hour) + r.NoError(err) + certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Data: map[string][]byte{ + "ca.crt": impersonationCA.Bundle(), + corev1.TLSPrivateKeyKey: keyPEM, + corev1.TLSCertKey: certPEM, + }, + } + } + var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -388,6 +512,30 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(client.Tracker().Add(loadBalancerService)) } + var addLoadBalancerServiceWithIPToTracker = func(resourceName string, client *kubernetesfake.Clientset) { + loadBalancerService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: "127.0.0.1"}, + }, + }, + }, + } + r.NoError(client.Tracker().Add(loadBalancerService)) + } + var deleteLoadBalancerServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { r.NoError(client.Tracker().Delete( schema.GroupVersionResource{Version: "v1", Resource: "services"}, @@ -419,9 +567,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var requireLoadBalancerWasCreated = func(action coretesting.Action) { - createLoadBalancerAction := action.(coretesting.CreateAction) - r.Equal("create", createLoadBalancerAction.GetVerb()) - createdLoadBalancerService := createLoadBalancerAction.GetObject().(*corev1.Service) + createAction := action.(coretesting.CreateAction) + r.Equal("create", createAction.GetVerb()) + createdLoadBalancerService := createAction.GetObject().(*corev1.Service) r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) @@ -430,9 +578,32 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var requireLoadBalancerDeleted = func(action coretesting.Action) { - deleteLoadBalancerAction := action.(coretesting.DeleteAction) - r.Equal("delete", deleteLoadBalancerAction.GetVerb()) - r.Equal(generatedLoadBalancerServiceName, deleteLoadBalancerAction.GetName()) + deleteAction := action.(coretesting.DeleteAction) + r.Equal("delete", deleteAction.GetVerb()) + r.Equal(generatedLoadBalancerServiceName, deleteAction.GetName()) + r.Equal("services", deleteAction.GetResource().Resource) + } + + var requireTLSSecretDeleted = func(action coretesting.Action) { + deleteAction := action.(coretesting.DeleteAction) + r.Equal("delete", deleteAction.GetVerb()) + r.Equal(tlsSecretName, deleteAction.GetName()) + r.Equal("secrets", deleteAction.GetResource().Resource) + } + + var requireTLSSecretWasCreated = func(action coretesting.Action) []byte { + createAction := action.(coretesting.CreateAction) + r.Equal("create", createAction.GetVerb()) + createdSecret := createAction.GetObject().(*corev1.Secret) + r.Equal(tlsSecretName, createdSecret.Name) + r.Equal(installedInNamespace, createdSecret.Namespace) + r.Equal(corev1.SecretTypeTLS, createdSecret.Type) + r.Equal(labels, createdSecret.Labels) + r.Len(createdSecret.Data, 3) + r.NotNil(createdSecret.Data["ca.crt"]) + r.NotNil(createdSecret.Data[corev1.TLSPrivateKeyKey]) + r.NotNil(createdSecret.Data[corev1.TLSCertKey]) + return createdSecret.Data["ca.crt"] } it.Before(func() { @@ -471,20 +642,24 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("there are visible control plane nodes and a loadbalancer", func() { + when("there are visible control plane nodes and a loadbalancer and a tls Secret", func() { it.Before(func() { addNodeWithRoleToTracker("control-plane") addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + tlsSecret := createStubTLSSecret(tlsSecretName) + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) }) - it("does not start the impersonator, deletes the loadbalancer", func() { + it("does not start the impersonator, deletes the loadbalancer, deletes the Secret", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerWasNeverStarted() - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) }) }) @@ -495,8 +670,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) }) - it("automatically starts the impersonator", func() { - requireTLSServerIsRunning() + it("starts the impersonator without tls certs", func() { + requireTLSServerIsRunningWithoutCerts() }) it("starts the load balancer automatically", func() { @@ -506,7 +681,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("there are not visible control plane nodes and a load balancer already exists", func() { + when("there are not visible control plane nodes and a load balancer already exists without an IP", func() { it.Before(func() { addNodeWithRoleToTracker("worker") addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) @@ -515,8 +690,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) }) - it("automatically starts the impersonator", func() { - requireTLSServerIsRunning() + it("starts the impersonator without tls certs", func() { + requireTLSServerIsRunningWithoutCerts() }) it("does not start the load balancer automatically", func() { @@ -537,6 +712,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) @@ -544,9 +720,39 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time - requireTLSServerIsRunning() // still running + requireTLSServerIsRunningWithoutCerts() // still running r.Len(kubeAPIClient.Actions(), 2) // no new API calls }) + + it("creates certs from the ip address listed on the load balancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunningWithoutCerts() + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + r.Len(kubeAPIClient.Actions(), 3) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunning(ca) // running with certs now + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + createdSecret := kubeAPIClient.Actions()[2].(coretesting.CreateAction).GetObject().(*corev1.Secret) + createdSecret.ResourceVersion = "1" + r.NoError(kubeInformerClient.Tracker().Add(createdSecret)) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 3) // no more actions + requireTLSServerIsRunning(ca) // still running + }) }) when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { @@ -587,8 +793,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: auto - endpoint: https://proxy.example.com:8443/ - `), + endpoint: 127.0.0.1 + `), // TODO what to do about ports + // TODO IP address and hostname should work ) }) @@ -614,9 +821,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator according to the settings in the ConfigMap", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) - r.Len(kubeAPIClient.Actions(), 1) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunning(ca) }) }) }) @@ -637,7 +845,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) when("the configuration is enabled mode", func() { - when("there are control plane nodes", func() { + when("no load balancer", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") addNodeWithRoleToTracker("control-plane") @@ -646,90 +854,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() - }) - - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") - }) - - it("does not start the load balancer", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 1) - requireNodesListed(kubeAPIClient.Actions()[0]) - }) - }) - - when("there are no control plane nodes but a loadbalancer already exists", func() { - it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) - }) - - it("starts the impersonator", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() - }) - - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") - }) - - it("does not start the load balancer", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 1) - requireNodesListed(kubeAPIClient.Actions()[0]) - }) - }) - - when("there are control plane nodes and a loadbalancer already exists", func() { - it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") - addNodeWithRoleToTracker("control-plane") - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) - }) - - it("starts the impersonator", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() - }) - - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") - }) - - it("stops the load balancer", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 2) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) - }) - }) - - when("there are no control plane nodes and there is no load balancer", func() { - it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") - addNodeWithRoleToTracker("worker") - }) - - it("starts the impersonator", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() + requireTLSServerIsRunningWithoutCerts() }) it("returns an error when the tls listener fails to start", func() { @@ -746,6 +871,55 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) }) }) + + when("a loadbalancer already exists", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + requireTLSServerIsRunningWithoutCerts() + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + }) + + it("does not start the load balancer", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) + }) + + when("a load balancer and a secret already exist", func() { + var tlsSecret *corev1.Secret + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + tlsSecret = createActualTLSSecret(tlsSecretName) + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("starts the impersonator with the existing tls certs, does not start loadbalancer or make tls secret", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSServerIsRunning(tlsSecret.Data["ca.crt"]) + }) + }) }) when("the configuration switches from enabled to disabled mode", func() { @@ -758,7 +932,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() + requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) @@ -782,7 +956,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() + requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 4) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) }) @@ -795,7 +969,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns the error from the sync function", func() { startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireTLSServerIsRunning() + requireTLSServerIsRunningWithoutCerts() updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") @@ -860,5 +1034,29 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not create load balancer: error on create") }) }) + + when("there is an error deleting the tls secret", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane") + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + tlsSecret := createStubTLSSecret(tlsSecretName) + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + startInformersAndController() + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("does not start the impersonator, deletes the loadbalancer, returns an error", func() { + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "error on delete") + requireTLSServerWasNeverStarted() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) + }) + }) }, spec.Parallel(), spec.Report(report.Terminal{})) } diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 73c61b9c..b76deec4 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -296,9 +296,11 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { client.Kubernetes, informers.installationNamespaceK8s.Core().V1().ConfigMaps(), informers.installationNamespaceK8s.Core().V1().Services(), + informers.installationNamespaceK8s.Core().V1().Secrets(), controllerlib.WithInformer, controllerlib.WithInitialEvent, - "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` + "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` + "pinniped-concierge-impersonation-proxy-tls-serving-certificate", // TODO this string should come from `c.NamesConfig` c.Labels, tls.Listen, func() (http.Handler, error) { From 943b0ff6ec56459368527cc40644641cc08d3a6f Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 24 Feb 2021 17:07:07 -0600 Subject: [PATCH 038/203] Switch login flags to use `--concierge-mode` flag instead of boolean flag. The login commands now expect either `--concierge-mode ImpersonationProxy` or `--concierge-mode TokenCredentialRequestAPI` (the default). This is partly a style choice, but I also think it helps in case we need to add a third major mode of operation at some point. I also cleaned up some other minor style items in the help text. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/flag_types.go | 46 +++++++++++++++++++++++++++ cmd/pinniped/cmd/flag_types_test.go | 30 +++++++++++++++++ cmd/pinniped/cmd/login_oidc.go | 40 +++++++++++++++-------- cmd/pinniped/cmd/login_oidc_test.go | 20 ++++++------ cmd/pinniped/cmd/login_static.go | 37 +++++++++++++-------- cmd/pinniped/cmd/login_static_test.go | 10 +++--- 6 files changed, 140 insertions(+), 43 deletions(-) create mode 100644 cmd/pinniped/cmd/flag_types.go create mode 100644 cmd/pinniped/cmd/flag_types_test.go diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go new file mode 100644 index 00000000..92019b12 --- /dev/null +++ b/cmd/pinniped/cmd/flag_types.go @@ -0,0 +1,46 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "flag" + "fmt" + "strings" +) + +// conciergeMode represents the method by which we should connect to the Concierge on a cluster during login. +// this is meant to be a valid flag.Value implementation. +type conciergeMode int + +var _ flag.Value = new(conciergeMode) + +const ( + modeTokenCredentialRequestAPI conciergeMode = iota + modeImpersonationProxy conciergeMode = iota +) + +func (c *conciergeMode) String() string { + switch *c { + case modeImpersonationProxy: + return "ImpersonationProxy" + default: + return "TokenCredentialRequestAPI" + } +} + +func (c *conciergeMode) Set(s string) error { + if strings.EqualFold(s, "TokenCredentialRequestAPI") { + *c = modeTokenCredentialRequestAPI + return nil + } + if strings.EqualFold(s, "ImpersonationProxy") { + *c = modeImpersonationProxy + return nil + } + return fmt.Errorf("invalid mode %q, valid modes are TokenCredentialRequestAPI and ImpersonationProxy", s) +} + +func (c *conciergeMode) Type() string { + return "mode" +} diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go new file mode 100644 index 00000000..101fae02 --- /dev/null +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConciergeModeFlag(t *testing.T) { + var m conciergeMode + require.Equal(t, "mode", m.Type()) + require.Equal(t, modeTokenCredentialRequestAPI, m) + require.EqualError(t, m.Set("foo"), `invalid mode "foo", valid modes are TokenCredentialRequestAPI and ImpersonationProxy`) + + require.NoError(t, m.Set("TokenCredentialRequestAPI")) + require.Equal(t, modeTokenCredentialRequestAPI, m) + require.Equal(t, "TokenCredentialRequestAPI", m.String()) + + require.NoError(t, m.Set("tokencredentialrequestapi")) + require.Equal(t, modeTokenCredentialRequestAPI, m) + require.Equal(t, "TokenCredentialRequestAPI", m.String()) + + require.NoError(t, m.Set("ImpersonationProxy")) + require.Equal(t, modeImpersonationProxy, m) + require.Equal(t, "ImpersonationProxy", m.String()) + + require.NoError(t, m.Set("impersonationproxy")) + require.Equal(t, modeImpersonationProxy, m) + require.Equal(t, "ImpersonationProxy", m.String()) +} diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 532afa05..ad1033c7 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -71,7 +71,7 @@ type oidcLoginFlags struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string - useImpersonationProxy bool + conciergeMode conciergeMode } func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { @@ -92,17 +92,17 @@ func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { cmd.Flags().BoolVar(&flags.skipBrowser, "skip-browser", false, "Skip opening the browser (just print the URL)") cmd.Flags().StringVar(&flags.sessionCachePath, "session-cache", filepath.Join(mustGetConfigDir(), "sessions.yaml"), "Path to session cache file") cmd.Flags().StringSliceVar(&flags.caBundlePaths, "ca-bundle", nil, "Path to TLS certificate authority bundle (PEM format, optional, can be repeated)") - cmd.Flags().StringSliceVar(&flags.caBundleData, "ca-bundle-data", nil, "Base64 endcoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated)") + cmd.Flags().StringSliceVar(&flags.caBundleData, "ca-bundle-data", nil, "Base64 encoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated)") cmd.Flags().BoolVar(&flags.debugSessionCache, "debug-session-cache", false, "Print debug logs related to the session cache") cmd.Flags().StringVar(&flags.requestAudience, "request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange") - cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Exchange the OIDC ID token with the Pinniped concierge during login") - cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the concierge was installed") + cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Use the Concierge to login") + cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") cmd.Flags().StringVar(&flags.conciergeAuthenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt')") cmd.Flags().StringVar(&flags.conciergeAuthenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name") - cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") - cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") + cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") + cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the Concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") - cmd.Flags().BoolVar(&flags.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") + cmd.Flags().Var(&flags.conciergeMode, "concierge-mode", "Concierge mode of operation") mustMarkHidden(cmd, "debug-session-cache") mustMarkRequired(cmd, "issuer") @@ -179,18 +179,27 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin } cred := tokenCredential(token) - // If the concierge was configured, exchange the credential for a separate short-lived, cluster-specific credential. + // If there is no concierge configuration, return the credential directly. + if concierge == nil { + return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) + } + + // If the concierge was configured, we need to do extra steps to make the credential usable. ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - // do a credential exchange request, unless impersonation proxy is configured - if concierge != nil && !flags.useImpersonationProxy { - cred, err = deps.exchangeToken(ctx, concierge, token.IDToken.Token) + // The exact behavior depends on in which mode the Concierge is operating. + switch flags.conciergeMode { + + case modeTokenCredentialRequestAPI: + // do a credential exchange request + cred, err := deps.exchangeToken(ctx, concierge, token.IDToken.Token) if err != nil { return fmt.Errorf("could not complete concierge credential exchange: %w", err) } - } - if concierge != nil && flags.useImpersonationProxy { + return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) + + case modeImpersonationProxy: // Put the token into a TokenCredentialRequest // put the TokenCredentialRequest in an ExecCredential req, err := execCredentialForImpersonationProxy(token.IDToken.Token, flags.conciergeAuthenticatorType, flags.conciergeAuthenticatorName, &token.IDToken.Expiry) @@ -198,9 +207,12 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin return err } return json.NewEncoder(cmd.OutOrStdout()).Encode(req) + + default: + return fmt.Errorf("unsupported Concierge mode %q", flags.conciergeMode.String()) } - return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) } + func makeClient(caBundlePaths []string, caBundleData []string) (*http.Client, error) { pool := x509.NewCertPool() for _, p := range caBundlePaths { diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index 198b7519..c6a4d78e 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -15,15 +15,13 @@ import ( "testing" "time" - corev1 "k8s.io/api/core/v1" - - authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" - "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientauthv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" + authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/testutil" @@ -64,15 +62,15 @@ func TestLoginOIDCCommand(t *testing.T) { Flags: --ca-bundle strings Path to TLS certificate authority bundle (PEM format, optional, can be repeated) - --ca-bundle-data strings Base64 endcoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated) + --ca-bundle-data strings Base64 encoded TLS certificate authority bundle (base64 encoded PEM format, optional, can be repeated) --client-id string OpenID Connect client ID (default "pinniped-cli") --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') - --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge - --concierge-endpoint string API base for the Pinniped concierge endpoint - --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy - --enable-concierge Exchange the OIDC ID token with the Pinniped concierge during login + --concierge-ca-bundle-data string CA bundle to use when connecting to the Concierge + --concierge-endpoint string API base for the Concierge endpoint + --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) + --enable-concierge Use the Concierge to login -h, --help help for oidc --issuer string OpenID Connect issuer URL --listen-port uint16 TCP port for localhost listener (authorization code flow only) @@ -207,7 +205,7 @@ func TestLoginOIDCCommand(t *testing.T) { "--client-id", "test-client-id", "--issuer", "test-issuer", "--enable-concierge", - "--concierge-use-impersonation-proxy", + "--concierge-mode", "ImpersonationProxy", "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", "--concierge-endpoint", "https://127.0.0.1:1234/", diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index c1e29869..94f9b6a1 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -47,7 +47,7 @@ type staticLoginParams struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string - useImpersonationProxy bool + conciergeMode conciergeMode } func staticLoginCommand(deps staticLoginDeps) *cobra.Command { @@ -63,14 +63,14 @@ func staticLoginCommand(deps staticLoginDeps) *cobra.Command { ) cmd.Flags().StringVar(&flags.staticToken, "token", "", "Static token to present during login") cmd.Flags().StringVar(&flags.staticTokenEnvName, "token-env", "", "Environment variable containing a static token") - cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Exchange the token with the Pinniped concierge during login") - cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the concierge was installed") + cmd.Flags().BoolVar(&flags.conciergeEnabled, "enable-concierge", false, "Use the Concierge to login") + cmd.Flags().StringVar(&conciergeNamespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") cmd.Flags().StringVar(&flags.conciergeAuthenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt')") cmd.Flags().StringVar(&flags.conciergeAuthenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name") - cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") - cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the concierge") + cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") + cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the Concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") - cmd.Flags().BoolVar(&flags.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") + cmd.Flags().Var(&flags.conciergeMode, "concierge-mode", "Concierge mode of operation") cmd.RunE = func(cmd *cobra.Command, args []string) error { return runStaticLogin(cmd.OutOrStdout(), deps, flags) } @@ -115,18 +115,26 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams } cred := tokenCredential(&oidctypes.Token{IDToken: &oidctypes.IDToken{Token: token}}) - // Exchange that token with the concierge, if configured. - if concierge != nil && !flags.useImpersonationProxy { + // If there is no concierge configuration, return the credential directly. + if concierge == nil { + return json.NewEncoder(out).Encode(cred) + } + + // If the concierge is enabled, we need to do extra steps. + switch flags.conciergeMode { + + case modeTokenCredentialRequestAPI: + // do a credential exchange request ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - var err error - cred, err = deps.exchangeToken(ctx, concierge, token) + cred, err := deps.exchangeToken(ctx, concierge, token) if err != nil { return fmt.Errorf("could not complete concierge credential exchange: %w", err) } - } - if concierge != nil && flags.useImpersonationProxy { + return json.NewEncoder(out).Encode(cred) + + case modeImpersonationProxy: // Put the token into a TokenCredentialRequest // put the TokenCredentialRequest in an ExecCredential req, err := execCredentialForImpersonationProxy(token, flags.conciergeAuthenticatorType, flags.conciergeAuthenticatorName, nil) @@ -134,6 +142,9 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams return err } return json.NewEncoder(out).Encode(req) + + default: + return fmt.Errorf("unsupported Concierge mode %q", flags.conciergeMode.String()) } - return json.NewEncoder(out).Encode(cred) + } diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index de71cae2..993313d5 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -54,10 +54,10 @@ func TestLoginStaticCommand(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') - --concierge-ca-bundle-data string CA bundle to use when connecting to the concierge - --concierge-endpoint string API base for the Pinniped concierge endpoint - --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy - --enable-concierge Exchange the token with the Pinniped concierge during login + --concierge-ca-bundle-data string CA bundle to use when connecting to the Concierge + --concierge-endpoint string API base for the Concierge endpoint + --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) + --enable-concierge Use the Concierge to login -h, --help help for static --token string Static token to present during login --token-env string Environment variable containing a static token @@ -156,7 +156,7 @@ func TestLoginStaticCommand(t *testing.T) { name: "impersonation proxy success", args: []string{ "--enable-concierge", - "--concierge-use-impersonation-proxy", + "--concierge-mode", "ImpersonationProxy", "--token", "test-token", "--concierge-endpoint", "https://127.0.0.1/", "--concierge-authenticator-type", "webhook", From aee7a7a72b9c575d0d263800a5b4d6f865b4dc77 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 24 Feb 2021 16:03:17 -0800 Subject: [PATCH 039/203] More WIP managing TLS secrets from the impersonation config controller Signed-off-by: Margo Crawford --- internal/concierge/impersonator/config.go | 10 +- .../impersonatorconfig/impersonator_config.go | 230 +++++++++++++----- .../impersonator_config_test.go | 164 ++++++++++--- 3 files changed, 316 insertions(+), 88 deletions(-) diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go index 7ced9b66..20d97647 100644 --- a/internal/concierge/impersonator/config.go +++ b/internal/concierge/impersonator/config.go @@ -45,9 +45,13 @@ type Config struct { // Enable or disable the impersonation proxy. Optional. Defaults to ModeAuto. Mode Mode `json:"mode,omitempty"` - // The HTTPS URL of the impersonation proxy for clients to use from outside the cluster. Used when creating TLS - // certificates and for clients to discover the endpoint. Optional. When not specified, if the impersonation proxy - // is started, then it will automatically create a LoadBalancer Service and use its ingress as the endpoint. + // Used when creating TLS certificates and for clients to discover the endpoint. Optional. When not specified, if the + // impersonation proxy is started, then it will automatically create a LoadBalancer Service and use its ingress as the + // endpoint. + // + // When specified, it may be a hostname or IP address, optionally with a port number, of the impersonation proxy + // for clients to use from outside the cluster. E.g. myhost.mycompany.com:8443. Clients should assume that they should + // connect via HTTPS to this service. Endpoint string `json:"endpoint,omitempty"` // The TLS configuration of the impersonation proxy's endpoints. Optional. When not specified, a CA and TLS diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 508d17a4..98dfd046 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -6,7 +6,9 @@ package impersonatorconfig import ( "context" "crypto/tls" + "crypto/x509" "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "net" @@ -170,11 +172,13 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { } if c.shouldHaveTLSSecret(config) { - if err = c.ensureTLSSecretIsCreated(ctx.Context, config); err != nil { - // return err // TODO + err = c.ensureTLSSecret(ctx, config) + if err != nil { + return err } } else { - if err = c.ensureTLSSecretIsRemoved(ctx.Context); err != nil { + err = c.ensureTLSSecretIsRemoved(ctx.Context) + if err != nil { return err } } @@ -184,8 +188,24 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { return nil } -func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { - return c.shouldHaveImpersonator(config) +func (c *impersonatorConfigController) ensureTLSSecret(ctx controllerlib.Context, config *impersonator.Config) error { + secret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) + notFound := k8serrors.IsNotFound(err) + if notFound { + secret = nil + } + if !notFound && err != nil { + return err + } + //nolint:staticcheck // TODO remove this nolint when we fix the TODO below + if secret, err = c.deleteTLSCertificateWithWrongName(ctx.Context, config, secret); err != nil { + // TODO + // return err + } + if err = c.ensureTLSSecretIsCreatedAndLoaded(ctx.Context, config, secret); err != nil { + return err + } + return nil } func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { @@ -196,6 +216,10 @@ func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonat return c.shouldHaveImpersonator(config) && config.Endpoint == "" } +func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { + return c.shouldHaveImpersonator(config) // TODO is this the logic that we want here? +} + func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { if c.server != nil { plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) @@ -266,26 +290,6 @@ func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, erro return true, secret, nil } -func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { - running, err := c.isLoadBalancerRunning() - if err != nil { - return err - } - if !running { - return nil - } - - plog.Info("Deleting load balancer for impersonation proxy", - "service", c.generatedLoadBalancerServiceName, - "namespace", c.namespace) - err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) - if err != nil { - return err - } - - return nil -} - func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.Context) error { running, err := c.isLoadBalancerRunning() if err != nil { @@ -323,56 +327,147 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C return nil } -func (c *impersonatorConfigController) ensureTLSSecretIsCreated(ctx context.Context, config *impersonator.Config) error { - tlsSecretExists, tlsSecret, err := c.tlsSecretExists() +func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { + running, err := c.isLoadBalancerRunning() if err != nil { return err } - if tlsSecretExists { - certPEM := tlsSecret.Data[v1.TLSCertKey] - keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] - tlsCert, _ := tls.X509KeyPair(certPEM, keyPEM) - // TODO handle err, like when the Secret did not contain the fields that we expected - c.setTLSCert(&tlsCert) + if !running { return nil } - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) // TODO change the length of this + + plog.Info("Deleting load balancer for impersonation proxy", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) if err != nil { - return fmt.Errorf("could not create impersonation CA: %w", err) + return err } - var ips []net.IP - if config.Endpoint == "" { // TODO are there other cases where we need to do this? - lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) - notFound := k8serrors.IsNotFound(err) - if notFound { - return nil + + return nil +} + +func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx context.Context, config *impersonator.Config, secret *v1.Secret) (*v1.Secret, error) { + if secret == nil { + // There is no Secret, so there is nothing to delete. + return secret, nil + } + + certPEM := secret.Data[v1.TLSCertKey] + block, _ := pem.Decode(certPEM) + if block == nil { + // The certPEM is not valid. + return secret, nil // TODO what should we do? + } + parsed, _ := x509.ParseCertificate(block.Bytes) + // TODO handle err + + desiredIPs, nameIsReady, err := c.findTLSCertificateName(config) + //nolint:staticcheck // TODO remove this nolint when we fix the TODO below + if err != nil { + // TODO return err + } + if !nameIsReady { + // We currently have a secret but we are waiting for a load balancer to be assigned an ingress, so + // our current secret must be old/unwanted. + err = c.ensureTLSSecretIsRemoved(ctx) + if err != nil { + return secret, err } + return nil, nil + } + + actualIPs := parsed.IPAddresses + // TODO handle multiple IPs, and handle when there is no IP + if desiredIPs[0].Equal(actualIPs[0]) { + // The cert matches the desired state, so we do not need to delete it. + return secret, nil + } + + err = c.ensureTLSSecretIsRemoved(ctx) + if err != nil { + return secret, err + } + return nil, nil +} + +func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret) error { + if secret != nil { + err := c.loadTLSCertFromSecret(secret) if err != nil { return err } + return nil + } + + // TODO create/save/watch the CA separately so we can reuse it to mint tls certs as the settings are dynamically changed, + // so that clients don't need to be updated to use a different CA just because the server-side settings were changed. + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) // TODO change the expiration of this to 100 years + if err != nil { + return fmt.Errorf("could not create impersonation CA: %w", err) + } + + ips, nameIsReady, err := c.findTLSCertificateName(config) + if err != nil { + return err + } + if !nameIsReady { + // Sync will get called again when the load balancer is updated with its ingress info, so this is not an error. + return nil + } + + newTLSSecret, err := c.createNewTLSSecret(ctx, impersonationCA, ips) + if err != nil { + return err + } + + err = c.loadTLSCertFromSecret(newTLSSecret) + if err != nil { + return err + } + + return nil +} + +func (c *impersonatorConfigController) findTLSCertificateName(config *impersonator.Config) ([]net.IP, bool, error) { + var ips []net.IP + if config.Endpoint != "" { + // TODO Endpoint could be a hostname + // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose + ips = []net.IP{net.ParseIP(config.Endpoint)} + } else { + lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + // TODO is this an error? we should have already created the load balancer, so why would it not exist here? + return nil, false, nil + } + if err != nil { + return nil, false, err + } ingresses := lb.Status.LoadBalancer.Ingress if len(ingresses) == 0 { - return nil + plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + return nil, false, nil } - ip := ingresses[0].IP // TODO multiple ips + ip := ingresses[0].IP // TODO handle multiple ips? ips = []net.IP{net.ParseIP(ip)} - // check with informer to get the ip address of the load balancer if its available - // if not, return - } else { - ips = []net.IP{net.ParseIP(config.Endpoint)} } - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, ips, 24*time.Hour) // TODO change the length of this + return ips, true, nil +} + +func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP) (*v1.Secret, error) { + impersonationCert, err := ca.Issue(pkix.Name{}, nil, ips, 24*time.Hour) // TODO change the length of this too 100 years for now? if err != nil { - return fmt.Errorf("could not create impersonation cert: %w", err) + return nil, fmt.Errorf("could not create impersonation cert: %w", err) } - c.setTLSCert(impersonationCert) + certPEM, keyPEM, _ := certauthority.ToPEM(impersonationCert) + // TODO handle err - certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) - // TODO error handling? - - // TODO handle error on create - c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, &v1.Secret{ + newTLSSecret := &v1.Secret{ Type: v1.SecretTypeTLS, ObjectMeta: metav1.ObjectMeta{ Name: c.tlsSecretName, @@ -380,11 +475,30 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreated(ctx context.Cont Labels: c.labels, }, Data: map[string][]byte{ - "ca.crt": impersonationCA.Bundle(), + "ca.crt": ca.Bundle(), v1.TLSPrivateKeyKey: keyPEM, v1.TLSCertKey: certPEM, }, - }, metav1.CreateOptions{}) + } + _, _ = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) + // TODO handle error on create + return newTLSSecret, nil +} + +func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secret) error { + certPEM := tlsSecret.Data[v1.TLSCertKey] + keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + plog.Error("Could not parse TLS cert PEM data from Secret", + err, + "secret", c.tlsSecretName, + "namespace", c.namespace, + ) + // TODO clear the secret if it was already set previously... c.setTLSCert(nil) + return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) + } + c.setTLSCert(&tlsCert) return nil } @@ -404,6 +518,8 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return err } + c.setTLSCert(nil) + return nil } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index b6fee18e..c63a9c54 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -289,7 +289,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return nil, startTLSListenerFuncError } var err error - //nolint: gosec // Intentionally binding to all network interfaces. startedTLSListener, err = tls.Listen(network, "127.0.0.1:0", config) // automatically choose the port for unit tests r.NoError(err) return &tlsListenerWrapper{listener: startedTLSListener, closeError: startTLSListenerUponCloseError}, nil @@ -306,13 +305,26 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var requireTLSServerIsRunning = func(caCrt []byte) { + var requireTLSServerIsRunning = func(caCrt []byte, addr string, dnsOverrides map[string]string) { r.Greater(startTLSListenerFuncWasCalled, 0) + realDialer := &net.Dialer{} + overrideDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { + replacementAddr, hasKey := dnsOverrides[addr] + if hasKey { + t.Logf("DialContext replacing addr %s with %s", addr, replacementAddr) + addr = replacementAddr + } else if dnsOverrides != nil { + t.Fatal("dnsOverrides was provided but not used, which was probably a mistake") + } + return realDialer.DialContext(ctx, network, addr) + } + var tr *http.Transport if caCrt == nil { tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + DialContext: overrideDialContext, } } else { rootCAs := x509.NewCertPool() @@ -320,10 +332,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { tr = &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: rootCAs}, + DialContext: overrideDialContext, } } client := &http.Client{Transport: tr} - url := "https://" + startedTLSListener.Addr().String() + url := "https://" + addr req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) resp, err := client.Do(req) @@ -347,7 +360,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { url := "https://" + startedTLSListener.Addr().String() req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) - _, err = client.Do(req) + _, err = client.Do(req) //nolint:bodyclose r.Error(err) r.Regexp("Get .*: remote error: tls: unrecognized name", err.Error()) } @@ -357,7 +370,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { _, err := tls.Dial( startedTLSListener.Addr().Network(), startedTLSListener.Addr().String(), - &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // TODO once we're using certs, do not skip verify + &tls.Config{InsecureSkipVerify: true}, //nolint:gosec ) r.Error(err) r.Regexp(`dial tcp .*: connect: connection refused`, err.Error()) @@ -380,6 +393,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }, 10*time.Second, time.Millisecond) } + var waitForTLSCertSecretToBeDeleted = func(informer corev1informers.SecretInformer, name string) { + r.Eventually(func() bool { + _, err := informer.Lister().Secrets(installedInNamespace).Get(name) + return k8serrors.IsNotFound(err) + }, 10*time.Second, time.Millisecond) + } + // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. var startInformersAndController = func() { @@ -477,6 +497,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, []net.IP{net.ParseIP("127.0.0.1")}, 24*time.Hour) r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + r.NoError(err) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -512,6 +533,31 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(client.Tracker().Add(loadBalancerService)) } + var updateLoadBalancerServiceInTracker = func(resourceName, lbIngressIP, newResourceVersion string) { + loadBalancerService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + ResourceVersion: newResourceVersion, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: lbIngressIP}, + }, + }, + }, + } + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + loadBalancerService, + installedInNamespace, + )) + } + var addLoadBalancerServiceWithIPToTracker = func(resourceName string, client *kubernetesfake.Clientset) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -544,6 +590,20 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } + var deleteTLSCertSecretFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Delete( + schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, + installedInNamespace, + resourceName, + )) + } + + var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + createdSecret.ResourceVersion = resourceVersion + r.NoError(client.Tracker().Add(createdSecret)) + } + var addNodeWithRoleToTracker = func(role string) { r.NoError(kubeAPIClient.Tracker().Add( &corev1.Node{ @@ -740,18 +800,16 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time r.Len(kubeAPIClient.Actions(), 3) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca) // running with certs now + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // running with certs now // update manually because the kubeAPIClient isn't connected to the informer in the tests - createdSecret := kubeAPIClient.Actions()[2].(coretesting.CreateAction).GetObject().(*corev1.Secret) - createdSecret.ResourceVersion = "1" - r.NoError(kubeInformerClient.Tracker().Add(createdSecret)) + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time - r.Len(kubeAPIClient.Actions(), 3) // no more actions - requireTLSServerIsRunning(ca) // still running + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 3) // no more actions + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // still running }) }) @@ -794,8 +852,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: auto endpoint: 127.0.0.1 - `), // TODO what to do about ports - // TODO IP address and hostname should work + `), ) }) @@ -824,7 +881,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunning(ca) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) }) }) }) @@ -900,12 +957,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("a load balancer and a secret already exist", func() { - var tlsSecret *corev1.Secret + when("a load balancer and a secret already exists", func() { + var ca []byte it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") addNodeWithRoleToTracker("worker") - tlsSecret = createActualTLSSecret(tlsSecretName) + tlsSecret := createActualTLSSecret(tlsSecretName) + ca = tlsSecret.Data["ca.crt"] r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) @@ -917,7 +975,29 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunning(tlsSecret.Data["ca.crt"]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + }) + + when("a load balancer and a secret already exists but the tls secret is not valid", func() { + var tlsSecret *corev1.Secret + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + tlsSecret = createStubTLSSecret(tlsSecretName) // secret exists but lacks certs + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("returns an error and leaves the impersonator running without tls certs", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), + "could not parse TLS cert PEM data from Secret: tls: failed to find any PEM data in certificate input") + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSServerIsRunningWithoutCerts() }) }) }) @@ -984,39 +1064,67 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: enabled - endpoint: https://proxy.example.com:8443/ + endpoint: 127.0.0.1 `)) addNodeWithRoleToTracker("worker") }) - it("doesn't start, then creates, then deletes the load balancer", func() { + it("doesn't create, then creates, then deletes the load balancer", func() { startInformersAndController() + // Should have started in "enabled" mode with an "endpoint", so no load balancer is needed. r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - r.Len(kubeAPIClient.Actions(), 1) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + // Switch to "enabled" mode without an "endpoint", so a load balancer is needed now. updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 2) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + r.Len(kubeAPIClient.Actions(), 4) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[2]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[3]) // the Secret was deleted because it contained a cert with the wrong IP + requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) + waitForTLSCertSecretToBeDeleted(kubeInformers.Core().V1().Secrets(), tlsSecretName) + // The controller should be waiting for the load balancer's ingress to become available. + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 4) // no new actions while it is waiting for the load balancer's ingress + requireTLSServerIsRunningWithoutCerts() + + // Update the ingress of the LB in the informer's client and run Sync again. + fakeIP := "127.0.0.123" + updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, fakeIP, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 5) + ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[4]) // created because the LB ingress became available + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + + // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. updateImpersonatorConfigMapInTracker(configMapResourceName, here.Doc(` mode: enabled - endpoint: https://proxy.example.com:8443/ + endpoint: 127.0.0.1 `), "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 3) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) + r.Len(kubeAPIClient.Actions(), 7) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[5]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[6]) // recreated because the endpoint was updated }) }) }) From 975d493b8a7fdc6713ccc1852d73c72357624c8d Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 24 Feb 2021 16:09:15 -0800 Subject: [PATCH 040/203] Fix some small lint errors Signed-off-by: Ryan Richard --- cmd/pinniped/cmd/flag_types.go | 2 ++ cmd/pinniped/cmd/flag_types_test.go | 3 +++ cmd/pinniped/cmd/login_oidc.go | 1 - cmd/pinniped/cmd/login_static.go | 2 -- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index 92019b12..262bd9b8 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -24,6 +24,8 @@ func (c *conciergeMode) String() string { switch *c { case modeImpersonationProxy: return "ImpersonationProxy" + case modeTokenCredentialRequestAPI: + return "TokenCredentialRequestAPI" default: return "TokenCredentialRequestAPI" } diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go index 101fae02..955ae624 100644 --- a/cmd/pinniped/cmd/flag_types_test.go +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -1,3 +1,6 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + package cmd import ( diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index ad1033c7..2eddee79 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -190,7 +190,6 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin // The exact behavior depends on in which mode the Concierge is operating. switch flags.conciergeMode { - case modeTokenCredentialRequestAPI: // do a credential exchange request cred, err := deps.exchangeToken(ctx, concierge, token.IDToken.Token) diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 94f9b6a1..fd1526c2 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -122,7 +122,6 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams // If the concierge is enabled, we need to do extra steps. switch flags.conciergeMode { - case modeTokenCredentialRequestAPI: // do a credential exchange request ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) @@ -146,5 +145,4 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams default: return fmt.Errorf("unsupported Concierge mode %q", flags.conciergeMode.String()) } - } From 8fc68a4b21f8be33135023db02863942d19079fa Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 24 Feb 2021 17:08:58 -0800 Subject: [PATCH 041/203] WIP improved cert management in impersonator config - Allows Endpoint to be a hostname, not just an IP address Signed-off-by: Ryan Richard --- .../impersonatorconfig/impersonator_config.go | 31 ++++++++++++------- .../impersonator_config_test.go | 21 +++++++++++++ 2 files changed, 40 insertions(+), 12 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 98dfd046..5497d805 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -362,7 +362,7 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con parsed, _ := x509.ParseCertificate(block.Bytes) // TODO handle err - desiredIPs, nameIsReady, err := c.findTLSCertificateName(config) + desiredIPs, _, nameIsReady, err := c.findTLSCertificateName(config) // TODO check this for hostnames too, not just ips //nolint:staticcheck // TODO remove this nolint when we fix the TODO below if err != nil { // TODO return err @@ -407,7 +407,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return fmt.Errorf("could not create impersonation CA: %w", err) } - ips, nameIsReady, err := c.findTLSCertificateName(config) + ips, hostnames, nameIsReady, err := c.findTLSCertificateName(config) if err != nil { return err } @@ -416,7 +416,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } - newTLSSecret, err := c.createNewTLSSecret(ctx, impersonationCA, ips) + newTLSSecret, err := c.createNewTLSSecret(ctx, impersonationCA, ips, hostnames) if err != nil { return err } @@ -429,37 +429,44 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } -func (c *impersonatorConfigController) findTLSCertificateName(config *impersonator.Config) ([]net.IP, bool, error) { +func (c *impersonatorConfigController) findTLSCertificateName(config *impersonator.Config) ([]net.IP, []string, bool, error) { var ips []net.IP + var hostnames []string if config.Endpoint != "" { + parsedAsIP := net.ParseIP(config.Endpoint) + if parsedAsIP != nil { + ips = []net.IP{parsedAsIP} + } else { + hostnames = []string{config.Endpoint} + } // TODO Endpoint could be a hostname // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose - ips = []net.IP{net.ParseIP(config.Endpoint)} } else { lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) notFound := k8serrors.IsNotFound(err) if notFound { // TODO is this an error? we should have already created the load balancer, so why would it not exist here? - return nil, false, nil + return nil, nil, false, nil } if err != nil { - return nil, false, err + return nil, nil, false, err } ingresses := lb.Status.LoadBalancer.Ingress if len(ingresses) == 0 { plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", "service", c.generatedLoadBalancerServiceName, "namespace", c.namespace) - return nil, false, nil + return nil, nil, false, nil } - ip := ingresses[0].IP // TODO handle multiple ips? + // TODO get all IPs and all hostnames from ingresses and put them in the cert + ip := ingresses[0].IP ips = []net.IP{net.ParseIP(ip)} } - return ips, true, nil + return ips, hostnames, true, nil } -func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP) (*v1.Secret, error) { - impersonationCert, err := ca.Issue(pkix.Name{}, nil, ips, 24*time.Hour) // TODO change the length of this too 100 years for now? +func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP, hostnames []string) (*v1.Secret, error) { + impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, 24*time.Hour) // TODO change the length of this too 100 years for now? if err != nil { return nil, fmt.Errorf("could not create impersonation cert: %w", err) } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index c63a9c54..c61026da 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -1000,6 +1000,27 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() }) }) + + when("we have a hostname specified for the endpoint", func() { + const fakeHostname = "fake.example.com" + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, here.Docf(` + mode: enabled + endpoint: %s + `, fakeHostname)) + addNodeWithRoleToTracker("worker") + }) + + it("starts the impersonator, generates a valid cert for the hostname", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + ":443": startedTLSListener.Addr().String()}) + }) + }) }) when("the configuration switches from enabled to disabled mode", func() { From 9a8c80f20afd74beefcdddd8cc617102fe5cf017 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Thu, 25 Feb 2021 10:27:19 -0800 Subject: [PATCH 042/203] Impersonator checks cert addresses when `endpoint` config is a hostname Also update concierge_impersonation_proxy_test.go integration test to use real TLS when calling the impersonator. Signed-off-by: Ryan Richard --- .../impersonatorconfig/impersonator_config.go | 98 ++++++++++++------- .../impersonator_config_test.go | 54 +++++++++- .../concierge_impersonation_proxy_test.go | 64 ++++++++---- 3 files changed, 161 insertions(+), 55 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 5497d805..3626a6c9 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -359,10 +359,10 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con // The certPEM is not valid. return secret, nil // TODO what should we do? } - parsed, _ := x509.ParseCertificate(block.Bytes) + actualCertFromSecret, _ := x509.ParseCertificate(block.Bytes) // TODO handle err - desiredIPs, _, nameIsReady, err := c.findTLSCertificateName(config) // TODO check this for hostnames too, not just ips + desiredIPs, desiredHostnames, nameIsReady, err := c.findTLSCertificateName(config) // TODO check this for hostnames too, not just ips //nolint:staticcheck // TODO remove this nolint when we fix the TODO below if err != nil { // TODO return err @@ -377,12 +377,24 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con return nil, nil } - actualIPs := parsed.IPAddresses - // TODO handle multiple IPs, and handle when there is no IP - if desiredIPs[0].Equal(actualIPs[0]) { + actualIPs := actualCertFromSecret.IPAddresses + actualHostnames := actualCertFromSecret.DNSNames + plog.Info("Checking TLS certificate names", + "desiredIPs", desiredIPs, + "desiredHostnames", desiredHostnames, + "actualIPs", actualIPs, + "actualHostnames", actualHostnames, + "secret", c.tlsSecretName, + "namespace", c.namespace) + + // TODO handle multiple IPs + if len(desiredIPs) == len(actualIPs) && len(desiredIPs) == 1 && desiredIPs[0].Equal(actualIPs[0]) { //nolint:gocritic // The cert matches the desired state, so we do not need to delete it. return secret, nil } + if len(desiredHostnames) == len(actualHostnames) && len(desiredHostnames) == 1 && desiredHostnames[0] == actualHostnames[0] { //nolint:gocritic + return secret, nil + } err = c.ensureTLSSecretIsRemoved(ctx) if err != nil { @@ -430,38 +442,47 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con } func (c *impersonatorConfigController) findTLSCertificateName(config *impersonator.Config) ([]net.IP, []string, bool, error) { + if config.Endpoint != "" { + return c.findTLSCertificateNameFromEndpointConfig(config) + } + return c.findTLSCertificateNameFromLoadBalancer() +} + +func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) ([]net.IP, []string, bool, error) { var ips []net.IP var hostnames []string - if config.Endpoint != "" { - parsedAsIP := net.ParseIP(config.Endpoint) - if parsedAsIP != nil { - ips = []net.IP{parsedAsIP} - } else { - hostnames = []string{config.Endpoint} - } - // TODO Endpoint could be a hostname - // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose + parsedAsIP := net.ParseIP(config.Endpoint) + if parsedAsIP != nil { + ips = []net.IP{parsedAsIP} } else { - lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) - notFound := k8serrors.IsNotFound(err) - if notFound { - // TODO is this an error? we should have already created the load balancer, so why would it not exist here? - return nil, nil, false, nil - } - if err != nil { - return nil, nil, false, err - } - ingresses := lb.Status.LoadBalancer.Ingress - if len(ingresses) == 0 { - plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", - "service", c.generatedLoadBalancerServiceName, - "namespace", c.namespace) - return nil, nil, false, nil - } - // TODO get all IPs and all hostnames from ingresses and put them in the cert - ip := ingresses[0].IP - ips = []net.IP{net.ParseIP(ip)} + hostnames = []string{config.Endpoint} } + // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose + return ips, hostnames, true, nil +} + +func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() ([]net.IP, []string, bool, error) { + var ips []net.IP + var hostnames []string + lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + // TODO is this an error? we should have already created the load balancer, so why would it not exist here? + return nil, nil, false, nil + } + if err != nil { + return nil, nil, false, err + } + ingresses := lb.Status.LoadBalancer.Ingress + if len(ingresses) == 0 { + plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + return nil, nil, false, nil + } + // TODO get all IPs and all hostnames from ingresses and put them in the cert + ip := ingresses[0].IP + ips = []net.IP{net.ParseIP(ip)} return ips, hostnames, true, nil } @@ -487,6 +508,11 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c v1.TLSCertKey: certPEM, }, } + plog.Info("Creating TLS certificates for impersonation proxy", + "ips", ips, + "hostnames", hostnames, + "secret", c.tlsSecretName, + "namespace", c.namespace) _, _ = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) // TODO handle error on create return newTLSSecret, nil @@ -505,6 +531,10 @@ func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secre // TODO clear the secret if it was already set previously... c.setTLSCert(nil) return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) } + plog.Info("Loading TLS certificates for impersonation proxy", + "certPEM", certPEM, + "secret", c.tlsSecretName, + "namespace", c.namespace) c.setTLSCert(&tlsCert) return nil } @@ -518,7 +548,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return nil } plog.Info("Deleting TLS certificates for impersonation proxy", - "service", c.tlsSecretName, + "secret", c.tlsSecretName, "namespace", c.namespace) err = c.k8sClient.CoreV1().Secrets(c.namespace).Delete(ctx, c.tlsSecretName, metav1.DeleteOptions{}) if err != nil { diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index c61026da..d8d31f1c 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -1017,10 +1017,62 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - // Check that the server is running and that TLS certs that are being served are are for fakeIP. + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + ":443": startedTLSListener.Addr().String()}) }) }) + + when("switching from ip address endpoint to hostname endpoint and back to ip address", func() { + const fakeHostname = "fake.example.com" + const fakeIP = "127.0.0.42" + var hostnameYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + var ipAddressYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIP) + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, ipAddressYAML) + addNodeWithRoleToTracker("worker") + }) + + it("regenerates the cert for the hostname, then regenerates it for the IP again", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + // Switch the endpoint config to a hostname. + updateImpersonatorConfigMapInTracker(configMapResourceName, hostnameYAML, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) + ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[3]) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + ":443": startedTLSListener.Addr().String()}) + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + + // Switch the endpoint config back to an IP. + updateImpersonatorConfigMapInTracker(configMapResourceName, ipAddressYAML, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 6) + requireTLSSecretDeleted(kubeAPIClient.Actions()[4]) + ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[5]) + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + }) + }) }) when("the configuration switches from enabled to disabled mode", func() { diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 1ec2fc08..b59fe91d 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -29,8 +29,11 @@ import ( "go.pinniped.dev/test/library" ) -// TODO don't hard code "pinniped-concierge-" in this string. It should be constructed from the env app name. -const impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" +const ( + // TODO don't hard code "pinniped-concierge-" in these strings. It should be constructed from the env app name. + impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" + impersonationProxyTLSSecretName = "pinniped-concierge-impersonation-proxy-tls-serving-certificate" //nolint:gosec // this is not a credential +) func TestImpersonationProxy(t *testing.T) { env := library.IntegrationEnv(t) @@ -49,23 +52,27 @@ func TestImpersonationProxy(t *testing.T) { authenticator := library.CreateTestWebhookAuthenticator(ctx, t) // The address of the ClusterIP service that points at the impersonation proxy's port - proxyServiceURL := fmt.Sprintf("https://%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) + proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) + proxyServiceURL := fmt.Sprintf("https://%s", proxyServiceEndpoint) t.Logf("making kubeconfig that points to %q", proxyServiceURL) - kubeconfig := &rest.Config{ - Host: proxyServiceURL, - TLSClientConfig: rest.TLSClientConfig{Insecure: true}, - BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), - Proxy: func(req *http.Request) (*url.URL, error) { - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil - }, - } + getImpersonationProxyClient := func(caData []byte) *kubernetes.Clientset { + kubeconfig := &rest.Config{ + Host: proxyServiceURL, + TLSClientConfig: rest.TLSClientConfig{Insecure: caData == nil, CAData: caData}, + BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), + Proxy: func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + }, + } - impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) - require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") + impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) + require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") + return impersonationProxyClient + } oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName, metav1.GetOptions{}) if oldConfigMap.Data != nil { @@ -74,6 +81,8 @@ func TestImpersonationProxy(t *testing.T) { } serviceUnavailableError := fmt.Sprintf(`Get "%s/api/v1/namespaces": Service Unavailable`, proxyServiceURL) + insecureImpersonationProxyClient := getImpersonationProxyClient(nil) + if env.HasCapability(library.HasExternalLoadBalancerProvider) { // Check that load balancer has been created require.Eventually(t, func() bool { @@ -86,13 +95,13 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet - _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = insecureImpersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer) configMap := configMapForConfig(t, impersonator.Config{ Mode: impersonator.ModeEnabled, - Endpoint: proxyServiceURL, + Endpoint: proxyServiceEndpoint, TLS: nil, }) t.Logf("creating configmap %s", configMap.Name) @@ -117,6 +126,16 @@ func TestImpersonationProxy(t *testing.T) { }) } + // Wait for ca data to be available at the secret location. + var caSecret *corev1.Secret + require.Eventually(t, + func() bool { + caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + return caSecret != nil && caSecret.Data["ca.crt"] != nil + }, 5*time.Minute, 250*time.Millisecond) + + // Create an impersonation proxy client with that ca data. + impersonationProxyClient := getImpersonationProxyClient(caSecret.Data["ca.crt"]) t.Run( "access as user", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyClient), @@ -296,9 +315,9 @@ func TestImpersonationProxy(t *testing.T) { require.Eventually(t, func() bool { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. - _, err = impersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = insecureImpersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return err.Error() == serviceUnavailableError - }, 10*time.Second, 500*time.Millisecond) + }, 20*time.Second, 500*time.Millisecond) if env.HasCapability(library.HasExternalLoadBalancerProvider) { // The load balancer should not exist after we disable the impersonation proxy. @@ -307,6 +326,11 @@ func TestImpersonationProxy(t *testing.T) { return !hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) }, time.Minute, 500*time.Millisecond) } + + require.Eventually(t, func() bool { + caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + return k8serrors.IsNotFound(err) + }, 10*time.Second, 250*time.Millisecond) } func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigMap { From 0cae72b391525301bed183cdca81d1bef1d3c598 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 25 Feb 2021 11:40:14 -0800 Subject: [PATCH 043/203] Get hostname from load balancer ingress to use for impersonator certs Signed-off-by: Margo Crawford --- .../impersonatorconfig/impersonator_config.go | 15 +++- .../impersonator_config_test.go | 84 +++++++++++++++++-- 2 files changed, 88 insertions(+), 11 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 3626a6c9..c428b26c 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -362,7 +362,7 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con actualCertFromSecret, _ := x509.ParseCertificate(block.Bytes) // TODO handle err - desiredIPs, desiredHostnames, nameIsReady, err := c.findTLSCertificateName(config) // TODO check this for hostnames too, not just ips + desiredIPs, desiredHostnames, nameIsReady, err := c.findTLSCertificateName(config) //nolint:staticcheck // TODO remove this nolint when we fix the TODO below if err != nil { // TODO return err @@ -474,15 +474,22 @@ func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() return nil, nil, false, err } ingresses := lb.Status.LoadBalancer.Ingress - if len(ingresses) == 0 { + if len(ingresses) == 0 || (ingresses[0].Hostname == "" && ingresses[0].IP == "") { plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", "service", c.generatedLoadBalancerServiceName, "namespace", c.namespace) return nil, nil, false, nil } - // TODO get all IPs and all hostnames from ingresses and put them in the cert + hostname := ingresses[0].Hostname + if hostname != "" { + return nil, []string{hostname}, true, nil + } ip := ingresses[0].IP - ips = []net.IP{net.ParseIP(ip)} + parsedIP := net.ParseIP(ip) + if parsedIP == nil { + return nil, nil, false, fmt.Errorf("could not parse IP address from load balancer %s: %s", lb.Name, ip) + } + ips = []net.IP{parsedIP} return ips, hostnames, true, nil } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index d8d31f1c..b5b26355 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -558,7 +558,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } - var addLoadBalancerServiceWithIPToTracker = func(resourceName string, client *kubernetesfake.Clientset) { + var addLoadBalancerServiceWithIngressToTracker = func(resourceName string, ip string, hostname string, client *kubernetesfake.Clientset) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -574,7 +574,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ Ingress: []corev1.LoadBalancerIngress{ - {IP: "127.0.0.1"}, + { + IP: ip, + Hostname: hostname, + }, }, }, }, @@ -759,6 +762,44 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) }) }) + + when("there are not visible control plane nodes and a load balancer already exists with empty ingress", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "", "", kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "", "", kubeAPIClient) + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + }) + + it("starts the impersonator without tls certs", func() { + requireTLSServerIsRunningWithoutCerts() + }) + + it("does not start the load balancer automatically", func() { + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with invalid ip", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "not-an-ip", "", kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "not-an-ip", "", kubeAPIClient) + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not parse IP address from load balancer some-service-resource-name: not-an-ip") + }) + + it("starts the impersonator without tls certs", func() { + requireTLSServerIsRunningWithoutCerts() + }) + + it("does not start the load balancer automatically", func() { + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + }) + }) }) when("sync is called more than once", func() { @@ -793,7 +834,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -811,6 +852,35 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 3) // no more actions requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // still running }) + + it("creates certs from the hostname listed on the load balancer", func() { + hostname := "fake.example.com" + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunningWithoutCerts() + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", hostname, kubeInformerClient) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + r.Len(kubeAPIClient.Actions(), 3) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + ":443": startedTLSListener.Addr().String()}) // running with certs now + + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 3) // no more actions + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + ":443": startedTLSListener.Addr().String()}) // still running + }) }) when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { @@ -966,8 +1036,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca = tlsSecret.Data["ca.crt"] r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeAPIClient) }) it("starts the impersonator with the existing tls certs, does not start loadbalancer or make tls secret", func() { @@ -987,8 +1057,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { tlsSecret = createStubTLSSecret(tlsSecretName) // secret exists but lacks certs r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeAPIClient) }) it("returns an error and leaves the impersonator running without tls certs", func() { From 1c7c22352f6bb7a9591d79fdfdb0377c0f5c1400 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 25 Feb 2021 11:31:11 -0600 Subject: [PATCH 044/203] Switch "get kubeconfig" flags to use `--concierge-mode` flag instead of boolean flag. This is the same as the previous change to the login commands. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 54 ++++++++++++++--------------- cmd/pinniped/cmd/kubeconfig_test.go | 23 ++++++++---- 2 files changed, 42 insertions(+), 35 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index d712021c..1d1e271d 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -73,13 +73,13 @@ type getKubeconfigOIDCParams struct { } type getKubeconfigConciergeParams struct { - disabled bool - authenticatorName string - authenticatorType string - apiGroupSuffix string - caBundlePath string - endpoint string - useImpersonationProxy bool + disabled bool + authenticatorName string + authenticatorType string + apiGroupSuffix string + caBundlePath string + endpoint string + mode conciergeMode } type getKubeconfigParams struct { @@ -107,15 +107,15 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.staticToken, "static-token", "", "Instead of doing an OIDC-based login, specify a static token") f.StringVar(&flags.staticTokenEnvName, "static-token-env", "", "Instead of doing an OIDC-based login, read a static token from the environment") - f.BoolVar(&flags.concierge.disabled, "no-concierge", false, "Generate a configuration which does not use the concierge, but sends the credential to the cluster directly") - f.StringVar(&namespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the concierge was installed") + f.BoolVar(&flags.concierge.disabled, "no-concierge", false, "Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly") + f.StringVar(&namespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") f.StringVar(&flags.concierge.authenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover)") f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") - f.StringVar(&flags.concierge.caBundlePath, "concierge-ca-bundle", "", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the concierge") - f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Pinniped concierge endpoint") - f.BoolVar(&flags.concierge.useImpersonationProxy, "concierge-use-impersonation-proxy", false, "Whether the concierge cluster uses an impersonation proxy") + f.StringVar(&flags.concierge.caBundlePath, "concierge-ca-bundle", "", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge") + f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") + f.Var(&flags.concierge.mode, "concierge-mode", "Concierge mode of operation") f.StringVar(&flags.oidc.issuer, "oidc-issuer", "", "OpenID Connect issuer URL (default: autodiscover)") f.StringVar(&flags.oidc.clientID, "oidc-client-id", "pinniped-cli", "OpenID Connect client ID (default: autodiscover)") @@ -172,17 +172,6 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if err != nil { return fmt.Errorf("could not load --kubeconfig/--kubeconfig-context: %w", err) } - if flags.concierge.useImpersonationProxy { - // TODO what to do if --use-impersonation-proxy is set but flags.concierge.caBundlePath is not??? - // TODO dont do this twice - conciergeCaBundleData, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) - if err != nil { - return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) - } - - cluster.CertificateAuthorityData = []byte(conciergeCaBundleData) - cluster.Server = flags.concierge.endpoint - } clientset, err := deps.getClientset(clientConfig, flags.concierge.apiGroupSuffix) if err != nil { return fmt.Errorf("could not configure Kubernetes client: %w", err) @@ -249,6 +238,19 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar } func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig) error { + + if flags.concierge.mode == modeImpersonationProxy { + // TODO what to do if --use-impersonation-proxy is set but flags.concierge.caBundlePath is not??? + // TODO dont do this twice + conciergeCaBundleData, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) + if err != nil { + return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + } + + v1Cluster.CertificateAuthorityData = []byte(conciergeCaBundleData) + v1Cluster.Server = flags.concierge.endpoint + } + switch auth := authenticator.(type) { case *conciergev1alpha1.WebhookAuthenticator: // If the --concierge-authenticator-type/--concierge-authenticator-name flags were not set explicitly, set @@ -309,12 +311,8 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, "--concierge-authenticator-type="+flags.concierge.authenticatorType, "--concierge-endpoint="+flags.concierge.endpoint, "--concierge-ca-bundle-data="+encodedConciergeCaBundleData, + "--concierge-mode="+flags.concierge.mode.String(), ) - if flags.concierge.useImpersonationProxy { - execConfig.Args = append(execConfig.Args, - "--concierge-use-impersonation-proxy", - ) - } return nil } diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index da8ba3a0..2a431c1e 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -64,13 +64,13 @@ func TestGetKubeconfig(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name (default: autodiscover) --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) - --concierge-ca-bundle string Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the concierge - --concierge-endpoint string API base for the Pinniped concierge endpoint - --concierge-use-impersonation-proxy Whether the concierge cluster uses an impersonation proxy + --concierge-ca-bundle string Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge + --concierge-endpoint string API base for the Concierge endpoint + --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) -h, --help help for kubeconfig --kubeconfig string Path to kubeconfig file --kubeconfig-context string Kubeconfig context name (default: current active context) - --no-concierge Generate a configuration which does not use the concierge, but sends the credential to the cluster directly + --no-concierge Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly --oidc-ca-bundle strings Path to TLS certificate authority bundle (PEM format, optional, can be repeated) --oidc-client-id string OpenID Connect client ID (default: autodiscover) (default "pinniped-cli") --oidc-issuer string OpenID Connect issuer URL (default: autodiscover) @@ -273,7 +273,12 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", "--concierge-ca-bundle", "./does/not/exist", "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", - "--concierge-use-impersonation-proxy", + "--concierge-authenticator-name", "test-authenticator", + "--concierge-authenticator-type", "webhook", + "--concierge-mode", "ImpersonationProxy", + }, + conciergeObjects: []runtime.Object{ + &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantError: true, wantStderr: here.Doc(` @@ -343,6 +348,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=webhook - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== + - --concierge-mode=TokenCredentialRequestAPI - --token=test-token command: '.../path/to/pinniped' env: [] @@ -387,6 +393,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=webhook - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== + - --concierge-mode=TokenCredentialRequestAPI - --token-env=TEST_TOKEN command: '.../path/to/pinniped' env: [] @@ -439,6 +446,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=jwt - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== + - --concierge-mode=TokenCredentialRequestAPI - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience @@ -498,6 +506,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=webhook - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== + - --concierge-mode=TokenCredentialRequestAPI - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience @@ -519,7 +528,7 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", "--concierge-ca-bundle", testConciergeCABundlePath, "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", - "--concierge-use-impersonation-proxy", + "--concierge-mode", "ImpersonationProxy", }, conciergeObjects: []runtime.Object{ &conciergev1alpha1.JWTAuthenticator{ @@ -562,7 +571,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=jwt - --concierge-endpoint=https://impersonation-proxy-endpoint.test - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= - - --concierge-use-impersonation-proxy + - --concierge-mode=ImpersonationProxy - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience From f937ae2c075c77ed20a6785f8b0296f3ddd7a0c4 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 25 Feb 2021 14:16:40 -0600 Subject: [PATCH 045/203] Add --concierge-credential-issuer flag to "pinniped get kubeconfig" command. This flag selects a CredentialIssuer to use when detecting what mode the Concierge is in on a cluster. If not specified, the command will look for a single CredentialIssuer. If there are multiple, then the flag is required. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/flag_types.go | 5 + cmd/pinniped/cmd/flag_types_test.go | 2 +- cmd/pinniped/cmd/kubeconfig.go | 92 ++++++++--- cmd/pinniped/cmd/kubeconfig_test.go | 230 +++++++++++++++++++++++++++- cmd/pinniped/cmd/login_oidc.go | 2 +- cmd/pinniped/cmd/login_static.go | 2 +- 6 files changed, 307 insertions(+), 26 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index 262bd9b8..8ca97681 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -16,6 +16,7 @@ type conciergeMode int var _ flag.Value = new(conciergeMode) const ( + modeUnknown conciergeMode = iota modeTokenCredentialRequestAPI conciergeMode = iota modeImpersonationProxy conciergeMode = iota ) @@ -32,6 +33,10 @@ func (c *conciergeMode) String() string { } func (c *conciergeMode) Set(s string) error { + if strings.EqualFold(s, "") { + *c = modeUnknown + return nil + } if strings.EqualFold(s, "TokenCredentialRequestAPI") { *c = modeTokenCredentialRequestAPI return nil diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go index 955ae624..26f107b7 100644 --- a/cmd/pinniped/cmd/flag_types_test.go +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -12,7 +12,7 @@ import ( func TestConciergeModeFlag(t *testing.T) { var m conciergeMode require.Equal(t, "mode", m.Type()) - require.Equal(t, modeTokenCredentialRequestAPI, m) + require.Equal(t, modeUnknown, m) require.EqualError(t, m.Set("foo"), `invalid mode "foo", valid modes are TokenCredentialRequestAPI and ImpersonationProxy`) require.NoError(t, m.Set("TokenCredentialRequestAPI")) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 1d1e271d..7bdcff64 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -25,6 +25,7 @@ import ( _ "k8s.io/client-go/plugin/pkg/client/auth" // Adds handlers for various dynamic auth plugins in client-go conciergev1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/groupsuffix" "go.pinniped.dev/internal/kubeclient" @@ -74,6 +75,7 @@ type getKubeconfigOIDCParams struct { type getKubeconfigConciergeParams struct { disabled bool + credentialIssuer string authenticatorName string authenticatorType string apiGroupSuffix string @@ -109,6 +111,7 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.BoolVar(&flags.concierge.disabled, "no-concierge", false, "Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly") f.StringVar(&namespace, "concierge-namespace", "pinniped-concierge", "Namespace in which the Concierge was installed") + f.StringVar(&flags.concierge.credentialIssuer, "concierge-credential-issuer", "", "Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover)") f.StringVar(&flags.concierge.authenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover)") f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") @@ -178,6 +181,11 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar } if !flags.concierge.disabled { + credentialIssuer, err := lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer) + if err != nil { + return err + } + authenticator, err := lookupAuthenticator( clientset, flags.concierge.authenticatorType, @@ -186,7 +194,8 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if err != nil { return err } - if err := configureConcierge(authenticator, &flags, cluster, &oidcCABundle, &execConfig); err != nil { + + if err := configureConcierge(credentialIssuer, authenticator, &flags, cluster, &oidcCABundle, &execConfig); err != nil { return err } } @@ -209,7 +218,7 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar // Otherwise continue to parse the OIDC-related flags and output a config that runs `pinniped login oidc`. execConfig.Args = append([]string{"login", "oidc"}, execConfig.Args...) if flags.oidc.issuer == "" { - return fmt.Errorf("could not autodiscover --oidc-issuer, and none was provided") + return fmt.Errorf("could not autodiscover --oidc-issuer and none was provided") } execConfig.Args = append(execConfig.Args, "--issuer="+flags.oidc.issuer, @@ -237,18 +246,30 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig)) } -func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig) error { +func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig) error { + var conciergeCABundleData []byte - if flags.concierge.mode == modeImpersonationProxy { - // TODO what to do if --use-impersonation-proxy is set but flags.concierge.caBundlePath is not??? - // TODO dont do this twice - conciergeCaBundleData, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) - if err != nil { - return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + // Autodiscover the --concierge-mode. + if flags.concierge.mode == modeUnknown { + + if credentialIssuer.Status.KubeConfigInfo != nil { + // Prefer the TokenCredentialRequest API if available. + flags.concierge.mode = modeTokenCredentialRequestAPI + } else if credentialIssuer.Status.ImpersonationProxyInfo != nil { + // Otherwise prefer the impersonation proxy if it seems configured. + flags.concierge.mode = modeImpersonationProxy + } else { + return fmt.Errorf("could not autodiscover --concierge-mode and none was provided") } + } - v1Cluster.CertificateAuthorityData = []byte(conciergeCaBundleData) - v1Cluster.Server = flags.concierge.endpoint + if flags.concierge.mode == modeImpersonationProxy && credentialIssuer.Status.ImpersonationProxyInfo != nil { + flags.concierge.endpoint = credentialIssuer.Status.ImpersonationProxyInfo.Endpoint + var err error + conciergeCABundleData, err = base64.StdEncoding.DecodeString(credentialIssuer.Status.ImpersonationProxyInfo.CertificateAuthorityData) + if err != nil { + return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) + } } switch auth := authenticator.(type) { @@ -292,15 +313,16 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, flags.concierge.endpoint = v1Cluster.Server } - var encodedConciergeCaBundleData string - if flags.concierge.caBundlePath == "" { - encodedConciergeCaBundleData = base64.StdEncoding.EncodeToString(v1Cluster.CertificateAuthorityData) - } else { - conciergeCaBundleData, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) - if err != nil { - return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + if conciergeCABundleData == nil { + if flags.concierge.caBundlePath == "" { + conciergeCABundleData = v1Cluster.CertificateAuthorityData + } else { + caBundleString, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) + if err != nil { + return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + } + conciergeCABundleData = []byte(caBundleString) } - encodedConciergeCaBundleData = base64.StdEncoding.EncodeToString([]byte(conciergeCaBundleData)) } // Append the flags to configure the Concierge credential exchange at runtime. @@ -310,9 +332,16 @@ func configureConcierge(authenticator metav1.Object, flags *getKubeconfigParams, "--concierge-authenticator-name="+flags.concierge.authenticatorName, "--concierge-authenticator-type="+flags.concierge.authenticatorType, "--concierge-endpoint="+flags.concierge.endpoint, - "--concierge-ca-bundle-data="+encodedConciergeCaBundleData, + "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(conciergeCABundleData), "--concierge-mode="+flags.concierge.mode.String(), ) + + // If we're in impersonation proxy mode, the main server endpoint for the kubeconfig also needs to point to the proxy + if flags.concierge.mode == modeImpersonationProxy { + v1Cluster.CertificateAuthorityData = conciergeCABundleData + v1Cluster.Server = flags.concierge.endpoint + } + return nil } @@ -343,6 +372,29 @@ func newExecKubeconfig(cluster *clientcmdapi.Cluster, execConfig *clientcmdapi.E } } +func lookupCredentialIssuer(clientset conciergeclientset.Interface, name string) (*configv1alpha1.CredentialIssuer, error) { + ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) + defer cancelFunc() + + // If the name is specified, get that object. + if name != "" { + return clientset.ConfigV1alpha1().CredentialIssuers().Get(ctx, name, metav1.GetOptions{}) + } + + // Otherwise list all the available CredentialIssuers and hope there's just a single one + results, err := clientset.ConfigV1alpha1().CredentialIssuers().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list CredentialIssuer objects for autodiscovery: %w", err) + } + if len(results.Items) == 0 { + return nil, fmt.Errorf("no CredentialIssuers were found") + } + if len(results.Items) > 1 { + return nil, fmt.Errorf("multiple CredentialIssuers were found, so the --concierge-credential-issuer flag must be specified") + } + return &results.Items[0], nil +} + func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authName string) (metav1.Object, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) defer cancelFunc() diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 2a431c1e..94bb5017 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -20,6 +20,7 @@ import ( "k8s.io/client-go/tools/clientcmd" conciergev1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" fakeconciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" "go.pinniped.dev/internal/certauthority" @@ -65,6 +66,7 @@ func TestGetKubeconfig(t *testing.T) { --concierge-authenticator-name string Concierge authenticator name (default: autodiscover) --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) --concierge-ca-bundle string Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge + --concierge-credential-issuer string Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover) --concierge-endpoint string API base for the Concierge endpoint --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) -h, --help help for kubeconfig @@ -134,6 +136,31 @@ func TestGetKubeconfig(t *testing.T) { Error: could not configure Kubernetes client: some kube error `), }, + { + name: "no credentialissuers", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + wantError: true, + wantStderr: here.Doc(` + Error: no CredentialIssuers were found + `), + }, + + { + name: "credentialissuer not found", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-credential-issuer", "does-not-exist", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, + wantError: true, + wantStderr: here.Doc(` + Error: credentialissuers.config.concierge.pinniped.dev "does-not-exist" not found + `), + }, { name: "webhook authenticator not found", args: []string{ @@ -141,6 +168,9 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, wantError: true, wantStderr: here.Doc(` Error: webhookauthenticators.authentication.concierge.pinniped.dev "test-authenticator" not found @@ -153,6 +183,9 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-authenticator-type", "jwt", "--concierge-authenticator-name", "test-authenticator", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, wantError: true, wantStderr: here.Doc(` Error: jwtauthenticators.authentication.concierge.pinniped.dev "test-authenticator" not found @@ -165,6 +198,9 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-authenticator-type", "invalid", "--concierge-authenticator-name", "test-authenticator", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, wantError: true, wantStderr: here.Doc(` Error: invalid authenticator type "invalid", supported values are "webhook" and "jwt" @@ -175,6 +211,9 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, conciergeReactions: []kubetesting.Reactor{ &kubetesting.SimpleReactor{ Verb: "*", @@ -194,6 +233,9 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, conciergeReactions: []kubetesting.Reactor{ &kubetesting.SimpleReactor{ Verb: "*", @@ -213,6 +255,9 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + }, wantError: true, wantStderr: here.Doc(` Error: no authenticators were found @@ -224,6 +269,7 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, &conciergev1alpha1.JWTAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-1"}}, &conciergev1alpha1.JWTAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-2"}}, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-3"}}, @@ -234,17 +280,62 @@ func TestGetKubeconfig(t *testing.T) { Error: multiple authenticators were found, so the --concierge-authenticator-type/--concierge-authenticator-name flags must be specified `), }, + { + name: "autodetect webhook authenticator, bad credential issuer with no status", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, + }, + wantError: true, + wantStderr: here.Doc(` + Error: could not autodiscover --concierge-mode and none was provided + `), + }, + { + name: "autodetect webhook authenticator, bad credential issuer with invalid impersonation CA", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + ImpersonationProxyInfo: &configv1alpha1.CredentialIssuerImpersonationProxyInfo{ + Endpoint: "https://impersonation-endpoint", + CertificateAuthorityData: "invalid-base-64", + }, + }, + }, + &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, + }, + wantError: true, + wantStderr: here.Doc(` + Error: autodiscovered Concierge CA bundle is invalid: illegal base64 data at input byte 7 + `), + }, { name: "autodetect webhook authenticator, missing --oidc-issuer", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantError: true, wantStderr: here.Doc(` - Error: could not autodiscover --oidc-issuer, and none was provided + Error: could not autodiscover --oidc-issuer and none was provided `), }, { @@ -253,6 +344,15 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, Spec: conciergev1alpha1.JWTAuthenticatorSpec{ @@ -268,7 +368,7 @@ func TestGetKubeconfig(t *testing.T) { `), }, { - name: " invalid concierge ca bundle", + name: "invalid concierge ca bundle", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", "--concierge-ca-bundle", "./does/not/exist", @@ -278,6 +378,15 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-mode", "ImpersonationProxy", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantError: true, @@ -293,6 +402,15 @@ func TestGetKubeconfig(t *testing.T) { "--static-token-env", "TEST_TOKEN", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantError: true, @@ -317,6 +435,15 @@ func TestGetKubeconfig(t *testing.T) { "--static-token", "test-token", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantStdout: here.Doc(` @@ -362,6 +489,15 @@ func TestGetKubeconfig(t *testing.T) { "--static-token-env", "TEST_TOKEN", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantStdout: here.Doc(` @@ -406,6 +542,15 @@ func TestGetKubeconfig(t *testing.T) { "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, Spec: conciergev1alpha1.JWTAuthenticatorSpec{ @@ -473,6 +618,15 @@ func TestGetKubeconfig(t *testing.T) { "--oidc-request-audience", "test-audience", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ + Server: "https://concierge-endpoint", + CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", + }, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, }, @@ -531,6 +685,76 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-mode", "ImpersonationProxy", }, conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{}, + }, + &conciergev1alpha1.JWTAuthenticator{ + ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, + Spec: conciergev1alpha1.JWTAuthenticatorSpec{ + Issuer: "https://example.com/issuer", + Audience: "test-audience", + TLS: &conciergev1alpha1.TLSSpec{ + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + }, + }, + }, + }, + wantStdout: here.Docf(` + apiVersion: v1 + clusters: + - cluster: + certificate-authority-data: dGVzdC1jb25jaWVyZ2UtY2E= + server: https://impersonation-proxy-endpoint.test + name: pinniped + contexts: + - context: + cluster: pinniped + user: pinniped + name: pinniped + current-context: pinniped + kind: Config + preferences: {} + users: + - name: pinniped + user: + exec: + apiVersion: client.authentication.k8s.io/v1beta1 + args: + - login + - oidc + - --enable-concierge + - --concierge-api-group-suffix=pinniped.dev + - --concierge-authenticator-name=test-authenticator + - --concierge-authenticator-type=jwt + - --concierge-endpoint=https://impersonation-proxy-endpoint.test + - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= + - --concierge-mode=ImpersonationProxy + - --issuer=https://example.com/issuer + - --client-id=pinniped-cli + - --scopes=offline_access,openid,pinniped:request-audience + - --ca-bundle-data=%s + - --request-audience=test-audience + command: '.../path/to/pinniped' + env: [] + provideClusterInfo: true + `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), + }, + { + name: "autodetect impersonation proxy with autodetected JWT authenticator", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + }, + conciergeObjects: []runtime.Object{ + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + ImpersonationProxyInfo: &configv1alpha1.CredentialIssuerImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.test", + CertificateAuthorityData: "dGVzdC1jb25jaWVyZ2UtY2E=", + }, + }, + }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, Spec: conciergev1alpha1.JWTAuthenticatorSpec{ @@ -604,7 +828,7 @@ func TestGetKubeconfig(t *testing.T) { } fake := fakeconciergeclientset.NewSimpleClientset(tt.conciergeObjects...) if len(tt.conciergeReactions) > 0 { - fake.ReactionChain = tt.conciergeReactions + fake.ReactionChain = append(tt.conciergeReactions, fake.ReactionChain...) } return fake, nil }, diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 2eddee79..e1db5689 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -190,7 +190,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin // The exact behavior depends on in which mode the Concierge is operating. switch flags.conciergeMode { - case modeTokenCredentialRequestAPI: + case modeUnknown, modeTokenCredentialRequestAPI: // do a credential exchange request cred, err := deps.exchangeToken(ctx, concierge, token.IDToken.Token) if err != nil { diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index fd1526c2..9141c4e6 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -122,7 +122,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams // If the concierge is enabled, we need to do extra steps. switch flags.conciergeMode { - case modeTokenCredentialRequestAPI: + case modeUnknown, modeTokenCredentialRequestAPI: // do a credential exchange request ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() From 3fcde8088cb361a7088ab847725be08f5169166a Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 25 Feb 2021 14:40:02 -0800 Subject: [PATCH 046/203] concierge_impersonation_proxy_test.go: Make it work on more clusters Should work on cluster which have: - load balancers not supported, has squid proxy (e.g. kind) - load balancers supported, has squid proxy (e.g. EKS) - load balancers supported, no squid proxy (e.g. GKE) When testing with a load balancer, call the impersonation proxy through the load balancer. Also, added a new library.RequireNeverWithoutError() helper. Signed-off-by: Margo Crawford --- .../concierge_impersonation_proxy_test.go | 186 ++++++++++++------ test/library/assertions.go | 25 ++- 2 files changed, 150 insertions(+), 61 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index b59fe91d..aec7bcdd 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -31,16 +31,17 @@ import ( const ( // TODO don't hard code "pinniped-concierge-" in these strings. It should be constructed from the env app name. - impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" - impersonationProxyTLSSecretName = "pinniped-concierge-impersonation-proxy-tls-serving-certificate" //nolint:gosec // this is not a credential + impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" + impersonationProxyTLSSecretName = "pinniped-concierge-impersonation-proxy-tls-serving-certificate" //nolint:gosec // this is not a credential + impersonationProxyLoadBalancerName = "pinniped-concierge-impersonation-proxy-load-balancer" ) +// Note that this test supports being run on all of our integration test cluster types: +// - load balancers not supported, has squid proxy (e.g. kind) +// - load balancers supported, has squid proxy (e.g. EKS) +// - load balancers supported, no squid proxy (e.g. GKE) func TestImpersonationProxy(t *testing.T) { env := library.IntegrationEnv(t) - if env.Proxy == "" { - t.Skip("this test can only run in environments with the in-cluster proxy right now") - return - } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() @@ -51,14 +52,15 @@ func TestImpersonationProxy(t *testing.T) { // Create a WebhookAuthenticator. authenticator := library.CreateTestWebhookAuthenticator(ctx, t) - // The address of the ClusterIP service that points at the impersonation proxy's port + // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) - proxyServiceURL := fmt.Sprintf("https://%s", proxyServiceEndpoint) - t.Logf("making kubeconfig that points to %q", proxyServiceURL) + // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. + serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) - getImpersonationProxyClient := func(caData []byte) *kubernetes.Clientset { + impersonationProxyViaSquidClient := func(caData []byte) *kubernetes.Clientset { + t.Helper() kubeconfig := &rest.Config{ - Host: proxyServiceURL, + Host: fmt.Sprintf("https://%s", proxyServiceEndpoint), TLSClientConfig: rest.TLSClientConfig{Insecure: caData == nil, CAData: caData}, BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), Proxy: func(req *http.Request) (*url.URL, error) { @@ -68,37 +70,68 @@ func TestImpersonationProxy(t *testing.T) { return proxyURL, nil }, } + impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) + require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") + return impersonationProxyClient + } + impersonationProxyViaLoadBalancerClient := func(host string, caData []byte) *kubernetes.Clientset { + t.Helper() + kubeconfig := &rest.Config{ + Host: fmt.Sprintf("https://%s", host), + TLSClientConfig: rest.TLSClientConfig{Insecure: caData == nil, CAData: caData}, + BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), + } impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") return impersonationProxyClient } oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName, metav1.GetOptions{}) - if oldConfigMap.Data != nil { + if !k8serrors.IsNotFound(err) { + require.NoError(t, err) // other errors aside from NotFound are unexpected t.Logf("stashing a pre-existing configmap %s", oldConfigMap.Name) require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName, metav1.DeleteOptions{})) } - serviceUnavailableError := fmt.Sprintf(`Get "%s/api/v1/namespaces": Service Unavailable`, proxyServiceURL) - insecureImpersonationProxyClient := getImpersonationProxyClient(nil) + impersonationProxyLoadBalancerIngress := "" - if env.HasCapability(library.HasExternalLoadBalancerProvider) { - // Check that load balancer has been created - require.Eventually(t, func() bool { - return hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) + if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test + // Check that load balancer has been created. + library.RequireEventuallyWithoutError(t, func() (bool, error) { + return hasImpersonationProxyLoadBalancerService(ctx, adminClient, env.ConciergeNamespace) }, 10*time.Second, 500*time.Millisecond) + + // Wait for the load balancer to get an ingress and make a note of its address. + var ingress *corev1.LoadBalancerIngress + library.RequireEventuallyWithoutError(t, func() (bool, error) { + ingress, err = getImpersonationProxyLoadBalancerIngress(ctx, adminClient, env.ConciergeNamespace) + if err != nil { + return false, err + } + return ingress != nil, nil + }, 10*time.Second, 500*time.Millisecond) + if ingress.Hostname != "" { + impersonationProxyLoadBalancerIngress = ingress.Hostname + } else { + require.NotEmpty(t, ingress.IP, "the ingress should have either a hostname or IP, but it didn't") + impersonationProxyLoadBalancerIngress = ingress.IP + } } else { - // Check that no load balancer has been created - require.Never(t, func() bool { - return hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) + require.NotEmpty(t, env.Proxy, + "test cluster does not support load balancers but also doesn't have a squid proxy... "+ + "this is not a supported configuration for test clusters") + + // Check that no load balancer has been created. + library.RequireNeverWithoutError(t, func() (bool, error) { + return hasImpersonationProxyLoadBalancerService(ctx, adminClient, env.ConciergeNamespace) }, 10*time.Second, 500*time.Millisecond) - // Check that we can't use the impersonation proxy to execute kubectl commands yet - _, err = insecureImpersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - require.EqualError(t, err, serviceUnavailableError) + // Check that we can't use the impersonation proxy to execute kubectl commands yet. + _, err = impersonationProxyViaSquidClient(nil).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + require.EqualError(t, err, serviceUnavailableViaSquidError) - // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer) + // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer). configMap := configMapForConfig(t, impersonator.Config{ Mode: impersonator.ModeEnabled, Endpoint: proxyServiceEndpoint, @@ -108,6 +141,7 @@ func TestImpersonationProxy(t *testing.T) { _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) require.NoError(t, err) + // At the end of the test, clean up the ConfigMap. t.Cleanup(func() { ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -134,8 +168,17 @@ func TestImpersonationProxy(t *testing.T) { return caSecret != nil && caSecret.Data["ca.crt"] != nil }, 5*time.Minute, 250*time.Millisecond) - // Create an impersonation proxy client with that ca data. - impersonationProxyClient := getImpersonationProxyClient(caSecret.Data["ca.crt"]) + // Create an impersonation proxy client with that CA data to use for the rest of this test. + // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. + var impersonationProxyClient *kubernetes.Clientset + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, caSecret.Data["ca.crt"]) + } else { + impersonationProxyClient = impersonationProxyViaSquidClient(caSecret.Data["ca.crt"]) + } + + // Test that the user can perform basic actions through the client with their username and group membership + // influencing RBAC checks correctly. t.Run( "access as user", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyClient), @@ -148,13 +191,14 @@ func TestImpersonationProxy(t *testing.T) { ) } - t.Run("watching all the verbs", func(t *testing.T) { - // Create a namespace, because it will be easier to deletecollection if we have a namespace. - // t.Cleanup Delete the namespace. + // Try more Kube API verbs through the impersonation proxy. + t.Run("watching all the basic verbs", func(t *testing.T) { + // Create a namespace, because it will be easier to exercise deletecollection if we have a namespace. namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, }, metav1.CreateOptions{}) require.NoError(t, err) + // Schedule the namespace for cleanup. t.Cleanup(func() { t.Logf("cleaning up test namespace %s", namespace.Name) err = adminClient.CoreV1().Namespaces().Delete(context.Background(), namespace.Name, metav1.DeleteOptions{}) @@ -175,6 +219,7 @@ func TestImpersonationProxy(t *testing.T) { Name: "cluster-admin", }, ) + // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Namespace: namespace.Name, Verb: "create", @@ -183,7 +228,7 @@ func TestImpersonationProxy(t *testing.T) { Resource: "configmaps", }) - // Create and start informer. + // Create and start informer to exercise the "watch" verb for us. informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( impersonationProxyClient, 0, @@ -198,10 +243,13 @@ func TestImpersonationProxy(t *testing.T) { }) informerFactory.WaitForCacheSync(ctx.Done()) - // Test "create" verb through the impersonation proxy. + // Use labels on our created ConfigMaps to avoid accidentally listing other ConfigMaps that might + // exist in the namespace. In Kube 1.20+ there is a default ConfigMap in every namespace. configMapLabels := labels.Set{ "pinniped.dev/testConfigMap": library.RandHex(t, 8), } + + // Test "create" verb through the impersonation proxy. _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, @@ -244,8 +292,7 @@ func TestImpersonationProxy(t *testing.T) { require.NoError(t, err) require.Equal(t, "bar", updateResult.Data["foo"]) - // Make sure that the updated ConfigMap shows up in the informer's cache to - // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + // Make sure that the updated ConfigMap shows up in the informer's cache. require.Eventually(t, func() bool { configMap, err := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") return err == nil && configMap.Data["foo"] == "bar" @@ -262,8 +309,7 @@ func TestImpersonationProxy(t *testing.T) { require.Equal(t, "bar", patchResult.Data["foo"]) require.Equal(t, "42", patchResult.Data["baz"]) - // Make sure that the patched ConfigMap shows up in the informer's cache to - // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + // Make sure that the patched ConfigMap shows up in the informer's cache. require.Eventually(t, func() bool { configMap, err := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") return err == nil && configMap.Data["foo"] == "bar" && configMap.Data["baz"] == "42" @@ -273,8 +319,7 @@ func TestImpersonationProxy(t *testing.T) { err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) require.NoError(t, err) - // Make sure that the deleted ConfigMap shows up in the informer's cache to - // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + // Make sure that the deleted ConfigMap shows up in the informer's cache. require.Eventually(t, func() bool { _, getErr := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(configMapLabels.AsSelector()) @@ -285,13 +330,13 @@ func TestImpersonationProxy(t *testing.T) { err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) require.NoError(t, err) - // Make sure that the deleted ConfigMaps shows up in the informer's cache to - // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + // Make sure that the deleted ConfigMaps shows up in the informer's cache. require.Eventually(t, func() bool { list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(configMapLabels.AsSelector()) return listErr == nil && len(list) == 0 }, 10*time.Second, 50*time.Millisecond) + // There should be no ConfigMaps left. listResult, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) @@ -311,22 +356,30 @@ func TestImpersonationProxy(t *testing.T) { require.NoError(t, err) } - // Check that the impersonation proxy has shut down - require.Eventually(t, func() bool { - // It's okay if this returns RBAC errors because this user has no role bindings. - // What we want to see is that the proxy eventually shuts down entirely. - _, err = insecureImpersonationProxyClient.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - return err.Error() == serviceUnavailableError - }, 20*time.Second, 500*time.Millisecond) - if env.HasCapability(library.HasExternalLoadBalancerProvider) { // The load balancer should not exist after we disable the impersonation proxy. // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). - require.Eventually(t, func() bool { - return !hasLoadBalancerService(ctx, t, adminClient, env.ConciergeNamespace) + library.RequireEventuallyWithoutError(t, func() (bool, error) { + hasService, err := hasImpersonationProxyLoadBalancerService(ctx, adminClient, env.ConciergeNamespace) + return !hasService, err }, time.Minute, 500*time.Millisecond) } + // Check that the impersonation proxy port has shut down. + // Ideally we could always check that the impersonation proxy's port has shut down, but on clusters where we + // do not run the squid proxy we have no easy way to see beyond the load balancer to see inside the cluster, + // so we'll skip this check on clusters which have load balancers but don't run the squid proxy. + // The other cluster types that do run the squid proxy will give us sufficient coverage here. + if env.Proxy != "" { + require.Eventually(t, func() bool { + // It's okay if this returns RBAC errors because this user has no role bindings. + // What we want to see is that the proxy eventually shuts down entirely. + _, err = impersonationProxyViaSquidClient(nil).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + return err.Error() == serviceUnavailableViaSquidError + }, 20*time.Second, 500*time.Millisecond) + } + + // Check that the generated TLS cert Secret was deleted by the controller. require.Eventually(t, func() bool { caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) return k8serrors.IsNotFound(err) @@ -346,15 +399,28 @@ func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigM return configMap } -func hasLoadBalancerService(ctx context.Context, t *testing.T, client kubernetes.Interface, namespace string) bool { - t.Helper() - - services, err := client.CoreV1().Services(namespace).List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - for _, service := range services.Items { - if service.Spec.Type == corev1.ServiceTypeLoadBalancer { - return true - } +func hasImpersonationProxyLoadBalancerService(ctx context.Context, client kubernetes.Interface, namespace string) (bool, error) { + service, err := client.CoreV1().Services(namespace).Get(ctx, impersonationProxyLoadBalancerName, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return false, nil } - return false + if err != nil { + return false, err + } + return service.Spec.Type == corev1.ServiceTypeLoadBalancer, nil +} + +func getImpersonationProxyLoadBalancerIngress(ctx context.Context, client kubernetes.Interface, namespace string) (*corev1.LoadBalancerIngress, error) { + service, err := client.CoreV1().Services(namespace).Get(ctx, impersonationProxyLoadBalancerName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + ingresses := service.Status.LoadBalancer.Ingress + if len(ingresses) > 1 { + return nil, fmt.Errorf("didn't expect multiple ingresses, but if it happens then maybe this test needs to be adjusted") + } + if len(ingresses) == 0 { + return nil, nil + } + return &ingresses[0], nil } diff --git a/test/library/assertions.go b/test/library/assertions.go index 4f42f5a7..4db0ee09 100644 --- a/test/library/assertions.go +++ b/test/library/assertions.go @@ -5,6 +5,7 @@ package library import ( "context" + "errors" "fmt" "testing" "time" @@ -15,7 +16,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" ) -// RequireEventuallyWithoutError is a wrapper around require.Eventually() that allows the caller to +// RequireEventuallyWithoutError is similar to require.Eventually() except that it also allows the caller to // return an error from the condition function. If the condition function returns an error at any // point, the assertion will immediately fail. func RequireEventuallyWithoutError( @@ -29,6 +30,28 @@ func RequireEventuallyWithoutError( require.NoError(t, wait.PollImmediate(tick, waitFor, f), msgAndArgs...) } +// RequireNeverWithoutError is similar to require.Never() except that it also allows the caller to +// return an error from the condition function. If the condition function returns an error at any +// point, the assertion will immediately fail. +func RequireNeverWithoutError( + t *testing.T, + f func() (bool, error), + waitFor time.Duration, + tick time.Duration, + msgAndArgs ...interface{}, +) { + t.Helper() + err := wait.PollImmediate(tick, waitFor, f) + isWaitTimeout := errors.Is(err, wait.ErrWaitTimeout) + if err != nil && !isWaitTimeout { + require.NoError(t, err, msgAndArgs...) // this will fail and throw the right error message + } + if err == nil { + // This prints the same error message that require.Never would print in this case. + require.Fail(t, "Condition satisfied", msgAndArgs...) + } +} + // NewRestartAssertion allows a caller to assert that there were no restarts for a Pod in the // provided namespace with the provided labelSelector during the lifetime of a test. func AssertNoRestartsDuringTest(t *testing.T, namespace, labelSelector string) { From ccb17843c1d3df155bb1e49829edd3a7a12f43fb Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Thu, 25 Feb 2021 15:06:24 -0800 Subject: [PATCH 047/203] Fix some lint errors that resulted from merging main Signed-off-by: Ryan Richard --- cmd/pinniped/cmd/flag_types.go | 2 +- cmd/pinniped/cmd/kubeconfig.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index 8ca97681..ad682a1d 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -25,7 +25,7 @@ func (c *conciergeMode) String() string { switch *c { case modeImpersonationProxy: return "ImpersonationProxy" - case modeTokenCredentialRequestAPI: + case modeTokenCredentialRequestAPI, modeUnknown: return "TokenCredentialRequestAPI" default: return "TokenCredentialRequestAPI" diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 7bdcff64..5dfe87b1 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -251,14 +251,14 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // Autodiscover the --concierge-mode. if flags.concierge.mode == modeUnknown { - - if credentialIssuer.Status.KubeConfigInfo != nil { + switch { + case credentialIssuer.Status.KubeConfigInfo != nil: // Prefer the TokenCredentialRequest API if available. flags.concierge.mode = modeTokenCredentialRequestAPI - } else if credentialIssuer.Status.ImpersonationProxyInfo != nil { + case credentialIssuer.Status.ImpersonationProxyInfo != nil: // Otherwise prefer the impersonation proxy if it seems configured. flags.concierge.mode = modeImpersonationProxy - } else { + default: return fmt.Errorf("could not autodiscover --concierge-mode and none was provided") } } From f709da556972ee33b39d749cf19629f9d8fb9f6a Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Thu, 25 Feb 2021 15:18:36 -0800 Subject: [PATCH 048/203] Updated test assertions for new logger version Signed-off-by: Ryan Richard --- .../impersonator/impersonator_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index ccd24856..3e590ccf 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -118,49 +118,49 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"\\\"Impersonate-User\\\" header already exists\" \"msg\"=\"impersonation header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-User\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "Impersonate-Group header already in request", request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"\\\"Impersonate-Group\\\" header already exists\" \"msg\"=\"impersonation header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Group\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "Impersonate-Extra header already in request", request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"\\\"Impersonate-Extra-\\\" header already exists\" \"msg\"=\"impersonation header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Extra-\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "missing authorization header", request: newRequest(map[string][]string{}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"token authenticator did not find token\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"token authenticator did not find token\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "authorization header missing bearer prefix", request: newRequest(map[string][]string{"Authorization": {impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"token authenticator did not find token\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"token authenticator did not find token\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token is not base64 encoded", request: newRequest(map[string][]string{"Authorization": {"Bearer !!!"}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"invalid base64 in encoded bearer token: illegal base64 data at input byte 0\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid base64 in encoded bearer token: illegal base64 data at input byte 0\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "base64 encoded token is not valid json", request: newRequest(map[string][]string{"Authorization": {"Bearer aGVsbG8gd29ybGQK"}}), // aGVsbG8gd29ybGQK is "hello world" base64 encoded wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: couldn't get version/kind; json parse error: invalid character 'h' looking for beginning of value\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid object encoded in bearer token: couldn't get version/kind; json parse error: invalid character 'h' looking for beginning of value\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "base64 encoded token is encoded with default api group but we are expecting custom api group", @@ -168,21 +168,21 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.pinniped.dev/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.pinniped.dev/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "base64 encoded token is encoded with custom api group but we are expecting default api group", request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}}), wantHTTPBody: "invalid token encoding\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.walrus.tld/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"msg\"=\"invalid token encoding\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.walrus.tld/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token could not be authenticated", request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "", &badAuthenticator, defaultAPIGroup)}}), wantHTTPBody: "invalid token\n", wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"error\"=\"no such authenticator\" \"msg\"=\"received invalid token\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"received invalid token\" \"error\"=\"no such authenticator\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "token authenticates as nil", From bbbb40994d085e2f182658dd66aae63f80333755 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 25 Feb 2021 17:03:34 -0800 Subject: [PATCH 049/203] Prefer hostnames over IPs when making certs to match load balancer ingress Signed-off-by: Margo Crawford --- .../impersonatorconfig/impersonator_config.go | 86 +++++----- .../impersonator_config_test.go | 152 +++++++++++++++--- 2 files changed, 182 insertions(+), 56 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index c428b26c..abda84a4 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -217,7 +217,17 @@ func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonat } func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { - return c.shouldHaveImpersonator(config) // TODO is this the logic that we want here? + return c.shouldHaveImpersonator(config) +} + +func certHostnameAndIPMatchDesiredState(desiredIP net.IP, actualIPs []net.IP, desiredHostname string, actualHostnames []string) bool { + if desiredIP != nil && len(actualIPs) == 1 && desiredIP.Equal(actualIPs[0]) && len(actualHostnames) == 0 { + return true + } + if desiredHostname != "" && len(actualHostnames) == 1 && desiredHostname == actualHostnames[0] && len(actualIPs) == 0 { + return true + } + return false } func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { @@ -357,12 +367,12 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con block, _ := pem.Decode(certPEM) if block == nil { // The certPEM is not valid. - return secret, nil // TODO what should we do? + return secret, nil // TODO delete this so the controller will create a valid one } actualCertFromSecret, _ := x509.ParseCertificate(block.Bytes) // TODO handle err - desiredIPs, desiredHostnames, nameIsReady, err := c.findTLSCertificateName(config) + desiredIP, desiredHostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) //nolint:staticcheck // TODO remove this nolint when we fix the TODO below if err != nil { // TODO return err @@ -380,22 +390,16 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con actualIPs := actualCertFromSecret.IPAddresses actualHostnames := actualCertFromSecret.DNSNames plog.Info("Checking TLS certificate names", - "desiredIPs", desiredIPs, - "desiredHostnames", desiredHostnames, + "desiredIP", desiredIP, + "desiredHostname", desiredHostname, "actualIPs", actualIPs, "actualHostnames", actualHostnames, "secret", c.tlsSecretName, "namespace", c.namespace) - // TODO handle multiple IPs - if len(desiredIPs) == len(actualIPs) && len(desiredIPs) == 1 && desiredIPs[0].Equal(actualIPs[0]) { //nolint:gocritic - // The cert matches the desired state, so we do not need to delete it. + if certHostnameAndIPMatchDesiredState(desiredIP, actualIPs, desiredHostname, actualHostnames) { return secret, nil } - if len(desiredHostnames) == len(actualHostnames) && len(desiredHostnames) == 1 && desiredHostnames[0] == actualHostnames[0] { //nolint:gocritic - return secret, nil - } - err = c.ensureTLSSecretIsRemoved(ctx) if err != nil { return secret, err @@ -419,7 +423,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return fmt.Errorf("could not create impersonation CA: %w", err) } - ips, hostnames, nameIsReady, err := c.findTLSCertificateName(config) + ip, hostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) if err != nil { return err } @@ -428,6 +432,14 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } + var hostnames []string + var ips []net.IP + if hostname != "" { + hostnames = []string{hostname} + } + if ip != nil { + ips = []net.IP{ip} + } newTLSSecret, err := c.createNewTLSSecret(ctx, impersonationCA, ips, hostnames) if err != nil { return err @@ -441,56 +453,54 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } -func (c *impersonatorConfigController) findTLSCertificateName(config *impersonator.Config) ([]net.IP, []string, bool, error) { +func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *impersonator.Config) (net.IP, string, bool, error) { if config.Endpoint != "" { return c.findTLSCertificateNameFromEndpointConfig(config) } return c.findTLSCertificateNameFromLoadBalancer() } -func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) ([]net.IP, []string, bool, error) { - var ips []net.IP - var hostnames []string +func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) (net.IP, string, bool, error) { + // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose parsedAsIP := net.ParseIP(config.Endpoint) if parsedAsIP != nil { - ips = []net.IP{parsedAsIP} - } else { - hostnames = []string{config.Endpoint} + return parsedAsIP, "", true, nil } - // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose - return ips, hostnames, true, nil + return nil, config.Endpoint, true, nil } -func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() ([]net.IP, []string, bool, error) { - var ips []net.IP - var hostnames []string +func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() (net.IP, string, bool, error) { lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) notFound := k8serrors.IsNotFound(err) if notFound { - // TODO is this an error? we should have already created the load balancer, so why would it not exist here? - return nil, nil, false, nil + // Maybe the loadbalancer hasn't been cached in the informer yet. We aren't ready and will try again later. + return nil, "", false, nil } if err != nil { - return nil, nil, false, err + return nil, "", false, err } ingresses := lb.Status.LoadBalancer.Ingress if len(ingresses) == 0 || (ingresses[0].Hostname == "" && ingresses[0].IP == "") { plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", "service", c.generatedLoadBalancerServiceName, "namespace", c.namespace) - return nil, nil, false, nil + return nil, "", false, nil } - hostname := ingresses[0].Hostname - if hostname != "" { - return nil, []string{hostname}, true, nil + for _, ingress := range ingresses { + hostname := ingress.Hostname + if hostname != "" { + return nil, hostname, true, nil + } } - ip := ingresses[0].IP - parsedIP := net.ParseIP(ip) - if parsedIP == nil { - return nil, nil, false, fmt.Errorf("could not parse IP address from load balancer %s: %s", lb.Name, ip) + for _, ingress := range ingresses { + ip := ingress.IP + parsedIP := net.ParseIP(ip) + if parsedIP != nil { + return parsedIP, "", true, nil + } } - ips = []net.IP{parsedIP} - return ips, hostnames, true, nil + + return nil, "", false, fmt.Errorf("could not find valid IP addresses or hostnames from load balancer %s/%s", c.namespace, lb.Name) } func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP, hostnames []string) (*v1.Secret, error) { diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index b5b26355..20512aad 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -516,6 +516,31 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } + var createTLSSecretWithMultipleHostnames = func(resourceName string, ip string) *corev1.Secret { + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) + r.NoError(err) + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, []string{"foo", "bar"}, []net.IP{net.ParseIP(ip)}, 24*time.Hour) + r.NoError(err) + certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + r.NoError(err) + + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Data: map[string][]byte{ + "ca.crt": impersonationCA.Bundle(), + corev1.TLSPrivateKeyKey: keyPEM, + corev1.TLSCertKey: certPEM, + }, + } + } + var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -558,7 +583,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } - var addLoadBalancerServiceWithIngressToTracker = func(resourceName string, ip string, hostname string, client *kubernetesfake.Clientset) { + var addLoadBalancerServiceWithIngressToTracker = func(resourceName string, ingress []corev1.LoadBalancerIngress, client *kubernetesfake.Clientset) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -573,12 +598,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }, Status: corev1.ServiceStatus{ LoadBalancer: corev1.LoadBalancerStatus{ - Ingress: []corev1.LoadBalancerIngress{ - { - IP: ip, - Hostname: hostname, - }, - }, + Ingress: ingress, }, }, } @@ -766,8 +786,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with empty ingress", func() { it.Before(func() { addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "", "", kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "", "", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeAPIClient) startInformersAndController() r.NoError(controllerlib.TestSync(t, subject, *syncContext)) }) @@ -785,10 +805,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with invalid ip", func() { it.Before(func() { addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "not-an-ip", "", kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "not-an-ip", "", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeAPIClient) startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not parse IP address from load balancer some-service-resource-name: not-an-ip") + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") }) it("starts the impersonator without tls certs", func() { @@ -800,6 +820,102 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) }) }) + + when("there are not visible control plane nodes and a load balancer already exists with multiple ips", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeAPIClient) + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + }) + + it("starts the impersonator with certs that match the first IP address", func() { + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunning(ca, "127.0.0.123", map[string]string{"127.0.0.123:443": startedTLSListener.Addr().String()}) + }) + + it("keeps the secret around after resync", func() { + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) // nothing changed + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with multiple hostnames", func() { + firstHostname := "fake-1.example.com" + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeAPIClient) + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + }) + + it("starts the impersonator with certs that match the first hostname", func() { + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + ":443": startedTLSListener.Addr().String()}) + }) + + it("keeps the secret around after resync", func() { + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) // nothing changed + }) + }) + + when("there are not visible control plane nodes and a load balancer already exists with hostnames and ips", func() { + firstHostname := "fake-1.example.com" + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeAPIClient) + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + }) + + it("starts the impersonator with certs that match the first hostname", func() { + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + ":443": startedTLSListener.Addr().String()}) + }) + + it("keeps the secret around after resync", func() { + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 2) // nothing changed + }) + }) + + when("there are not visible control plane nodes, a secret exists with multiple hostnames and an IP", func() { + var tlsSecret *corev1.Secret + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + tlsSecret = createTLSSecretWithMultipleHostnames(tlsSecretName, "127.0.0.1") + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + }) + + it("deletes and recreates the secret to match the IP in the load balancer without the extra hostnames", func() { + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + }) }) when("sync is called more than once", func() { @@ -834,7 +950,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -863,7 +979,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", hostname, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1", Hostname: hostname}}, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1036,8 +1152,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca = tlsSecret.Data["ca.crt"] r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) }) it("starts the impersonator with the existing tls certs, does not start loadbalancer or make tls secret", func() { @@ -1057,8 +1173,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { tlsSecret = createStubTLSSecret(tlsSecretName) // secret exists but lacks certs r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, "127.0.0.1", "", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) }) it("returns an error and leaves the impersonator running without tls certs", func() { From 5b01e4be2dfcb3b4eccf529ab8dfd1c583c9df8f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 26 Feb 2021 10:58:56 -0800 Subject: [PATCH 050/203] impersonator_config.go: handle more error cases Signed-off-by: Margo Crawford --- .../impersonatorconfig/impersonator_config.go | 68 +++-- .../impersonator_config_test.go | 235 ++++++++++++++++-- 2 files changed, 260 insertions(+), 43 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index abda84a4..c7b6d3e2 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -197,10 +197,8 @@ func (c *impersonatorConfigController) ensureTLSSecret(ctx controllerlib.Context if !notFound && err != nil { return err } - //nolint:staticcheck // TODO remove this nolint when we fix the TODO below - if secret, err = c.deleteTLSCertificateWithWrongName(ctx.Context, config, secret); err != nil { - // TODO - // return err + if secret, err = c.deleteWhenTLSCertificateDoesNotMatchDesiredState(ctx.Context, config, secret); err != nil { + return err } if err = c.ensureTLSSecretIsCreatedAndLoaded(ctx.Context, config, secret); err != nil { return err @@ -357,7 +355,7 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.C return nil } -func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx context.Context, config *impersonator.Config, secret *v1.Secret) (*v1.Secret, error) { +func (c *impersonatorConfigController) deleteWhenTLSCertificateDoesNotMatchDesiredState(ctx context.Context, config *impersonator.Config, secret *v1.Secret) (*v1.Secret, error) { if secret == nil { // There is no Secret, so there is nothing to delete. return secret, nil @@ -366,16 +364,46 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con certPEM := secret.Data[v1.TLSCertKey] block, _ := pem.Decode(certPEM) if block == nil { - // The certPEM is not valid. - return secret, nil // TODO delete this so the controller will create a valid one + plog.Warning("Found missing or not PEM-encoded data in TLS Secret", + "invalidCertPEM", certPEM, + "secret", c.tlsSecretName, + "namespace", c.namespace) + deleteErr := c.ensureTLSSecretIsRemoved(ctx) + if deleteErr != nil { + return nil, fmt.Errorf("found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: %w", deleteErr) + } + return nil, nil + } + + actualCertFromSecret, err := x509.ParseCertificate(block.Bytes) + if err != nil { + plog.Error("Found invalid PEM data in TLS Secret", err, + "invalidCertPEM", certPEM, + "secret", c.tlsSecretName, + "namespace", c.namespace) + deleteErr := c.ensureTLSSecretIsRemoved(ctx) + if deleteErr != nil { + return nil, fmt.Errorf("PEM data represented an invalid cert, but got error while deleting it: %w", deleteErr) + } + return nil, nil + } + + keyPEM := secret.Data[v1.TLSPrivateKeyKey] + _, err = tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + plog.Error("Found invalid private key PEM data in TLS Secret", err, + "secret", c.tlsSecretName, + "namespace", c.namespace) + deleteErr := c.ensureTLSSecretIsRemoved(ctx) + if deleteErr != nil { + return nil, fmt.Errorf("cert had an invalid private key, but got error while deleting it: %w", deleteErr) + } + return nil, nil } - actualCertFromSecret, _ := x509.ParseCertificate(block.Bytes) - // TODO handle err desiredIP, desiredHostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) - //nolint:staticcheck // TODO remove this nolint when we fix the TODO below if err != nil { - // TODO return err + return secret, err } if !nameIsReady { // We currently have a secret but we are waiting for a load balancer to be assigned an ingress, so @@ -398,8 +426,10 @@ func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx con "namespace", c.namespace) if certHostnameAndIPMatchDesiredState(desiredIP, actualIPs, desiredHostname, actualHostnames) { + // The cert already matches the desired state, so there is no need to delete/recreate it. return secret, nil } + err = c.ensureTLSSecretIsRemoved(ctx) if err != nil { return secret, err @@ -509,8 +539,10 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c return nil, fmt.Errorf("could not create impersonation cert: %w", err) } - certPEM, keyPEM, _ := certauthority.ToPEM(impersonationCert) - // TODO handle err + certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + if err != nil { + return nil, err + } newTLSSecret := &v1.Secret{ Type: v1.SecretTypeTLS, @@ -525,13 +557,17 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c v1.TLSCertKey: certPEM, }, } + plog.Info("Creating TLS certificates for impersonation proxy", "ips", ips, "hostnames", hostnames, "secret", c.tlsSecretName, "namespace", c.namespace) - _, _ = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) - // TODO handle error on create + _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) + if err != nil { + return nil, err + } + return newTLSSecret, nil } @@ -545,7 +581,7 @@ func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secre "secret", c.tlsSecretName, "namespace", c.namespace, ) - // TODO clear the secret if it was already set previously... c.setTLSCert(nil) + c.setTLSCert(nil) return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) } plog.Info("Loading TLS certificates for impersonation proxy", diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 20512aad..04678706 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -896,12 +896,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) when("there are not visible control plane nodes, a secret exists with multiple hostnames and an IP", func() { - var tlsSecret *corev1.Secret it.Before(func() { addNodeWithRoleToTracker("worker") addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) - tlsSecret = createTLSSecretWithMultipleHostnames(tlsSecretName, "127.0.0.1") + tlsSecret := createTLSSecretWithMultipleHostnames(tlsSecretName, "127.0.0.1") r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) startInformersAndController() @@ -916,6 +915,58 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) }) }) + + when("the cert's name needs to change but there is an error while deleting the tls Secret", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeAPIClient) + tlsSecret := createActualTLSSecret(tlsSecretName) + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("returns an error and runs the proxy without certs", func() { + startInformersAndController() + r.Error(controllerlib.TestSync(t, subject, *syncContext), "error on delete") + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) + requireTLSServerIsRunningWithoutCerts() + }) + }) + + when("the cert's name might need to change but there is an error while determining the new name", func() { + var ca []byte + it.Before(func() { + addNodeWithRoleToTracker("worker") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + tlsSecret := createActualTLSSecret(tlsSecretName) + ca = tlsSecret.Data["ca.crt"] + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + }) + + it("returns an error and keeps running the proxy with the old cert", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + + updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, "not-an-ip", "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), + "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") + r.Len(kubeAPIClient.Actions(), 1) // no new actions + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + }) }) when("sync is called more than once", func() { @@ -1165,28 +1216,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("a load balancer and a secret already exists but the tls secret is not valid", func() { - var tlsSecret *corev1.Secret - it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") - addNodeWithRoleToTracker("worker") - tlsSecret = createStubTLSSecret(tlsSecretName) // secret exists but lacks certs - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) - }) - - it("returns an error and leaves the impersonator running without tls certs", func() { - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), - "could not parse TLS cert PEM data from Secret: tls: failed to find any PEM data in certificate input") - r.Len(kubeAPIClient.Actions(), 1) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunningWithoutCerts() - }) - }) - when("we have a hostname specified for the endpoint", func() { const fakeHostname = "fake.example.com" it.Before(func() { @@ -1319,7 +1348,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("the endpoint switches from specified, to not specified, to specified", func() { + when("the endpoint switches from specified, to not specified, to specified again", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: enabled @@ -1373,6 +1402,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[4], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. updateImpersonatorConfigMapInTracker(configMapResourceName, here.Doc(` mode: enabled @@ -1381,9 +1414,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 7) + r.Len(kubeAPIClient.Actions(), 8) requireLoadBalancerDeleted(kubeAPIClient.Actions()[5]) - requireTLSSecretWasCreated(kubeAPIClient.Actions()[6]) // recreated because the endpoint was updated + requireTLSSecretDeleted(kubeAPIClient.Actions()[6]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[7]) // recreated because the endpoint was updated }) }) }) @@ -1402,6 +1436,25 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) + when("there is an error creating the tls secret", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}") + addNodeWithRoleToTracker("control-plane") + kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on create") + }) + }) + + it("starts the impersonator without certs and returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "error on create") + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + }) + }) + when("there is an error deleting the tls secret", func() { it.Before(func() { addNodeWithRoleToTracker("control-plane") @@ -1425,5 +1478,133 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) }) }) + + when("the PEM formatted data in the Secret is not a valid cert", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}") + addNodeWithRoleToTracker("worker") + tlsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: tlsSecretName, + Namespace: installedInNamespace, + }, + Data: map[string][]byte{ + // "aGVsbG8gd29ybGQK" is "hello world" base64 encoded + corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"), + }, + } + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + }) + + it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + + when("there is an error while the invalid cert is being deleted", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "PEM data represented an invalid cert, but got error while deleting it: error on delete") + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSServerIsRunningWithoutCerts() + }) + }) + }) + + when("a tls secret already exists but it is not valid", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + tlsSecret := createStubTLSSecret(tlsSecretName) // secret exists but lacks certs + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + }) + + it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + + when("there is an error while the invalid cert is being deleted", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete") + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSServerIsRunningWithoutCerts() + }) + }) + }) + + when("a tls secret already exists but the private key is not valid", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + tlsSecret := createActualTLSSecret(tlsSecretName) + tlsSecret.Data["tls.key"] = nil + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + }) + + it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { + startInformersAndController() + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + + when("there is an error while the invalid cert is being deleted", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on delete") + }) + }) + + it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "cert had an invalid private key, but got error while deleting it: error on delete") + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSServerIsRunningWithoutCerts() + }) + }) + }) }, spec.Parallel(), spec.Report(report.Terminal{})) } From 9bd206cedb9d3dceee88702e30922956e7fb9ff9 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 26 Feb 2021 11:27:19 -0800 Subject: [PATCH 051/203] impersonator_config_test.go: small refactor of test helpers Signed-off-by: Ryan Richard --- .../impersonator_config_test.go | 164 +++++++----------- 1 file changed, 67 insertions(+), 97 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 04678706..27e8f6ea 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -33,7 +33,6 @@ import ( "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/testutil" ) @@ -263,6 +262,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { const configMapResourceName = "some-configmap-resource-name" const generatedLoadBalancerServiceName = "some-service-resource-name" const tlsSecretName = "some-secret-name" + const localhostIP = "127.0.0.1" var labels = map[string]string{"app": "app-name", "other-key": "other-value"} var r *require.Assertions @@ -440,7 +440,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { controllerlib.TestRunSynchronously(t, subject) } - var addImpersonatorConfigMapToTracker = func(resourceName, configYAML string) { + var addImpersonatorConfigMapToTracker = func(resourceName, configYAML string, client *kubernetesfake.Clientset) { impersonatorConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -454,10 +454,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { "config.yaml": configYAML, }, } - r.NoError(kubeInformerClient.Tracker().Add(impersonatorConfigMap)) + r.NoError(client.Tracker().Add(impersonatorConfigMap)) } - var updateImpersonatorConfigMapInTracker = func(resourceName, configYAML, newResourceVersion string) { + var updateImpersonatorConfigMapInTracker = func(resourceName, configYAML string, client *kubernetesfake.Clientset, newResourceVersion string) { impersonatorConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -470,14 +470,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { "config.yaml": configYAML, }, } - r.NoError(kubeInformerClient.Tracker().Update( + r.NoError(client.Tracker().Update( schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, impersonatorConfigMap, installedInNamespace, )) } - var createStubTLSSecret = func(resourceName string) *corev1.Secret { + var secretWithData = func(resourceName string, data map[string][]byte) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -487,33 +487,26 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // that the initial version is version 0. ResourceVersion: "0", }, - Data: map[string][]byte{}, + Data: data, } } - var createActualTLSSecret = func(resourceName string) *corev1.Secret { + var createStubTLSSecret = func(resourceName string) *corev1.Secret { + return secretWithData(resourceName, map[string][]byte{}) + } + + var createActualTLSSecret = func(resourceName string, ip string) *corev1.Secret { impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) r.NoError(err) - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, []net.IP{net.ParseIP("127.0.0.1")}, 24*time.Hour) + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, []net.IP{net.ParseIP(ip)}, 24*time.Hour) r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) r.NoError(err) - - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", - }, - Data: map[string][]byte{ - "ca.crt": impersonationCA.Bundle(), - corev1.TLSPrivateKeyKey: keyPEM, - corev1.TLSCertKey: certPEM, - }, - } + return secretWithData(resourceName, map[string][]byte{ + "ca.crt": impersonationCA.Bundle(), + corev1.TLSPrivateKeyKey: keyPEM, + corev1.TLSCertKey: certPEM, + }) } var createTLSSecretWithMultipleHostnames = func(resourceName string, ip string) *corev1.Secret { @@ -523,22 +516,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) r.NoError(err) - - return &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", - }, - Data: map[string][]byte{ - "ca.crt": impersonationCA.Bundle(), - corev1.TLSPrivateKeyKey: keyPEM, - corev1.TLSCertKey: certPEM, - }, - } + return secretWithData(resourceName, map[string][]byte{ + "ca.crt": impersonationCA.Bundle(), + corev1.TLSPrivateKeyKey: keyPEM, + corev1.TLSCertKey: certPEM, + }) } var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { @@ -558,7 +540,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(client.Tracker().Add(loadBalancerService)) } - var updateLoadBalancerServiceInTracker = func(resourceName, lbIngressIP, newResourceVersion string) { + var updateLoadBalancerServiceInTracker = func(resourceName, lbIngressIP string, client *kubernetesfake.Clientset, newResourceVersion string) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -576,7 +558,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }, }, } - r.NoError(kubeInformerClient.Tracker().Update( + r.NoError(client.Tracker().Update( schema.GroupVersionResource{Version: "v1", Resource: "services"}, loadBalancerService, installedInNamespace, @@ -708,7 +690,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the ConfigMap does not yet exist in the installation namespace or it was deleted (defaults to auto mode)", func() { it.Before(func() { - addImpersonatorConfigMapToTracker("some-other-ConfigMap", "foo: bar") + addImpersonatorConfigMapToTracker("some-other-ConfigMap", "foo: bar", kubeInformerClient) }) when("there are visible control plane nodes", func() { @@ -898,9 +880,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes, a secret exists with multiple hostnames and an IP", func() { it.Before(func() { addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) - tlsSecret := createTLSSecretWithMultipleHostnames(tlsSecretName, "127.0.0.1") + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + tlsSecret := createTLSSecretWithMultipleHostnames(tlsSecretName, localhostIP) r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) startInformersAndController() @@ -921,7 +903,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("worker") addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeAPIClient) - tlsSecret := createActualTLSSecret(tlsSecretName) + tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { @@ -943,9 +925,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var ca []byte it.Before(func() { addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) - tlsSecret := createActualTLSSecret(tlsSecretName) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) ca = tlsSecret.Data["ca.crt"] r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) @@ -958,7 +940,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) - updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, "not-an-ip", "1") + updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, "not-an-ip", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") r.EqualError(controllerlib.TestSync(t, subject, *syncContext), @@ -1001,7 +983,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1030,7 +1012,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1", Hostname: hostname}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1073,7 +1055,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the configmap is invalid", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "not yaml") + addImpersonatorConfigMapToTracker(configMapResourceName, "not yaml", kubeInformerClient) }) it("returns an error", func() { @@ -1086,11 +1068,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the ConfigMap is already in the installation namespace", func() { when("the configuration is auto mode with an endpoint", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` - mode: auto - endpoint: 127.0.0.1 - `), - ) + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: auto, endpoint: 127.0.0.1}", kubeInformerClient) }) when("there are visible control plane nodes", func() { @@ -1125,7 +1103,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the configuration is disabled mode", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: disabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: disabled", kubeInformerClient) addNodeWithRoleToTracker("worker") }) @@ -1141,7 +1119,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the configuration is enabled mode", func() { when("no load balancer", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("control-plane") }) @@ -1168,7 +1146,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a loadbalancer already exists", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker") addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) @@ -1197,14 +1175,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a load balancer and a secret already exists", func() { var ca []byte it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker") - tlsSecret := createActualTLSSecret(tlsSecretName) + tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) ca = tlsSecret.Data["ca.crt"] r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) it("starts the impersonator with the existing tls certs, does not start loadbalancer or make tls secret", func() { @@ -1219,10 +1197,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("we have a hostname specified for the endpoint", func() { const fakeHostname = "fake.example.com" it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, here.Docf(` - mode: enabled - endpoint: %s - `, fakeHostname)) + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) addNodeWithRoleToTracker("worker") }) @@ -1243,7 +1219,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var hostnameYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) var ipAddressYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIP) it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, ipAddressYAML) + addImpersonatorConfigMapToTracker(configMapResourceName, ipAddressYAML, kubeInformerClient) addNodeWithRoleToTracker("worker") }) @@ -1261,7 +1237,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") // Switch the endpoint config to a hostname. - updateImpersonatorConfigMapInTracker(configMapResourceName, hostnameYAML, "1") + updateImpersonatorConfigMapInTracker(configMapResourceName, hostnameYAML, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1277,7 +1253,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // Switch the endpoint config back to an IP. - updateImpersonatorConfigMapInTracker(configMapResourceName, ipAddressYAML, "2") + updateImpersonatorConfigMapInTracker(configMapResourceName, ipAddressYAML, kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1292,7 +1268,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the configuration switches from enabled to disabled mode", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker") }) @@ -1309,7 +1285,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1320,7 +1296,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { deleteLoadBalancerServiceFromTracker(generatedLoadBalancerServiceName, kubeInformerClient) waitForLoadBalancerToBeDeleted(kubeInformers.Core().V1().Services(), generatedLoadBalancerServiceName) - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "2") + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1339,7 +1315,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) requireTLSServerIsRunningWithoutCerts() - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", "1") + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "fake server close error") @@ -1350,10 +1326,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the endpoint switches from specified, to not specified, to specified again", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` - mode: enabled - endpoint: 127.0.0.1 - `)) + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}", kubeInformerClient) addNodeWithRoleToTracker("worker") }) @@ -1372,7 +1345,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") // Switch to "enabled" mode without an "endpoint", so a load balancer is needed now. - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "1") + updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1394,7 +1367,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Update the ingress of the LB in the informer's client and run Sync again. fakeIP := "127.0.0.123" - updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, fakeIP, "1") + updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, fakeIP, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Len(kubeAPIClient.Actions(), 5) @@ -1407,10 +1380,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. - updateImpersonatorConfigMapInTracker(configMapResourceName, here.Doc(` - mode: enabled - endpoint: 127.0.0.1 - `), "2") + updateImpersonatorConfigMapInTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}", kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) @@ -1438,7 +1408,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there is an error creating the tls secret", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}") + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) addNodeWithRoleToTracker("control-plane") kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on create") @@ -1481,7 +1451,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the PEM formatted data in the Secret is not a valid cert", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}") + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}", kubeInformerClient) addNodeWithRoleToTracker("worker") tlsSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -1528,13 +1498,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a tls secret already exists but it is not valid", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker") tlsSecret := createStubTLSSecret(tlsSecretName) // secret exists but lacks certs r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { @@ -1568,14 +1538,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a tls secret already exists but the private key is not valid", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker") - tlsSecret := createActualTLSSecret(tlsSecretName) + tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) tlsSecret.Data["tls.key"] = nil r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.1"}}, kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { From fa49beb6238257c51fd98b90c0adcca2f7653d13 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 26 Feb 2021 12:05:17 -0800 Subject: [PATCH 052/203] Change length of TLS certs and CA. Signed-off-by: Ryan Richard --- .../impersonatorconfig/impersonator_config.go | 4 ++-- .../impersonatorconfig/impersonator_config_test.go | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index c7b6d3e2..b4eba710 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -448,7 +448,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con // TODO create/save/watch the CA separately so we can reuse it to mint tls certs as the settings are dynamically changed, // so that clients don't need to be updated to use a different CA just because the server-side settings were changed. - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) // TODO change the expiration of this to 100 years + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "Pinniped Impersonation Proxy CA"}, 100*365*24*time.Hour) if err != nil { return fmt.Errorf("could not create impersonation CA: %w", err) } @@ -534,7 +534,7 @@ func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() } func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP, hostnames []string) (*v1.Secret, error) { - impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, 24*time.Hour) // TODO change the length of this too 100 years for now? + impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, 100*365*24*time.Hour) if err != nil { return nil, fmt.Errorf("could not create impersonation cert: %w", err) } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 27e8f6ea..a246bbca 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "io/ioutil" @@ -668,6 +669,17 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NotNil(createdSecret.Data["ca.crt"]) r.NotNil(createdSecret.Data[corev1.TLSPrivateKeyKey]) r.NotNil(createdSecret.Data[corev1.TLSCertKey]) + validCert := testutil.ValidateCertificate(t, string(createdSecret.Data["ca.crt"]), string(createdSecret.Data[corev1.TLSCertKey])) + validCert.RequireMatchesPrivateKey(string(createdSecret.Data[corev1.TLSPrivateKeyKey])) + validCert.RequireLifetime(time.Now().Add(-10*time.Second), time.Now().Add(100*time.Hour*24*365), 10*time.Second) + // Make sure the CA certificate looks roughly like what we expect. + block, _ := pem.Decode(createdSecret.Data["ca.crt"]) + require.NotNil(t, block) + caCert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + require.Equal(t, "Pinniped Impersonation Proxy CA", caCert.Subject.CommonName) + require.WithinDuration(t, time.Now().Add(-10*time.Second), caCert.NotBefore, 10*time.Second) + require.WithinDuration(t, time.Now().Add(100*time.Hour*24*365), caCert.NotAfter, 10*time.Second) return createdSecret.Data["ca.crt"] } From 41e4a74b57e51eea98457604252afa974546f01b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 26 Feb 2021 13:53:30 -0800 Subject: [PATCH 053/203] impersonator_config_test.go: more small refactoring of test helpers --- .../impersonator_config_test.go | 632 +++++++++--------- 1 file changed, 310 insertions(+), 322 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index a246bbca..bc7e4975 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -261,9 +261,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" - const generatedLoadBalancerServiceName = "some-service-resource-name" + const loadBalancerServiceName = "some-service-resource-name" const tlsSecretName = "some-secret-name" const localhostIP = "127.0.0.1" + const httpsPort = ":443" var labels = map[string]string{"app": "app-name", "other-key": "other-value"} var r *require.Assertions @@ -290,11 +291,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return nil, startTLSListenerFuncError } var err error - startedTLSListener, err = tls.Listen(network, "127.0.0.1:0", config) // automatically choose the port for unit tests + startedTLSListener, err = tls.Listen(network, localhostIP+":0", config) // automatically choose the port for unit tests r.NoError(err) return &tlsListenerWrapper{listener: startedTLSListener, closeError: startTLSListenerUponCloseError}, nil } + var testServerAddr = func() string { + return startedTLSListener.Addr().String() + } + var closeTLSListener = func() { if startedTLSListener != nil { err := startedTLSListener.Close() @@ -352,13 +357,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var requireTLSServerIsRunningWithoutCerts = func() { r.Greater(startTLSListenerFuncWasCalled, 0) - tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec } - client := &http.Client{Transport: tr} - url := "https://" + startedTLSListener.Addr().String() + url := "https://" + testServerAddr() req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) _, err = client.Do(req) //nolint:bodyclose @@ -370,7 +373,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Greater(startTLSListenerFuncWasCalled, 0) _, err := tls.Dial( startedTLSListener.Addr().Network(), - startedTLSListener.Addr().String(), + testServerAddr(), &tls.Config{InsecureSkipVerify: true}, //nolint:gosec ) r.Error(err) @@ -414,7 +417,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { kubeInformers.Core().V1().Secrets(), controllerlib.WithInformer, controllerlib.WithInitialEvent, - generatedLoadBalancerServiceName, + loadBalancerServiceName, tlsSecretName, labels, startTLSListenerFunc, @@ -459,26 +462,25 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var updateImpersonatorConfigMapInTracker = func(resourceName, configYAML string, client *kubernetesfake.Clientset, newResourceVersion string) { - impersonatorConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: installedInNamespace, - // Different resource version compared to the initial version when this resource was created - // so we can tell when the informer cache has cached this newly updated version. - ResourceVersion: newResourceVersion, - }, - Data: map[string]string{ - "config.yaml": configYAML, - }, + configMapObj, err := client.Tracker().Get( + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + installedInNamespace, + resourceName, + ) + r.NoError(err) + configMap := configMapObj.(*corev1.ConfigMap) + configMap.ResourceVersion = newResourceVersion + configMap.Data = map[string]string{ + "config.yaml": configYAML, } r.NoError(client.Tracker().Update( schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, - impersonatorConfigMap, + configMap, installedInNamespace, )) } - var secretWithData = func(resourceName string, data map[string][]byte) *corev1.Secret { + var newSecretWithData = func(resourceName string, data map[string][]byte) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -492,102 +494,94 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var createStubTLSSecret = func(resourceName string) *corev1.Secret { - return secretWithData(resourceName, map[string][]byte{}) + var newStubTLSSecret = func(resourceName string) *corev1.Secret { + return newSecretWithData(resourceName, map[string][]byte{}) } - var createActualTLSSecret = func(resourceName string, ip string) *corev1.Secret { + var createCertSecretData = func(dnsNames []string, ip string) map[string][]byte { impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) r.NoError(err) - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, []net.IP{net.ParseIP(ip)}, 24*time.Hour) + impersonationCert, err := impersonationCA.Issue(pkix.Name{}, dnsNames, []net.IP{net.ParseIP(ip)}, 24*time.Hour) r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) r.NoError(err) - return secretWithData(resourceName, map[string][]byte{ + return map[string][]byte{ "ca.crt": impersonationCA.Bundle(), corev1.TLSPrivateKeyKey: keyPEM, corev1.TLSCertKey: certPEM, - }) + } } - var createTLSSecretWithMultipleHostnames = func(resourceName string, ip string) *corev1.Secret { - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) - r.NoError(err) - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, []string{"foo", "bar"}, []net.IP{net.ParseIP(ip)}, 24*time.Hour) - r.NoError(err) - certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) - r.NoError(err) - return secretWithData(resourceName, map[string][]byte{ - "ca.crt": impersonationCA.Bundle(), - corev1.TLSPrivateKeyKey: keyPEM, - corev1.TLSCertKey: certPEM, - }) + var newActualTLSSecret = func(resourceName string, ip string) *corev1.Secret { + return newSecretWithData(resourceName, createCertSecretData(nil, ip)) + } + + var newActualTLSSecretWithMultipleHostnames = func(resourceName string, ip string) *corev1.Secret { + return newSecretWithData(resourceName, createCertSecretData([]string{"foo", "bar"}, ip)) + } + + var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + createdSecret.ResourceVersion = resourceVersion + r.NoError(client.Tracker().Add(createdSecret)) + } + + var addServiceFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { + createdService := action.(coretesting.CreateAction).GetObject().(*corev1.Service) + createdService.ResourceVersion = resourceVersion + r.NoError(client.Tracker().Add(createdService)) + } + + var newLoadBalancerService = func(resourceName string, status corev1.ServiceStatus) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + // Note that this seems to be ignored by the informer during initial creation, so actually + // the informer will see this as resource version "". Leaving it here to express the intent + // that the initial version is version 0. + ResourceVersion: "0", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: status, + } } var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { - loadBalancerService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - }, - } + loadBalancerService := newLoadBalancerService(resourceName, corev1.ServiceStatus{}) r.NoError(client.Tracker().Add(loadBalancerService)) } - var updateLoadBalancerServiceInTracker = func(resourceName, lbIngressIP string, client *kubernetesfake.Clientset, newResourceVersion string) { - loadBalancerService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: installedInNamespace, - ResourceVersion: newResourceVersion, - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - }, - Status: corev1.ServiceStatus{ - LoadBalancer: corev1.LoadBalancerStatus{ - Ingress: []corev1.LoadBalancerIngress{ - {IP: lbIngressIP}, - }, - }, - }, - } - r.NoError(client.Tracker().Update( - schema.GroupVersionResource{Version: "v1", Resource: "services"}, - loadBalancerService, - installedInNamespace, - )) - } - var addLoadBalancerServiceWithIngressToTracker = func(resourceName string, ingress []corev1.LoadBalancerIngress, client *kubernetesfake.Clientset) { - loadBalancerService := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: resourceName, - Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", - }, - Spec: corev1.ServiceSpec{ - Type: corev1.ServiceTypeLoadBalancer, - }, - Status: corev1.ServiceStatus{ - LoadBalancer: corev1.LoadBalancerStatus{ - Ingress: ingress, - }, - }, - } + loadBalancerService := newLoadBalancerService(resourceName, corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingress}, + }) r.NoError(client.Tracker().Add(loadBalancerService)) } + var addSecretToTracker = func(secret *corev1.Secret, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Add(secret)) + } + + var updateLoadBalancerServiceInTracker = func(resourceName string, ingresses []corev1.LoadBalancerIngress, client *kubernetesfake.Clientset, newResourceVersion string) { + serviceObj, err := client.Tracker().Get( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + installedInNamespace, + resourceName, + ) + r.NoError(err) + service := serviceObj.(*corev1.Service) + service.ResourceVersion = newResourceVersion + service.Status = corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingresses}} + r.NoError(client.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + service, + installedInNamespace, + )) + } + var deleteLoadBalancerServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { r.NoError(client.Tracker().Delete( schema.GroupVersionResource{Version: "v1", Resource: "services"}, @@ -604,14 +598,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } - var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { - createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) - createdSecret.ResourceVersion = resourceVersion - r.NoError(client.Tracker().Add(createdSecret)) - } - - var addNodeWithRoleToTracker = func(role string) { - r.NoError(kubeAPIClient.Tracker().Add( + var addNodeWithRoleToTracker = func(role string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Add( &corev1.Node{ ObjectMeta: metav1.ObjectMeta{ Name: "node", @@ -636,7 +624,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { createAction := action.(coretesting.CreateAction) r.Equal("create", createAction.GetVerb()) createdLoadBalancerService := createAction.GetObject().(*corev1.Service) - r.Equal(generatedLoadBalancerServiceName, createdLoadBalancerService.Name) + r.Equal(loadBalancerServiceName, createdLoadBalancerService.Name) r.Equal(installedInNamespace, createdLoadBalancerService.Namespace) r.Equal(corev1.ServiceTypeLoadBalancer, createdLoadBalancerService.Spec.Type) r.Equal("app-name", createdLoadBalancerService.Spec.Selector["app"]) @@ -646,7 +634,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var requireLoadBalancerDeleted = func(action coretesting.Action) { deleteAction := action.(coretesting.DeleteAction) r.Equal("delete", deleteAction.GetVerb()) - r.Equal(generatedLoadBalancerServiceName, deleteAction.GetName()) + r.Equal(loadBalancerServiceName, deleteAction.GetName()) r.Equal("services", deleteAction.GetResource().Resource) } @@ -683,11 +671,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return createdSecret.Data["ca.crt"] } + var runControllerSync = func() error { + return controllerlib.TestSync(t, subject, *syncContext) + } + it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) - kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactoryWithOptions(kubeInformerClient, 0, kubeinformers.WithNamespace(installedInNamespace), @@ -702,17 +692,17 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the ConfigMap does not yet exist in the installation namespace or it was deleted (defaults to auto mode)", func() { it.Before(func() { - addImpersonatorConfigMapToTracker("some-other-ConfigMap", "foo: bar", kubeInformerClient) + addImpersonatorConfigMapToTracker("some-other-configmap", "foo: bar", kubeInformerClient) }) when("there are visible control plane nodes", func() { it.Before(func() { - addNodeWithRoleToTracker("control-plane") + addNodeWithRoleToTracker("control-plane", kubeAPIClient) }) it("does not start the impersonator or load balancer", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -721,17 +711,17 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are visible control plane nodes and a loadbalancer and a tls Secret", func() { it.Before(func() { - addNodeWithRoleToTracker("control-plane") - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) - tlsSecret := createStubTLSSecret(tlsSecretName) - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) + tlsSecret := newStubTLSSecret(tlsSecretName) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) }) it("does not start the impersonator, deletes the loadbalancer, deletes the Secret", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -742,16 +732,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - }) - - it("starts the impersonator without tls certs", func() { - requireTLSServerIsRunningWithoutCerts() + r.NoError(runControllerSync()) }) it("starts the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) @@ -760,18 +747,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists without an IP", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - }) - - it("starts the impersonator without tls certs", func() { - requireTLSServerIsRunningWithoutCerts() + r.NoError(runControllerSync()) }) it("does not start the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) @@ -779,18 +763,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with empty ingress", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "", Hostname: ""}}, kubeAPIClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - }) - - it("starts the impersonator without tls certs", func() { - requireTLSServerIsRunningWithoutCerts() + r.NoError(runControllerSync()) }) it("does not start the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) @@ -798,18 +779,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with invalid ip", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeAPIClient) startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") - }) - - it("starts the impersonator without tls certs", func() { - requireTLSServerIsRunningWithoutCerts() + r.EqualError(runControllerSync(), "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") }) it("does not start the load balancer automatically", func() { + requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) @@ -817,24 +795,24 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with multiple ips", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeAPIClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) }) it("starts the impersonator with certs that match the first IP address", func() { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunning(ca, "127.0.0.123", map[string]string{"127.0.0.123:443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, "127.0.0.123", map[string]string{"127.0.0.123:443": testServerAddr()}) }) it("keeps the secret around after resync", func() { addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) // nothing changed }) }) @@ -842,24 +820,24 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with multiple hostnames", func() { firstHostname := "fake-1.example.com" it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{Hostname: firstHostname}, {Hostname: "fake-2.example.com"}}, kubeAPIClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) }) it("starts the impersonator with certs that match the first hostname", func() { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) }) it("keeps the secret around after resync", func() { addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) // nothing changed }) }) @@ -867,38 +845,38 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes and a load balancer already exists with hostnames and ips", func() { firstHostname := "fake-1.example.com" it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.254"}, {Hostname: firstHostname}}, kubeAPIClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) }) it("starts the impersonator with certs that match the first hostname", func() { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) }) it("keeps the secret around after resync", func() { addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) // nothing changed }) }) when("there are not visible control plane nodes, a secret exists with multiple hostnames and an IP", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) - tlsSecret := createTLSSecretWithMultipleHostnames(tlsSecretName, localhostIP) - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + tlsSecret := newActualTLSSecretWithMultipleHostnames(tlsSecretName, localhostIP) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) }) it("deletes and recreates the secret to match the IP in the load balancer without the extra hostnames", func() { @@ -906,18 +884,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) }) when("the cert's name needs to change but there is an error while deleting the tls Secret", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeAPIClient) - tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeAPIClient) + tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on delete") }) @@ -925,7 +903,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns an error and runs the proxy without certs", func() { startInformersAndController() - r.Error(controllerlib.TestSync(t, subject, *syncContext), "error on delete") + r.Error(runControllerSync(), "error on delete") r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) @@ -936,51 +914,51 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the cert's name might need to change but there is an error while determining the new name", func() { var ca []byte it.Before(func() { - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) - tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) ca = tlsSecret.Data["ca.crt"] - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) }) it("returns an error and keeps running the proxy with the old cert", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) - updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, "not-an-ip", kubeInformerClient, "1") + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), + r.EqualError(runControllerSync(), "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") r.Len(kubeAPIClient.Actions(), 1) // no new actions - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) }) }) when("sync is called more than once", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("only starts the impersonator once and only lists the cluster's nodes once", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() - // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time requireTLSServerIsRunningWithoutCerts() // still running r.Len(kubeAPIClient.Actions(), 2) // no new API calls @@ -988,79 +966,85 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("creates certs from the ip address listed on the load balancer", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() - // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + + r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time r.Len(kubeAPIClient.Actions(), 3) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // running with certs now + requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now - // update manually because the kubeAPIClient isn't connected to the informer in the tests + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time - r.Len(kubeAPIClient.Actions(), 3) // no more actions - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // still running + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 3) // no more actions + requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running }) it("creates certs from the hostname listed on the load balancer", func() { hostname := "fake.example.com" startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() - // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient) + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + + r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time r.Len(kubeAPIClient.Actions(), 3) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + ":443": startedTLSListener.Addr().String()}) // running with certs now + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now - // update manually because the kubeAPIClient isn't connected to the informer in the tests + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time - r.Len(kubeAPIClient.Actions(), 3) // no more actions - requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + ":443": startedTLSListener.Addr().String()}) // still running + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 3) // no more actions + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running }) }) when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { it("returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "no nodes found") + r.EqualError(runControllerSync(), "no nodes found") requireTLSServerWasNeverStarted() }) }) when("the http handler factory function returns an error", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) httpHanderFactoryFuncError = errors.New("some factory error") }) it("returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "some factory error") + r.EqualError(runControllerSync(), "some factory error") requireTLSServerWasNeverStarted() }) }) @@ -1072,7 +1056,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config") + r.EqualError(runControllerSync(), "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config") requireTLSServerWasNeverStarted() }) }) @@ -1080,17 +1064,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the ConfigMap is already in the installation namespace", func() { when("the configuration is auto mode with an endpoint", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: auto, endpoint: 127.0.0.1}", kubeInformerClient) + configMapYAML := fmt.Sprintf("{mode: auto, endpoint: %s}", localhostIP) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) }) when("there are visible control plane nodes", func() { it.Before(func() { - addNodeWithRoleToTracker("control-plane") + addNodeWithRoleToTracker("control-plane", kubeAPIClient) }) it("does not start the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerWasNeverStarted() requireNodesListed(kubeAPIClient.Actions()[0]) r.Len(kubeAPIClient.Actions(), 1) @@ -1099,16 +1084,16 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there are not visible control plane nodes", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("starts the impersonator according to the settings in the ConfigMap", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) }) }) @@ -1116,12 +1101,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the configuration is disabled mode", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: disabled", kubeInformerClient) - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("does not start the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerWasNeverStarted() requireNodesListed(kubeAPIClient.Actions()[0]) r.Len(kubeAPIClient.Actions(), 1) @@ -1132,24 +1117,24 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("no load balancer", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) - addNodeWithRoleToTracker("control-plane") + addNodeWithRoleToTracker("control-plane", kubeAPIClient) }) it("starts the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() }) it("returns an error when the tls listener fails to start", func() { startTLSListenerFuncError = errors.New("tls error") startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + r.EqualError(runControllerSync(), "tls error") }) it("starts the load balancer", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) @@ -1159,26 +1144,26 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a loadbalancer already exists", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) - addNodeWithRoleToTracker("worker") - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) }) it("starts the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() }) it("returns an error when the tls listener fails to start", func() { startTLSListenerFuncError = errors.New("tls error") startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "tls error") + r.EqualError(runControllerSync(), "tls error") }) it("does not start the load balancer", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) }) @@ -1188,21 +1173,21 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var ca []byte it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) - addNodeWithRoleToTracker("worker") - tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) + addNodeWithRoleToTracker("worker", kubeAPIClient) + tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) ca = tlsSecret.Data["ca.crt"] - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) it("starts the impersonator with the existing tls certs, does not start loadbalancer or make tls secret", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) }) @@ -1211,17 +1196,17 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it.Before(func() { configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("starts the impersonator, generates a valid cert for the hostname", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. - requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) }) }) @@ -1232,19 +1217,19 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var ipAddressYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIP) it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, ipAddressYAML, kubeInformerClient) - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("regenerates the cert for the hostname, then regenerates it for the IP again", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) // Check that the server is running and that TLS certs that are being served are are for fakeIP. - requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) - // update manually because the kubeAPIClient isn't connected to the informer in the tests + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") @@ -1252,14 +1237,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateImpersonatorConfigMapInTracker(configMapResourceName, hostnameYAML, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[3]) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. - requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - // update manually because the kubeAPIClient isn't connected to the informer in the tests + // Simulate the informer cache's background update from its watch. deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") @@ -1268,12 +1253,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateImpersonatorConfigMapInTracker(configMapResourceName, ipAddressYAML, kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 6) requireTLSSecretDeleted(kubeAPIClient.Actions()[4]) ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[5]) // Check that the server is running and that TLS certs that are being served are are for fakeIP. - requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) }) }) }) @@ -1281,37 +1266,37 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the configuration switches from enabled to disabled mode", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("starts the impersonator and loadbalancer, then shuts it down, then starts it again", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) - // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerIsNoLongerRunning() r.Len(kubeAPIClient.Actions(), 3) requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) - deleteLoadBalancerServiceFromTracker(generatedLoadBalancerServiceName, kubeInformerClient) - waitForLoadBalancerToBeDeleted(kubeInformers.Core().V1().Services(), generatedLoadBalancerServiceName) + deleteLoadBalancerServiceFromTracker(loadBalancerServiceName, kubeInformerClient) + waitForLoadBalancerToBeDeleted(kubeInformers.Core().V1().Services(), loadBalancerServiceName) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 4) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) @@ -1324,13 +1309,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns the error from the sync function", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "fake server close error") + r.EqualError(runControllerSync(), "fake server close error") requireTLSServerIsNoLongerRunning() }) }) @@ -1338,21 +1323,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the endpoint switches from specified, to not specified, to specified again", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}", kubeInformerClient) - addNodeWithRoleToTracker("worker") + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) }) it("doesn't create, then creates, then deletes the load balancer", func() { startInformersAndController() // Should have started in "enabled" mode with an "endpoint", so no load balancer is needed. - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) - // update manually because the kubeAPIClient isn't connected to the informer in the tests + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") @@ -1360,42 +1346,43 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 4) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[2]) requireTLSSecretDeleted(kubeAPIClient.Actions()[3]) // the Secret was deleted because it contained a cert with the wrong IP requireTLSServerIsRunningWithoutCerts() - // update manually because the kubeAPIClient isn't connected to the informer in the tests - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) waitForTLSCertSecretToBeDeleted(kubeInformers.Core().V1().Secrets(), tlsSecretName) // The controller should be waiting for the load balancer's ingress to become available. - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 4) // no new actions while it is waiting for the load balancer's ingress requireTLSServerIsRunningWithoutCerts() // Update the ingress of the LB in the informer's client and run Sync again. fakeIP := "127.0.0.123" - updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, fakeIP, kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}}, kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 5) ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[4]) // created because the LB ingress became available // Check that the server is running and that TLS certs that are being served are are for fakeIP. - requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) - // update manually because the kubeAPIClient isn't connected to the informer in the tests + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[4], kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. - updateImpersonatorConfigMapInTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}", kubeInformerClient, "2") + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) + updateImpersonatorConfigMapInTracker(configMapResourceName, configMapYAML, kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 8) requireLoadBalancerDeleted(kubeAPIClient.Actions()[5]) requireTLSSecretDeleted(kubeAPIClient.Actions()[6]) @@ -1406,7 +1393,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there is an error creating the load balancer", func() { it.Before(func() { - addNodeWithRoleToTracker("worker") + addNodeWithRoleToTracker("worker", kubeAPIClient) startInformersAndController() kubeAPIClient.PrependReactor("create", "services", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on create") @@ -1414,14 +1401,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("exits with an error", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "could not create load balancer: error on create") + r.EqualError(runControllerSync(), "could not create load balancer: error on create") }) }) when("there is an error creating the tls secret", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) - addNodeWithRoleToTracker("control-plane") + addNodeWithRoleToTracker("control-plane", kubeAPIClient) kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on create") }) @@ -1429,7 +1416,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator without certs and returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "error on create") + r.EqualError(runControllerSync(), "error on create") requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1439,12 +1426,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there is an error deleting the tls secret", func() { it.Before(func() { - addNodeWithRoleToTracker("control-plane") - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) - addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeAPIClient) - tlsSecret := createStubTLSSecret(tlsSecretName) - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) + tlsSecret := newStubTLSSecret(tlsSecretName) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) startInformersAndController() kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on delete") @@ -1452,7 +1439,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("does not start the impersonator, deletes the loadbalancer, returns an error", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "error on delete") + r.EqualError(runControllerSync(), "error on delete") requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1463,8 +1450,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the PEM formatted data in the Secret is not a valid cert", func() { it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: 127.0.0.1}", kubeInformerClient) - addNodeWithRoleToTracker("worker") + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) tlsSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: tlsSecretName, @@ -1475,18 +1463,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"), }, } - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) }) it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1498,7 +1486,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "PEM data represented an invalid cert, but got error while deleting it: error on delete") + r.EqualError(runControllerSync(), "PEM data represented an invalid cert, but got error while deleting it: error on delete") requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1511,22 +1499,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a tls secret already exists but it is not valid", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) - addNodeWithRoleToTracker("worker") - tlsSecret := createStubTLSSecret(tlsSecretName) // secret exists but lacks certs - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + tlsSecret := newStubTLSSecret(tlsSecretName) // secret exists but lacks certs + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1538,7 +1526,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete") + r.EqualError(runControllerSync(), "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete") requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1551,23 +1539,23 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a tls secret already exists but the private key is not valid", func() { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) - addNodeWithRoleToTracker("worker") - tlsSecret := createActualTLSSecret(tlsSecretName, localhostIP) + addNodeWithRoleToTracker("worker", kubeAPIClient) + tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) tlsSecret.Data["tls.key"] = nil - r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) - r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(generatedLoadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeAPIClient) + addSecretToTracker(tlsSecret, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + requireTLSServerIsRunning(ca, testServerAddr(), nil) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1579,7 +1567,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), "cert had an invalid private key, but got error while deleting it: error on delete") + r.EqualError(runControllerSync(), "cert had an invalid private key, but got error while deleting it: error on delete") requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) From f1eeae8c713758070137ef1f6c0fa88d94e44a5a Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 26 Feb 2021 15:01:38 -0800 Subject: [PATCH 054/203] Parse out ports from impersonation proxy endpoint config Signed-off-by: Margo Crawford --- .../impersonatorconfig/impersonator_config.go | 7 ++-- .../impersonator_config_test.go | 38 +++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index b4eba710..4e8468d1 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -13,6 +13,7 @@ import ( "fmt" "net" "net/http" + "strings" "sync" "time" @@ -491,12 +492,12 @@ func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *imp } func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) (net.IP, string, bool, error) { - // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose - parsedAsIP := net.ParseIP(config.Endpoint) + endpointWithoutPort := strings.Split(config.Endpoint, ":")[0] + parsedAsIP := net.ParseIP(endpointWithoutPort) if parsedAsIP != nil { return parsedAsIP, "", true, nil } - return nil, config.Endpoint, true, nil + return nil, endpointWithoutPort, true, nil } func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() (net.IP, string, bool, error) { diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index bc7e4975..7570e2f5 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -1210,6 +1210,44 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) + when("endpoint is IP address with a port", func() { + const fakeIpWithPort = "127.0.0.1:3000" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIpWithPort) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator, generates a valid cert for the hostname", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + // Check that the server is running and that TLS certs that are being served are are for fakeIpWithPort. + requireTLSServerIsRunning(ca, fakeIpWithPort, map[string]string{fakeIpWithPort: testServerAddr()}) + }) + }) + + when("endpoint is hostname with a port", func() { + const fakeHostnameWithPort = "fake.example.com:3000" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostnameWithPort) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("starts the impersonator, generates a valid cert for the hostname", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + // Check that the server is running and that TLS certs that are being served are are for fakeHostnameWithPort. + requireTLSServerIsRunning(ca, fakeHostnameWithPort, map[string]string{fakeHostnameWithPort: testServerAddr()}) + }) + }) + when("switching from ip address endpoint to hostname endpoint and back to ip address", func() { const fakeHostname = "fake.example.com" const fakeIP = "127.0.0.42" From a2ecd052400c42731097459413aeb2d92f08d527 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 1 Mar 2021 17:02:08 -0800 Subject: [PATCH 055/203] Impersonator config controller writes CA cert & key to different Secret - The CA cert will end up in the end user's kubeconfig on their client machine, so if it changes they would need to fetch the new one and update their kubeconfig. Therefore, we should avoid changing it as much as possible. - Now the controller writes the CA to a different Secret. It writes both the cert and the key so it can reuse them to create more TLS certificates in the future. - For now, it only needs to make more TLS certificates if the old TLS cert Secret gets deleted or updated to be invalid. This allows for manual rotation of the TLS certs by simply deleting the Secret. In the future, we may want to implement some kind of auto rotation. - For now, rotation of both the CA and TLS certs will also happen if you manually delete the CA Secret. However, this would cause the end users to immediately need to get the new CA into their kubeconfig, so this is not as elegant as a normal rotation flow where you would have a window of time where you have more than one CA. --- internal/certauthority/certauthority.go | 27 +- internal/certauthority/certauthority_test.go | 46 +- .../impersonatorconfig/impersonator_config.go | 420 +++++---- .../impersonator_config_test.go | 874 ++++++++++++------ .../controllermanager/prepare_controllers.go | 1 + .../concierge_impersonation_proxy_test.go | 28 +- 6 files changed, 892 insertions(+), 504 deletions(-) diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index 87bdd784..64fa2ecb 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package certauthority implements a simple x509 certificate authority suitable for use in an aggregated API service. @@ -44,12 +44,17 @@ type env struct { // CA holds the state for a simple x509 certificate authority suitable for use in an aggregated API service. type CA struct { - // caCert is the DER-encoded certificate for the current CA. + // caCertBytes is the DER-encoded certificate for the current CA. caCertBytes []byte // signer is the private key for the current CA. signer crypto.Signer + // privateKey is the same private key represented by signer, but in a format which allows export. + // It is only set by New, not by Load, since Load can handle various types of PrivateKey but New + // only needs to create keys of type ecdsa.PrivateKey. + privateKey *ecdsa.PrivateKey + // env is our reference to the outside world (clocks and random number generation). env env } @@ -99,11 +104,11 @@ func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { } // Generate a new P256 keypair. - privateKey, err := ecdsa.GenerateKey(elliptic.P256(), env.keygenRNG) + ca.privateKey, err = ecdsa.GenerateKey(elliptic.P256(), env.keygenRNG) if err != nil { return nil, fmt.Errorf("could not generate CA private key: %w", err) } - ca.signer = privateKey + ca.signer = ca.privateKey // Make a CA certificate valid for some ttl and backdated by some amount. now := env.clock() @@ -123,7 +128,7 @@ func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { } // Self-sign the CA to get the DER certificate. - caCertBytes, err := x509.CreateCertificate(env.signingRNG, &caTemplate, &caTemplate, &privateKey.PublicKey, privateKey) + caCertBytes, err := x509.CreateCertificate(env.signingRNG, &caTemplate, &caTemplate, &ca.privateKey.PublicKey, ca.privateKey) if err != nil { return nil, fmt.Errorf("could not issue CA certificate: %w", err) } @@ -136,6 +141,18 @@ func (c *CA) Bundle() []byte { return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: c.caCertBytes}) } +// PrivateKeyToPEM returns the current CA private key in PEM format, if this CA was constructed by New. +func (c *CA) PrivateKeyToPEM() ([]byte, error) { + if c.privateKey == nil { + return nil, fmt.Errorf("no private key data (did you try to use this after Load?)") + } + derKey, err := x509.MarshalECPrivateKey(c.privateKey) + if err != nil { + return nil, err + } + return pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: derKey}), nil +} + // Pool returns the current CA signing bundle as a *x509.CertPool. func (c *CA) Pool() *x509.CertPool { pool := x509.NewCertPool() diff --git a/internal/certauthority/certauthority_test.go b/internal/certauthority/certauthority_test.go index 4c1fdf8e..4193990b 100644 --- a/internal/certauthority/certauthority_test.go +++ b/internal/certauthority/certauthority_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package certauthority @@ -80,22 +80,25 @@ func TestLoad(t *testing.T) { require.NoError(t, err) require.NotEmpty(t, ca.caCertBytes) require.NotNil(t, ca.signer) + require.Nil(t, ca.privateKey) // this struct field is only used for CA's created by New() }) } } func TestNew(t *testing.T) { now := time.Now() - got, err := New(pkix.Name{CommonName: "Test CA"}, time.Minute) + ca, err := New(pkix.Name{CommonName: "Test CA"}, time.Minute) require.NoError(t, err) - require.NotNil(t, got) + require.NotNil(t, ca) // Make sure the CA certificate looks roughly like what we expect. - caCert, err := x509.ParseCertificate(got.caCertBytes) + caCert, err := x509.ParseCertificate(ca.caCertBytes) require.NoError(t, err) require.Equal(t, "Test CA", caCert.Subject.CommonName) require.WithinDuration(t, now.Add(-10*time.Second), caCert.NotBefore, 10*time.Second) require.WithinDuration(t, now.Add(time.Minute), caCert.NotAfter, 10*time.Second) + + require.NotNil(t, ca.privateKey) } func TestNewInternal(t *testing.T) { @@ -175,21 +178,34 @@ func TestNewInternal(t *testing.T) { } func TestBundle(t *testing.T) { - t.Run("success", func(t *testing.T) { - ca := CA{caCertBytes: []byte{1, 2, 3, 4, 5, 6, 7, 8}} - got := ca.Bundle() - require.Equal(t, "-----BEGIN CERTIFICATE-----\nAQIDBAUGBwg=\n-----END CERTIFICATE-----\n", string(got)) - }) + ca := CA{caCertBytes: []byte{1, 2, 3, 4, 5, 6, 7, 8}} + certPEM := ca.Bundle() + require.Equal(t, "-----BEGIN CERTIFICATE-----\nAQIDBAUGBwg=\n-----END CERTIFICATE-----\n", string(certPEM)) +} + +func TestPrivateKeyToPEM(t *testing.T) { + ca, err := New(pkix.Name{CommonName: "Test CA"}, time.Hour) + require.NoError(t, err) + keyPEM, err := ca.PrivateKeyToPEM() + require.NoError(t, err) + require.Regexp(t, "(?s)-----BEGIN EC "+"PRIVATE KEY-----\n.*\n-----END EC PRIVATE KEY-----", string(keyPEM)) + certPEM := ca.Bundle() + // Check that the public and private keys work together. + _, err = tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + reloaded, err := Load(string(certPEM), string(keyPEM)) + require.NoError(t, err) + _, err = reloaded.PrivateKeyToPEM() + require.EqualError(t, err, "no private key data (did you try to use this after Load?)") } func TestPool(t *testing.T) { - t.Run("success", func(t *testing.T) { - ca, err := New(pkix.Name{CommonName: "test"}, 1*time.Hour) - require.NoError(t, err) + ca, err := New(pkix.Name{CommonName: "test"}, 1*time.Hour) + require.NoError(t, err) - got := ca.Pool() - require.Len(t, got.Subjects(), 1) - }) + pool := ca.Pool() + require.Len(t, pool.Subjects(), 1) } type errSigner struct { diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 4e8468d1..661fd086 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -33,7 +33,13 @@ import ( ) const ( - impersonationProxyPort = ":8444" + impersonationProxyPort = "8444" + defaultHTTPSPort = 443 + oneYear = 100 * 365 * 24 * time.Hour + caCommonName = "Pinniped Impersonation Proxy CA" + caCrtKey = "ca.crt" + caKeyKey = "ca.key" + appLabelKey = "app" ) type impersonatorConfigController struct { @@ -45,13 +51,14 @@ type impersonatorConfigController struct { secretsInformer corev1informers.SecretInformer generatedLoadBalancerServiceName string tlsSecretName string + caSecretName string labels map[string]string startTLSListenerFunc StartTLSListenerFunc httpHandlerFactory func() (http.Handler, error) server *http.Server hasControlPlaneNodes *bool - tlsCert *tls.Certificate + tlsCert *tls.Certificate // always read/write using tlsCertMutex tlsCertMutex sync.RWMutex } @@ -68,6 +75,7 @@ func NewImpersonatorConfigController( withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, generatedLoadBalancerServiceName string, tlsSecretName string, + caSecretName string, labels map[string]string, startTLSListenerFunc StartTLSListenerFunc, httpHandlerFactory func() (http.Handler, error), @@ -84,6 +92,7 @@ func NewImpersonatorConfigController( secretsInformer: secretsInformer, generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, tlsSecretName: tlsSecretName, + caSecretName: caSecretName, labels: labels, startTLSListenerFunc: startTLSListenerFunc, httpHandlerFactory: httpHandlerFactory, @@ -101,10 +110,13 @@ func NewImpersonatorConfigController( ), withInformer( secretsInformer, - pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(tlsSecretName, namespace), + pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { + return (obj.GetName() == tlsSecretName || obj.GetName() == caSecretName) && obj.GetNamespace() == namespace + }, nil), controllerlib.InformerOption{}, ), - // Be sure to run once even if the ConfigMap that the informer is watching doesn't exist. + // Be sure to run once even if the ConfigMap that the informer is watching doesn't exist so we can implement + // the default configuration behavior. withInitialEvent(controllerlib.Key{ Namespace: namespace, Name: configMapResourceName, @@ -112,31 +124,13 @@ func NewImpersonatorConfigController( ) } -func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { +func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error { plog.Debug("Starting impersonatorConfigController Sync") + ctx := syncCtx.Context - configMap, err := c.configMapsInformer.Lister().ConfigMaps(c.namespace).Get(c.configMapResourceName) - notFound := k8serrors.IsNotFound(err) - if err != nil && !notFound { - return fmt.Errorf("failed to get %s/%s configmap: %w", c.namespace, c.configMapResourceName, err) - } - - var config *impersonator.Config - if notFound { - plog.Info("Did not find impersonation proxy config: using default config values", - "configmap", c.configMapResourceName, - "namespace", c.namespace, - ) - config = impersonator.NewConfig() // use default configuration options - } else { - config, err = impersonator.ConfigFromConfigMap(configMap) - if err != nil { - return fmt.Errorf("invalid impersonator configuration: %v", err) - } - plog.Info("Read impersonation proxy config", - "configmap", c.configMapResourceName, - "namespace", c.namespace, - ) + config, err := c.loadImpersonationProxyConfiguration() + if err != nil { + return err } // Make a live API call to avoid the cost of having an informer watch all node changes on the cluster, @@ -144,7 +138,7 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { // Once we have concluded that there is or is not a visible control plane, then cache that decision // to avoid listing nodes very often. if c.hasControlPlaneNodes == nil { - hasControlPlaneNodes, err := clusterhost.New(c.k8sClient).HasControlPlaneNodes(ctx.Context) + hasControlPlaneNodes, err := clusterhost.New(c.k8sClient).HasControlPlaneNodes(ctx) if err != nil { return err } @@ -163,25 +157,25 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { } if c.shouldHaveLoadBalancer(config) { - if err = c.ensureLoadBalancerIsStarted(ctx.Context); err != nil { + if err = c.ensureLoadBalancerIsStarted(ctx); err != nil { return err } } else { - if err = c.ensureLoadBalancerIsStopped(ctx.Context); err != nil { + if err = c.ensureLoadBalancerIsStopped(ctx); err != nil { return err } } if c.shouldHaveTLSSecret(config) { - err = c.ensureTLSSecret(ctx, config) - if err != nil { + var impersonationCA *certauthority.CA + if impersonationCA, err = c.ensureCASecretIsCreated(ctx); err != nil { return err } - } else { - err = c.ensureTLSSecretIsRemoved(ctx.Context) - if err != nil { + if err = c.ensureTLSSecret(ctx, config, impersonationCA); err != nil { return err } + } else if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return err } plog.Debug("Successfully finished impersonatorConfigController Sync") @@ -189,22 +183,32 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { return nil } -func (c *impersonatorConfigController) ensureTLSSecret(ctx controllerlib.Context, config *impersonator.Config) error { - secret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) +func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*impersonator.Config, error) { + configMap, err := c.configMapsInformer.Lister().ConfigMaps(c.namespace).Get(c.configMapResourceName) notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return nil, fmt.Errorf("failed to get %s/%s configmap: %w", c.namespace, c.configMapResourceName, err) + } + + var config *impersonator.Config if notFound { - secret = nil + plog.Info("Did not find impersonation proxy config: using default config values", + "configmap", c.configMapResourceName, + "namespace", c.namespace, + ) + config = impersonator.NewConfig() // use default configuration options + } else { + config, err = impersonator.ConfigFromConfigMap(configMap) + if err != nil { + return nil, fmt.Errorf("invalid impersonator configuration: %v", err) + } + plog.Info("Read impersonation proxy config", + "configmap", c.configMapResourceName, + "namespace", c.namespace, + ) } - if !notFound && err != nil { - return err - } - if secret, err = c.deleteWhenTLSCertificateDoesNotMatchDesiredState(ctx.Context, config, secret); err != nil { - return err - } - if err = c.ensureTLSSecretIsCreatedAndLoaded(ctx.Context, config, secret); err != nil { - return err - } - return nil + + return config, nil } func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { @@ -219,63 +223,7 @@ func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator. return c.shouldHaveImpersonator(config) } -func certHostnameAndIPMatchDesiredState(desiredIP net.IP, actualIPs []net.IP, desiredHostname string, actualHostnames []string) bool { - if desiredIP != nil && len(actualIPs) == 1 && desiredIP.Equal(actualIPs[0]) && len(actualHostnames) == 0 { - return true - } - if desiredHostname != "" && len(actualHostnames) == 1 && desiredHostname == actualHostnames[0] && len(actualIPs) == 0 { - return true - } - return false -} - -func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { - if c.server != nil { - plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) - err := c.server.Close() - c.server = nil - if err != nil { - return err - } - } - return nil -} - -func (c *impersonatorConfigController) ensureImpersonatorIsStarted() error { - if c.server != nil { - return nil - } - - handler, err := c.httpHandlerFactory() - if err != nil { - return err - } - - listener, err := c.startTLSListenerFunc("tcp", impersonationProxyPort, &tls.Config{ - MinVersion: tls.VersionTLS12, // Allow v1.2 because clients like the default `curl` on MacOS don't support 1.3 yet. - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - return c.getTLSCert(), nil - }, - }) - if err != nil { - return err - } - - c.server = &http.Server{Handler: handler} - - go func() { - plog.Info("Starting impersonation proxy", "port", impersonationProxyPort) - err = c.server.Serve(listener) - if errors.Is(err, http.ErrServerClosed) { - plog.Info("The impersonation proxy server has shut down") - } else { - plog.Error("Unexpected shutdown of the impersonation proxy server", err) - } - }() - return nil -} - -func (c *impersonatorConfigController) isLoadBalancerRunning() (bool, error) { +func (c *impersonatorConfigController) loadBalancerExists() (bool, error) { _, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) notFound := k8serrors.IsNotFound(err) if notFound { @@ -299,26 +247,72 @@ func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, erro return true, secret, nil } +func (c *impersonatorConfigController) ensureImpersonatorIsStarted() error { + if c.server != nil { + return nil + } + + handler, err := c.httpHandlerFactory() + if err != nil { + return err + } + + listener, err := c.startTLSListenerFunc("tcp", ":"+impersonationProxyPort, &tls.Config{ + MinVersion: tls.VersionTLS12, // Allow v1.2 because clients like the default `curl` on MacOS don't support 1.3 yet. + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + return c.getTLSCert(), nil + }, + }) + if err != nil { + return err + } + + c.server = &http.Server{Handler: handler} + + go func() { + plog.Info("Starting impersonation proxy", "port", impersonationProxyPort) + err = c.server.Serve(listener) + if errors.Is(err, http.ErrServerClosed) { + plog.Info("The impersonation proxy server has shut down") + } else { + plog.Error("Unexpected shutdown of the impersonation proxy server", err) + } + }() + return nil +} + +func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { + if c.server != nil { + plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) + err := c.server.Close() + c.server = nil + if err != nil { + return err + } + } + return nil +} + func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.Context) error { - running, err := c.isLoadBalancerRunning() + running, err := c.loadBalancerExists() if err != nil { return err } if running { return nil } - appNameLabel := c.labels["app"] + appNameLabel := c.labels[appLabelKey] loadBalancer := v1.Service{ Spec: v1.ServiceSpec{ - Type: "LoadBalancer", + Type: v1.ServiceTypeLoadBalancer, Ports: []v1.ServicePort{ { - TargetPort: intstr.FromInt(8444), - Port: 443, + TargetPort: intstr.FromString(impersonationProxyPort), + Port: defaultHTTPSPort, Protocol: v1.ProtocolTCP, }, }, - Selector: map[string]string{"app": appNameLabel}, + Selector: map[string]string{appLabelKey: appNameLabel}, }, ObjectMeta: metav1.ObjectMeta{ Name: c.generatedLoadBalancerServiceName, @@ -330,14 +324,11 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C "service", c.generatedLoadBalancerServiceName, "namespace", c.namespace) _, err = c.k8sClient.CoreV1().Services(c.namespace).Create(ctx, &loadBalancer, metav1.CreateOptions{}) - if err != nil { - return fmt.Errorf("could not create load balancer: %w", err) - } - return nil + return err } func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { - running, err := c.isLoadBalancerRunning() + running, err := c.loadBalancerExists() if err != nil { return err } @@ -348,45 +339,56 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.C plog.Info("Deleting load balancer for impersonation proxy", "service", c.generatedLoadBalancerServiceName, "namespace", c.namespace) - err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) - if err != nil { + return c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) +} + +func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, config *impersonator.Config, ca *certauthority.CA) error { + secretFromInformer, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) + notFound := k8serrors.IsNotFound(err) + if !notFound && err != nil { return err } - return nil -} - -func (c *impersonatorConfigController) deleteWhenTLSCertificateDoesNotMatchDesiredState(ctx context.Context, config *impersonator.Config, secret *v1.Secret) (*v1.Secret, error) { - if secret == nil { - // There is no Secret, so there is nothing to delete. - return secret, nil + if !notFound { + secretWasDeleted, err := c.deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx, config, ca, secretFromInformer) + if err != nil { + return err + } + // If it was deleted by the above call, then set it to nil. This allows us to avoid waiting + // for the informer cache to update before deciding to proceed to create the new Secret below. + if secretWasDeleted { + secretFromInformer = nil + } } + return c.ensureTLSSecretIsCreatedAndLoaded(ctx, config, secretFromInformer, ca) +} + +func (c *impersonatorConfigController) deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx context.Context, config *impersonator.Config, ca *certauthority.CA, secret *v1.Secret) (bool, error) { certPEM := secret.Data[v1.TLSCertKey] block, _ := pem.Decode(certPEM) if block == nil { plog.Warning("Found missing or not PEM-encoded data in TLS Secret", - "invalidCertPEM", certPEM, + "invalidCertPEM", string(certPEM), "secret", c.tlsSecretName, "namespace", c.namespace) deleteErr := c.ensureTLSSecretIsRemoved(ctx) if deleteErr != nil { - return nil, fmt.Errorf("found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: %w", deleteErr) + return false, fmt.Errorf("found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: %w", deleteErr) } - return nil, nil + return true, nil } actualCertFromSecret, err := x509.ParseCertificate(block.Bytes) if err != nil { plog.Error("Found invalid PEM data in TLS Secret", err, - "invalidCertPEM", certPEM, + "invalidCertPEM", string(certPEM), "secret", c.tlsSecretName, "namespace", c.namespace) - deleteErr := c.ensureTLSSecretIsRemoved(ctx) - if deleteErr != nil { - return nil, fmt.Errorf("PEM data represented an invalid cert, but got error while deleting it: %w", deleteErr) + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, fmt.Errorf("PEM data represented an invalid cert, but got error while deleting it: %w", err) } - return nil, nil + return true, nil } keyPEM := secret.Data[v1.TLSPrivateKeyKey] @@ -395,25 +397,33 @@ func (c *impersonatorConfigController) deleteWhenTLSCertificateDoesNotMatchDesir plog.Error("Found invalid private key PEM data in TLS Secret", err, "secret", c.tlsSecretName, "namespace", c.namespace) - deleteErr := c.ensureTLSSecretIsRemoved(ctx) - if deleteErr != nil { - return nil, fmt.Errorf("cert had an invalid private key, but got error while deleting it: %w", deleteErr) + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, fmt.Errorf("cert had an invalid private key, but got error while deleting it: %w", err) } - return nil, nil + return true, nil + } + + opts := x509.VerifyOptions{Roots: ca.Pool()} + if _, err = actualCertFromSecret.Verify(opts); err != nil { + // The TLS cert was not signed by the current CA. Since they are mismatched, delete the TLS cert + // so we can recreate it using the current CA. + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, err + } + return true, nil } desiredIP, desiredHostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) if err != nil { - return secret, err + return false, err } if !nameIsReady { // We currently have a secret but we are waiting for a load balancer to be assigned an ingress, so // our current secret must be old/unwanted. - err = c.ensureTLSSecretIsRemoved(ctx) - if err != nil { - return secret, err + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, err } - return nil, nil + return true, nil } actualIPs := actualCertFromSecret.IPAddresses @@ -428,17 +438,26 @@ func (c *impersonatorConfigController) deleteWhenTLSCertificateDoesNotMatchDesir if certHostnameAndIPMatchDesiredState(desiredIP, actualIPs, desiredHostname, actualHostnames) { // The cert already matches the desired state, so there is no need to delete/recreate it. - return secret, nil + return false, nil } - err = c.ensureTLSSecretIsRemoved(ctx) - if err != nil { - return secret, err + if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { + return false, err } - return nil, nil + return true, nil } -func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret) error { +func certHostnameAndIPMatchDesiredState(desiredIP net.IP, actualIPs []net.IP, desiredHostname string, actualHostnames []string) bool { + if desiredIP != nil && len(actualIPs) == 1 && desiredIP.Equal(actualIPs[0]) && len(actualHostnames) == 0 { + return true + } + if desiredHostname != "" && len(actualHostnames) == 1 && desiredHostname == actualHostnames[0] && len(actualIPs) == 0 { + return true + } + return false +} + +func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret, ca *certauthority.CA) error { if secret != nil { err := c.loadTLSCertFromSecret(secret) if err != nil { @@ -447,13 +466,6 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } - // TODO create/save/watch the CA separately so we can reuse it to mint tls certs as the settings are dynamically changed, - // so that clients don't need to be updated to use a different CA just because the server-side settings were changed. - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "Pinniped Impersonation Proxy CA"}, 100*365*24*time.Hour) - if err != nil { - return fmt.Errorf("could not create impersonation CA: %w", err) - } - ip, hostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) if err != nil { return err @@ -463,15 +475,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } - var hostnames []string - var ips []net.IP - if hostname != "" { - hostnames = []string{hostname} - } - if ip != nil { - ips = []net.IP{ip} - } - newTLSSecret, err := c.createNewTLSSecret(ctx, impersonationCA, ips, hostnames) + newTLSSecret, err := c.createNewTLSSecret(ctx, ca, ip, hostname) if err != nil { return err } @@ -484,6 +488,61 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx con return nil } +func (c *impersonatorConfigController) ensureCASecretIsCreated(ctx context.Context) (*certauthority.CA, error) { + caSecret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.caSecretName) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, err + } + + var impersonationCA *certauthority.CA + if k8serrors.IsNotFound(err) { + impersonationCA, err = c.createCASecret(ctx) + } else { + crtBytes := caSecret.Data[caCrtKey] + keyBytes := caSecret.Data[caKeyKey] + impersonationCA, err = certauthority.Load(string(crtBytes), string(keyBytes)) + } + if err != nil { + return nil, err + } + + return impersonationCA, nil +} + +func (c *impersonatorConfigController) createCASecret(ctx context.Context) (*certauthority.CA, error) { + impersonationCA, err := certauthority.New(pkix.Name{CommonName: caCommonName}, oneYear) + if err != nil { + return nil, fmt.Errorf("could not create impersonation CA: %w", err) + } + + caPrivateKeyPEM, err := impersonationCA.PrivateKeyToPEM() + if err != nil { + return nil, err + } + + secret := v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.caSecretName, + Namespace: c.namespace, + Labels: c.labels, + }, + Data: map[string][]byte{ + caCrtKey: impersonationCA.Bundle(), + caKeyKey: caPrivateKeyPEM, + }, + Type: v1.SecretTypeOpaque, + } + + plog.Info("Creating CA certificates for impersonation proxy", + "secret", c.caSecretName, + "namespace", c.namespace) + if _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, &secret, metav1.CreateOptions{}); err != nil { + return nil, err + } + + return impersonationCA, nil +} + func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *impersonator.Config) (net.IP, string, bool, error) { if config.Endpoint != "" { return c.findTLSCertificateNameFromEndpointConfig(config) @@ -504,7 +563,8 @@ func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) notFound := k8serrors.IsNotFound(err) if notFound { - // Maybe the loadbalancer hasn't been cached in the informer yet. We aren't ready and will try again later. + // Although we created the load balancer, maybe it hasn't been cached in the informer yet. + // We aren't ready and will try again later in this case. return nil, "", false, nil } if err != nil { @@ -534,8 +594,17 @@ func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() return nil, "", false, fmt.Errorf("could not find valid IP addresses or hostnames from load balancer %s/%s", c.namespace, lb.Name) } -func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP, hostnames []string) (*v1.Secret, error) { - impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, 100*365*24*time.Hour) +func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ip net.IP, hostname string) (*v1.Secret, error) { + var hostnames []string + var ips []net.IP + if hostname != "" { + hostnames = []string{hostname} + } + if ip != nil { + ips = []net.IP{ip} + } + + impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, oneYear) if err != nil { return nil, fmt.Errorf("could not create impersonation cert: %w", err) } @@ -546,17 +615,16 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c } newTLSSecret := &v1.Secret{ - Type: v1.SecretTypeTLS, ObjectMeta: metav1.ObjectMeta{ Name: c.tlsSecretName, Namespace: c.namespace, Labels: c.labels, }, Data: map[string][]byte{ - "ca.crt": ca.Bundle(), v1.TLSPrivateKeyKey: keyPEM, v1.TLSCertKey: certPEM, }, + Type: v1.SecretTypeTLS, } plog.Info("Creating TLS certificates for impersonation proxy", @@ -564,12 +632,7 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c "hostnames", hostnames, "secret", c.tlsSecretName, "namespace", c.namespace) - _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) - if err != nil { - return nil, err - } - - return newTLSSecret, nil + return c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) } func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secret) error { @@ -577,16 +640,11 @@ func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secre keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) if err != nil { - plog.Error("Could not parse TLS cert PEM data from Secret", - err, - "secret", c.tlsSecretName, - "namespace", c.namespace, - ) c.setTLSCert(nil) return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) } plog.Info("Loading TLS certificates for impersonation proxy", - "certPEM", certPEM, + "certPEM", string(certPEM), "secret", c.tlsSecretName, "namespace", c.namespace) c.setTLSCert(&tlsCert) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 7570e2f5..3b779179 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -64,7 +64,8 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" const generatedLoadBalancerServiceName = "some-service-resource-name" - const tlsSecretName = "some-secret-name" + const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential + const caSecretName = "some-ca-secret-name" var r *require.Assertions var observableWithInformerOption *testutil.ObservableWithInformerOption @@ -93,6 +94,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { observableWithInitialEventOption.WithInitialEvent, generatedLoadBalancerServiceName, tlsSecretName, + caSecretName, nil, nil, nil, @@ -200,31 +202,41 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { when("watching Secret objects", func() { var subject controllerlib.Filter - var target, wrongNamespace, wrongName, unrelated *corev1.Secret + var target1, target2, wrongNamespace1, wrongNamespace2, wrongName, unrelated *corev1.Secret it.Before(func() { subject = secretsInformerFilter - target = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: installedInNamespace}} - wrongNamespace = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: "wrong-namespace"}} + target1 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: installedInNamespace}} + target2 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: installedInNamespace}} + wrongNamespace1 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: "wrong-namespace"}} + wrongNamespace2 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: "wrong-namespace"}} wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} }) - when("the target Secret changes", func() { + when("one of the target Secrets changes", func() { it("returns true to trigger the sync method", func() { - r.True(subject.Add(target)) - r.True(subject.Update(target, unrelated)) - r.True(subject.Update(unrelated, target)) - r.True(subject.Delete(target)) + r.True(subject.Add(target1)) + r.True(subject.Update(target1, unrelated)) + r.True(subject.Update(unrelated, target1)) + r.True(subject.Delete(target1)) + r.True(subject.Add(target2)) + r.True(subject.Update(target2, unrelated)) + r.True(subject.Update(unrelated, target2)) + r.True(subject.Delete(target2)) }) }) when("a Secret from another namespace changes", func() { it("returns false to avoid triggering the sync method", func() { - r.False(subject.Add(wrongNamespace)) - r.False(subject.Update(wrongNamespace, unrelated)) - r.False(subject.Update(unrelated, wrongNamespace)) - r.False(subject.Delete(wrongNamespace)) + r.False(subject.Add(wrongNamespace1)) + r.False(subject.Update(wrongNamespace1, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace1)) + r.False(subject.Delete(wrongNamespace1)) + r.False(subject.Add(wrongNamespace2)) + r.False(subject.Update(wrongNamespace2, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace2)) + r.False(subject.Delete(wrongNamespace2)) }) }) @@ -262,7 +274,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" const loadBalancerServiceName = "some-service-resource-name" - const tlsSecretName = "some-secret-name" + const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential + const caSecretName = "some-ca-secret-name" const localhostIP = "127.0.0.1" const httpsPort = ":443" var labels = map[string]string{"app": "app-name", "other-key": "other-value"} @@ -279,7 +292,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var startTLSListenerFuncWasCalled int var startTLSListenerFuncError error var startTLSListenerUponCloseError error - var httpHanderFactoryFuncError error + var httpHandlerFactoryFuncError error var startedTLSListener net.Listener var startTLSListenerFunc = func(network, listenAddress string, config *tls.Config) (net.Listener, error) { @@ -335,7 +348,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } else { rootCAs := x509.NewCertPool() rootCAs.AppendCertsFromPEM(caCrt) - tr = &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: rootCAs}, DialContext: overrideDialContext, @@ -390,14 +402,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }, 10*time.Second, time.Millisecond) } - var waitForLoadBalancerToBeDeleted = func(informer corev1informers.ServiceInformer, name string) { + var waitForServiceToBeDeleted = func(informer corev1informers.ServiceInformer, name string) { r.Eventually(func() bool { _, err := informer.Lister().Services(installedInNamespace).Get(name) return k8serrors.IsNotFound(err) }, 10*time.Second, time.Millisecond) } - var waitForTLSCertSecretToBeDeleted = func(informer corev1informers.SecretInformer, name string) { + var waitForSecretToBeDeleted = func(informer corev1informers.SecretInformer, name string) { r.Eventually(func() bool { _, err := informer.Lister().Secrets(installedInNamespace).Get(name) return k8serrors.IsNotFound(err) @@ -419,13 +431,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { controllerlib.WithInitialEvent, loadBalancerServiceName, tlsSecretName, + caSecretName, labels, startTLSListenerFunc, func() (http.Handler, error) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { _, err := fmt.Fprintf(w, "hello world") r.NoError(err) - }), httpHanderFactoryFuncError + }), httpHandlerFactoryFuncError }, ) @@ -494,34 +507,51 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var newStubTLSSecret = func(resourceName string) *corev1.Secret { + var newEmptySecret = func(resourceName string) *corev1.Secret { return newSecretWithData(resourceName, map[string][]byte{}) } - var createCertSecretData = func(dnsNames []string, ip string) map[string][]byte { - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) + var newCA = func() *certauthority.CA { + ca, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) r.NoError(err) - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, dnsNames, []net.IP{net.ParseIP(ip)}, 24*time.Hour) + return ca + } + + var newCACertSecretData = func(ca *certauthority.CA) map[string][]byte { + keyPEM, err := ca.PrivateKeyToPEM() + r.NoError(err) + return map[string][]byte{ + "ca.crt": ca.Bundle(), + "ca.key": keyPEM, + } + } + + var newTLSCertSecretData = func(ca *certauthority.CA, dnsNames []string, ip string) map[string][]byte { + impersonationCert, err := ca.Issue(pkix.Name{}, dnsNames, []net.IP{net.ParseIP(ip)}, 24*time.Hour) r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) r.NoError(err) return map[string][]byte{ - "ca.crt": impersonationCA.Bundle(), corev1.TLSPrivateKeyKey: keyPEM, corev1.TLSCertKey: certPEM, } } - var newActualTLSSecret = func(resourceName string, ip string) *corev1.Secret { - return newSecretWithData(resourceName, createCertSecretData(nil, ip)) + var newActualCASecret = func(ca *certauthority.CA, resourceName string) *corev1.Secret { + return newSecretWithData(resourceName, newCACertSecretData(ca)) } - var newActualTLSSecretWithMultipleHostnames = func(resourceName string, ip string) *corev1.Secret { - return newSecretWithData(resourceName, createCertSecretData([]string{"foo", "bar"}, ip)) + var newActualTLSSecret = func(ca *certauthority.CA, resourceName string, ip string) *corev1.Secret { + return newSecretWithData(resourceName, newTLSCertSecretData(ca, nil, ip)) + } + + var newActualTLSSecretWithMultipleHostnames = func(ca *certauthority.CA, resourceName string, ip string) *corev1.Secret { + return newSecretWithData(resourceName, newTLSCertSecretData(ca, []string{"foo", "bar"}, ip)) } var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { - createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + createdSecret, ok := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) createdSecret.ResourceVersion = resourceVersion r.NoError(client.Tracker().Add(createdSecret)) } @@ -561,8 +591,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(client.Tracker().Add(loadBalancerService)) } - var addSecretToTracker = func(secret *corev1.Secret, client *kubernetesfake.Clientset) { - r.NoError(client.Tracker().Add(secret)) + var addSecretToTrackers = func(secret *corev1.Secret, clients ...*kubernetesfake.Clientset) { + for _, client := range clients { + r.NoError(client.Tracker().Add(secret)) + } } var updateLoadBalancerServiceInTracker = func(resourceName string, ingresses []corev1.LoadBalancerIngress, client *kubernetesfake.Clientset, newResourceVersion string) { @@ -582,7 +614,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } - var deleteLoadBalancerServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + var deleteServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { r.NoError(client.Tracker().Delete( schema.GroupVersionResource{Version: "v1", Resource: "services"}, installedInNamespace, @@ -590,7 +622,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } - var deleteTLSCertSecretFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + var deleteSecretFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { r.NoError(client.Tracker().Delete( schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, installedInNamespace, @@ -621,7 +653,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var requireLoadBalancerWasCreated = func(action coretesting.Action) { - createAction := action.(coretesting.CreateAction) + createAction, ok := action.(coretesting.CreateAction) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) r.Equal("create", createAction.GetVerb()) createdLoadBalancerService := createAction.GetObject().(*corev1.Service) r.Equal(loadBalancerServiceName, createdLoadBalancerService.Name) @@ -631,44 +664,66 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(labels, createdLoadBalancerService.Labels) } - var requireLoadBalancerDeleted = func(action coretesting.Action) { - deleteAction := action.(coretesting.DeleteAction) + var requireLoadBalancerWasDeleted = func(action coretesting.Action) { + deleteAction, ok := action.(coretesting.DeleteAction) + r.True(ok, "should have been able to cast this action to DeleteAction: %v", action) r.Equal("delete", deleteAction.GetVerb()) r.Equal(loadBalancerServiceName, deleteAction.GetName()) r.Equal("services", deleteAction.GetResource().Resource) } - var requireTLSSecretDeleted = func(action coretesting.Action) { - deleteAction := action.(coretesting.DeleteAction) + var requireTLSSecretWasDeleted = func(action coretesting.Action) { + deleteAction, ok := action.(coretesting.DeleteAction) + r.True(ok, "should have been able to cast this action to DeleteAction: %v", action) r.Equal("delete", deleteAction.GetVerb()) r.Equal(tlsSecretName, deleteAction.GetName()) r.Equal("secrets", deleteAction.GetResource().Resource) } - var requireTLSSecretWasCreated = func(action coretesting.Action) []byte { - createAction := action.(coretesting.CreateAction) + var requireCASecretWasCreated = func(action coretesting.Action) []byte { + createAction, ok := action.(coretesting.CreateAction) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) r.Equal("create", createAction.GetVerb()) createdSecret := createAction.GetObject().(*corev1.Secret) - r.Equal(tlsSecretName, createdSecret.Name) + r.Equal(caSecretName, createdSecret.Name) r.Equal(installedInNamespace, createdSecret.Namespace) - r.Equal(corev1.SecretTypeTLS, createdSecret.Type) + r.Equal(corev1.SecretTypeOpaque, createdSecret.Type) r.Equal(labels, createdSecret.Labels) - r.Len(createdSecret.Data, 3) - r.NotNil(createdSecret.Data["ca.crt"]) - r.NotNil(createdSecret.Data[corev1.TLSPrivateKeyKey]) - r.NotNil(createdSecret.Data[corev1.TLSCertKey]) - validCert := testutil.ValidateCertificate(t, string(createdSecret.Data["ca.crt"]), string(createdSecret.Data[corev1.TLSCertKey])) - validCert.RequireMatchesPrivateKey(string(createdSecret.Data[corev1.TLSPrivateKeyKey])) - validCert.RequireLifetime(time.Now().Add(-10*time.Second), time.Now().Add(100*time.Hour*24*365), 10*time.Second) - // Make sure the CA certificate looks roughly like what we expect. - block, _ := pem.Decode(createdSecret.Data["ca.crt"]) + r.Len(createdSecret.Data, 2) + createdCertPEM := createdSecret.Data["ca.crt"] + createdKeyPEM := createdSecret.Data["ca.key"] + r.NotNil(createdCertPEM) + r.NotNil(createdKeyPEM) + _, err := tls.X509KeyPair(createdCertPEM, createdKeyPEM) + r.NoError(err, "key does not match cert") + // Decode and parse the cert to check some of its fields. + block, _ := pem.Decode(createdCertPEM) require.NotNil(t, block) caCert, err := x509.ParseCertificate(block.Bytes) require.NoError(t, err) require.Equal(t, "Pinniped Impersonation Proxy CA", caCert.Subject.CommonName) require.WithinDuration(t, time.Now().Add(-10*time.Second), caCert.NotBefore, 10*time.Second) require.WithinDuration(t, time.Now().Add(100*time.Hour*24*365), caCert.NotAfter, 10*time.Second) - return createdSecret.Data["ca.crt"] + return createdCertPEM + } + + var requireTLSSecretWasCreated = func(action coretesting.Action, caCert []byte) { + createAction, ok := action.(coretesting.CreateAction) + r.True(ok, "should have been able to cast this action to CreateAction: %v", action) + r.Equal("create", createAction.GetVerb()) + createdSecret := createAction.GetObject().(*corev1.Secret) + r.Equal(tlsSecretName, createdSecret.Name) + r.Equal(installedInNamespace, createdSecret.Namespace) + r.Equal(corev1.SecretTypeTLS, createdSecret.Type) + r.Equal(labels, createdSecret.Labels) + r.Len(createdSecret.Data, 2) + createdCertPEM := createdSecret.Data[corev1.TLSCertKey] + createdKeyPEM := createdSecret.Data[corev1.TLSPrivateKeyKey] + r.NotNil(createdKeyPEM) + r.NotNil(createdCertPEM) + validCert := testutil.ValidateCertificate(t, string(caCert), string(createdCertPEM)) + validCert.RequireMatchesPrivateKey(string(createdKeyPEM)) + validCert.RequireLifetime(time.Now().Add(-10*time.Second), time.Now().Add(100*time.Hour*24*365), 10*time.Second) } var runControllerSync = func() error { @@ -714,9 +769,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("control-plane", kubeAPIClient) addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) - tlsSecret := newStubTLSSecret(tlsSecretName) - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + addSecretToTrackers(newEmptySecret(tlsSecretName), kubeAPIClient, kubeInformerClient) }) it("does not start the impersonator, deletes the loadbalancer, deletes the Secret", func() { @@ -725,8 +778,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) }) }) @@ -739,9 +792,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the load balancer automatically", func() { requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) }) }) @@ -756,8 +810,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer automatically", func() { requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 1) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) }) }) @@ -772,8 +827,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer automatically", func() { requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 1) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) }) }) @@ -788,8 +844,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer automatically", func() { requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 1) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) }) }) @@ -803,17 +860,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("starts the impersonator with certs that match the first IP address", func() { - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, "127.0.0.123", map[string]string{"127.0.0.123:443": testServerAddr()}) }) it("keeps the secret around after resync", func() { + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) // nothing changed + r.Len(kubeAPIClient.Actions(), 3) // nothing changed }) }) @@ -828,17 +890,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("starts the impersonator with certs that match the first hostname", func() { - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) }) it("keeps the secret around after resync", func() { + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) // nothing changed + r.Len(kubeAPIClient.Actions(), 3) // nothing changed }) }) @@ -853,28 +920,36 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("starts the impersonator with certs that match the first hostname", func() { - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) }) it("keeps the secret around after resync", func() { + // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) // nothing changed + r.Len(kubeAPIClient.Actions(), 3) // nothing changed }) }) - when("there are not visible control plane nodes, a secret exists with multiple hostnames and an IP", func() { + when("there are not visible control plane nodes, a TLS secret exists with multiple hostnames and an IP", func() { + var caCrt []byte it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) - tlsSecret := newActualTLSSecretWithMultipleHostnames(tlsSecretName, localhostIP) - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + addSecretToTrackers(newActualTLSSecretWithMultipleHostnames(ca, tlsSecretName, localhostIP), kubeAPIClient, kubeInformerClient) startInformersAndController() r.NoError(runControllerSync()) }) @@ -882,9 +957,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("deletes and recreates the secret to match the IP in the load balancer without the extra hostnames", func() { r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) }) }) @@ -893,9 +968,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("worker", kubeAPIClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.42"}}, kubeAPIClient) - tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + ca := newCA() + addSecretToTrackers(newActualCASecret(ca, caSecretName), kubeAPIClient, kubeInformerClient) + addSecretToTrackers(newActualTLSSecretWithMultipleHostnames(ca, tlsSecretName, localhostIP), kubeAPIClient, kubeInformerClient) kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on delete") }) @@ -906,21 +981,23 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Error(runControllerSync(), "error on delete") r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() }) }) when("the cert's name might need to change but there is an error while determining the new name", func() { - var ca []byte + var caCrt []byte it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) - tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) - ca = tlsSecret.Data["ca.crt"] - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + tlsSecret := newActualTLSSecret(ca, tlsSecretName, localhostIP) + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) }) it("returns an error and keeps running the proxy with the old cert", func() { @@ -928,7 +1005,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") @@ -936,131 +1013,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.EqualError(runControllerSync(), "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") r.Len(kubeAPIClient.Actions(), 1) // no new actions - requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) }) }) }) - when("sync is called more than once", func() { - it.Before(func() { - addNodeWithRoleToTracker("worker", kubeAPIClient) - }) - - it("only starts the impersonator once and only lists the cluster's nodes once", func() { - startInformersAndController() - r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunningWithoutCerts() - - // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - - r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time - requireTLSServerIsRunningWithoutCerts() // still running - r.Len(kubeAPIClient.Actions(), 2) // no new API calls - }) - - it("creates certs from the ip address listed on the load balancer", func() { - startInformersAndController() - r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunningWithoutCerts() - - // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") - - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - - r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time - r.Len(kubeAPIClient.Actions(), 3) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now - - // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - - r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time - r.Len(kubeAPIClient.Actions(), 3) // no more actions - requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running - }) - - it("creates certs from the hostname listed on the load balancer", func() { - hostname := "fake.example.com" - startInformersAndController() - r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunningWithoutCerts() - - // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") - - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - - r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time - r.Len(kubeAPIClient.Actions(), 3) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now - - // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - - r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time - r.Len(kubeAPIClient.Actions(), 3) // no more actions - requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running - }) - }) - - when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { - it("returns an error", func() { - startInformersAndController() - r.EqualError(runControllerSync(), "no nodes found") - requireTLSServerWasNeverStarted() - }) - }) - - when("the http handler factory function returns an error", func() { - it.Before(func() { - addNodeWithRoleToTracker("worker", kubeAPIClient) - httpHanderFactoryFuncError = errors.New("some factory error") - }) - - it("returns an error", func() { - startInformersAndController() - r.EqualError(runControllerSync(), "some factory error") - requireTLSServerWasNeverStarted() - }) - }) - - when("the configmap is invalid", func() { - it.Before(func() { - addImpersonatorConfigMapToTracker(configMapResourceName, "not yaml", kubeInformerClient) - }) - - it("returns an error", func() { - startInformersAndController() - r.EqualError(runControllerSync(), "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config") - requireTLSServerWasNeverStarted() - }) - }) - when("the ConfigMap is already in the installation namespace", func() { when("the configuration is auto mode with an endpoint", func() { it.Before(func() { @@ -1090,9 +1047,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator according to the settings in the ConfigMap", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) }) }) @@ -1135,9 +1093,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the load balancer", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) }) }) @@ -1164,20 +1123,23 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 1) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) }) }) when("a load balancer and a secret already exists", func() { - var ca []byte + var caCrt []byte it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) - tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) - ca = tlsSecret.Data["ca.crt"] - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + tlsSecret := newActualTLSSecret(ca, tlsSecretName, localhostIP) + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) @@ -1187,7 +1149,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) }) }) @@ -1202,18 +1164,19 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator, generates a valid cert for the hostname", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) }) }) when("endpoint is IP address with a port", func() { - const fakeIpWithPort = "127.0.0.1:3000" + const fakeIPWithPort = "127.0.0.1:3000" it.Before(func() { - configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIpWithPort) + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIPWithPort) addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) }) @@ -1221,11 +1184,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator, generates a valid cert for the hostname", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - // Check that the server is running and that TLS certs that are being served are are for fakeIpWithPort. - requireTLSServerIsRunning(ca, fakeIpWithPort, map[string]string{fakeIpWithPort: testServerAddr()}) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeIPWithPort. + requireTLSServerIsRunning(ca, fakeIPWithPort, map[string]string{fakeIPWithPort: testServerAddr()}) }) }) @@ -1240,9 +1204,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator, generates a valid cert for the hostname", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostnameWithPort. requireTLSServerIsRunning(ca, fakeHostnameWithPort, map[string]string{fakeHostnameWithPort: testServerAddr()}) }) @@ -1261,44 +1226,180 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("regenerates the cert for the hostname, then regenerates it for the IP again", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // Switch the endpoint config to a hostname. updateImpersonatorConfigMapInTracker(configMapResourceName, hostnameYAML, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 4) - requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) - ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[3]) + r.Len(kubeAPIClient.Actions(), 5) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], ca) // reuses the old CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) // Simulate the informer cache's background update from its watch. - deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + deleteSecretFromTracker(tlsSecretName, kubeInformerClient) + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[4], kubeInformerClient, "3") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "3") // Switch the endpoint config back to an IP. updateImpersonatorConfigMapInTracker(configMapResourceName, ipAddressYAML, kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 6) - requireTLSSecretDeleted(kubeAPIClient.Actions()[4]) - ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[5]) + r.Len(kubeAPIClient.Actions(), 7) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[5]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[6], ca) // reuses the old CA again // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) }) }) + + when("the TLS cert goes missing and needs to be recreated, e.g. when a user manually deleted it", func() { + const fakeHostname = "fake.example.com" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + }) + + it("uses the existing CA cert the make a new TLS cert", func() { + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + + // Simulate the informer cache's background update from its watch for the CA Secret. + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + // Delete the TLS Secret that was just created from the Kube API server. Note that we never + // simulated it getting added to the informer cache, so we don't need to remove it from there. + deleteSecretFromTracker(tlsSecretName, kubeAPIClient) + + // Run again. It should create a new TLS cert using the old CA cert. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + }) + }) + + when("the CA cert goes missing and needs to be recreated, e.g. when a user manually deleted it", func() { + const fakeHostname = "fake.example.com" + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + }) + + it("makes a new CA cert, deletes the old TLS cert, and makes a new TLS cert using the new CA", func() { + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + + // Simulate the informer cache's background update from its watch for the CA Secret. + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + // Delete the CA Secret that was just created from the Kube API server. Note that we never + // simulated it getting added to the informer cache, so we don't need to remove it from there. + deleteSecretFromTracker(caSecretName, kubeAPIClient) + + // Run again. It should create both a new CA cert and a new TLS cert using the new CA cert. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 6) + ca = requireCASecretWasCreated(kubeAPIClient.Actions()[3]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // created using the new CA + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + }) + }) + + when("the CA cert is overwritten by another valid CA cert", func() { + const fakeHostname = "fake.example.com" + var caCrt []byte + it.Before(func() { + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) + addNodeWithRoleToTracker("worker", kubeAPIClient) + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + + // Simulate the informer cache's background update from its watch for the CA Secret. + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + // Simulate someone updating the CA Secret out of band, e.g. when a human edits it with kubectl. + // Delete the CA Secret that was just created from the Kube API server. Note that we never + // simulated it getting added to the informer cache, so we don't need to remove it from there. + // Then add a new one. Delete + new = update, since only the final state is observed. + deleteSecretFromTracker(caSecretName, kubeAPIClient) + anotherCA := newCA() + newCASecret := newActualCASecret(anotherCA, caSecretName) + caCrt = newCASecret.Data["ca.crt"] + newCASecret.ResourceVersion = "2" + addSecretToTrackers(newCASecret, kubeInformerClient, kubeAPIClient) + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + }) + + it("deletes the old TLS cert and makes a new TLS cert using the new CA", func() { + // Run again. It should use the updated CA cert to create a new TLS cert. + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 5) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], caCrt) // created using the updated CA + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(caCrt, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + }) + + when("deleting the TLS cert due to mismatched CA results in an error", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + if action.(coretesting.DeleteAction).GetName() == tlsSecretName { + return true, nil, fmt.Errorf("error on tls secret delete") + } + return false, nil, nil + }) + }) + + it("returns an error", func() { + r.Error(runControllerSync(), "error on tls secret delete") + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) // tried to delete cert but failed + }) + }) + }) }) when("the configuration switches from enabled to disabled mode", func() { @@ -1312,32 +1413,35 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) // Simulate the informer cache's background update from its watch. addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(runControllerSync()) requireTLSServerIsNoLongerRunning() - r.Len(kubeAPIClient.Actions(), 3) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) + r.Len(kubeAPIClient.Actions(), 4) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[3]) - deleteLoadBalancerServiceFromTracker(loadBalancerServiceName, kubeInformerClient) - waitForLoadBalancerToBeDeleted(kubeInformers.Core().V1().Services(), loadBalancerServiceName) + deleteServiceFromTracker(loadBalancerServiceName, kubeInformerClient) + waitForServiceToBeDeleted(kubeInformers.Core().V1().Services(), loadBalancerServiceName) updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 4) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) + r.Len(kubeAPIClient.Actions(), 5) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[4]) }) when("there is an error while shutting down the server", func() { @@ -1371,34 +1475,37 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Should have started in "enabled" mode with an "endpoint", so no load balancer is needed. r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // Switch to "enabled" mode without an "endpoint", so a load balancer is needed now. updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 4) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[2]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[3]) // the Secret was deleted because it contained a cert with the wrong IP + r.Len(kubeAPIClient.Actions(), 5) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) // the Secret was deleted because it contained a cert with the wrong IP requireTLSServerIsRunningWithoutCerts() // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) - waitForTLSCertSecretToBeDeleted(kubeInformers.Core().V1().Secrets(), tlsSecretName) + deleteSecretFromTracker(tlsSecretName, kubeInformerClient) + waitForSecretToBeDeleted(kubeInformers.Core().V1().Secrets(), tlsSecretName) // The controller should be waiting for the load balancer's ingress to become available. r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 4) // no new actions while it is waiting for the load balancer's ingress + r.Len(kubeAPIClient.Actions(), 5) // no new actions while it is waiting for the load balancer's ingress requireTLSServerIsRunningWithoutCerts() // Update the ingress of the LB in the informer's client and run Sync again. @@ -1406,14 +1513,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}}, kubeInformerClient, "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 5) - ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[4]) // created because the LB ingress became available + r.Len(kubeAPIClient.Actions(), 6) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // reuses the existing CA // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[4], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[5], kubeInformerClient, "3") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "3") // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) @@ -1421,14 +1528,143 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 8) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[5]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[6]) - requireTLSSecretWasCreated(kubeAPIClient.Actions()[7]) // recreated because the endpoint was updated + r.Len(kubeAPIClient.Actions(), 9) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[6]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[7]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[8], ca) // recreated because the endpoint was updated, reused the old CA }) }) }) + when("sync is called more than once", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("only starts the impersonator once and only lists the cluster's nodes once", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + requireTLSServerIsRunningWithoutCerts() // still running + r.Len(kubeAPIClient.Actions(), 3) // no new API calls + }) + + it("creates certs from the ip address listed on the load balancer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time + requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now + + // Simulate the informer cache's background update from its watch. + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started again + r.Len(kubeAPIClient.Actions(), 4) // no more actions + requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running + }) + + it("creates certs from the hostname listed on the load balancer", func() { + hostname := "fake.example.com" + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + + // Simulate the informer cache's background update from its watch. + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "0") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") + + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + r.Len(kubeAPIClient.Actions(), 4) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now + + // Simulate the informer cache's background update from its watch. + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + r.NoError(runControllerSync()) + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 4) // no more actions + requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running + }) + }) + + when("getting the control plane nodes returns an error, e.g. when there are no nodes", func() { + it("returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "no nodes found") + requireTLSServerWasNeverStarted() + }) + }) + + when("the http handler factory function returns an error", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + httpHandlerFactoryFuncError = errors.New("some factory error") + }) + + it("returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "some factory error") + requireTLSServerWasNeverStarted() + }) + }) + + when("the configmap is invalid", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "not yaml", kubeInformerClient) + }) + + it("returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config") + requireTLSServerWasNeverStarted() + }) + }) + when("there is an error creating the load balancer", func() { it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) @@ -1439,7 +1675,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it("exits with an error", func() { - r.EqualError(runControllerSync(), "could not create load balancer: error on create") + r.EqualError(runControllerSync(), "error on create") }) }) @@ -1448,17 +1684,61 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) addNodeWithRoleToTracker("control-plane", kubeAPIClient) kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { - return true, nil, fmt.Errorf("error on create") + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + if createdSecret.Name == tlsSecretName { + return true, nil, fmt.Errorf("error on tls secret create") + } + return false, nil, nil }) }) it("starts the impersonator without certs and returns an error", func() { startInformersAndController() - r.EqualError(runControllerSync(), "error on create") + r.EqualError(runControllerSync(), "error on tls secret create") + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + }) + }) + + when("there is an error creating the CA secret", func() { + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + kubeAPIClient.PrependReactor("create", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + if createdSecret.Name == caSecretName { + return true, nil, fmt.Errorf("error on ca secret create") + } + return false, nil, nil + }) + }) + + it("starts the impersonator without certs and returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "error on ca secret create") requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + }) + }) + + when("the CA secret exists but is invalid while the TLS secret needs to be created", func() { + it.Before(func() { + addNodeWithRoleToTracker("control-plane", kubeAPIClient) + addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: enabled, endpoint: example.com}", kubeInformerClient) + addSecretToTrackers(newEmptySecret(caSecretName), kubeAPIClient, kubeInformerClient) + }) + + it("starts the impersonator without certs and returns an error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "could not load CA: tls: failed to find any PEM data in certificate input") + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) }) }) @@ -1467,9 +1747,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("control-plane", kubeAPIClient) addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) - tlsSecret := newStubTLSSecret(tlsSecretName) - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + addSecretToTrackers(newEmptySecret(tlsSecretName), kubeAPIClient, kubeInformerClient) startInformersAndController() kubeAPIClient.PrependReactor("delete", "secrets", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on delete") @@ -1481,12 +1759,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[1]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[2]) + requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) }) }) - when("the PEM formatted data in the Secret is not a valid cert", func() { + when("the PEM formatted data in the TLS Secret is not a valid cert", func() { it.Before(func() { configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) @@ -1501,17 +1779,17 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"), }, } - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) }) it("deletes the invalid certs, creates new certs, and starts the impersonator", func() { startInformersAndController() r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 3) + r.Len(kubeAPIClient.Actions(), 4) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) // deleted the bad cert + requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) }) @@ -1526,21 +1804,25 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.EqualError(runControllerSync(), "PEM data represented an invalid cert, but got error while deleting it: error on delete") requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) // tried deleted the bad cert, which failed requireTLSServerIsRunningWithoutCerts() }) }) }) when("a tls secret already exists but it is not valid", func() { + var caCrt []byte it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) - tlsSecret := newStubTLSSecret(tlsSecretName) // secret exists but lacks certs - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + addSecretToTrackers(newEmptySecret(tlsSecretName), kubeAPIClient, kubeInformerClient) // secret exists but lacks certs addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) @@ -1550,9 +1832,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1568,20 +1850,24 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed requireTLSServerIsRunningWithoutCerts() }) }) }) when("a tls secret already exists but the private key is not valid", func() { + var caCrt []byte it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) - tlsSecret := newActualTLSSecret(tlsSecretName, localhostIP) + ca := newCA() + caSecret := newActualCASecret(ca, caSecretName) + caCrt = caSecret.Data["ca.crt"] + addSecretToTrackers(caSecret, kubeAPIClient, kubeInformerClient) + tlsSecret := newActualTLSSecret(ca, tlsSecretName, localhostIP) tlsSecret.Data["tls.key"] = nil - addSecretToTracker(tlsSecret, kubeAPIClient) - addSecretToTracker(tlsSecret, kubeInformerClient) + addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient) addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeAPIClient) }) @@ -1591,9 +1877,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert - ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) + requireTLSServerIsRunning(caCrt, testServerAddr(), nil) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1609,7 +1895,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSSecretDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed + requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // tried deleted the bad cert, which failed requireTLSServerIsRunningWithoutCerts() }) }) diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index b76deec4..ce63fc91 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -301,6 +301,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { controllerlib.WithInitialEvent, "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` "pinniped-concierge-impersonation-proxy-tls-serving-certificate", // TODO this string should come from `c.NamesConfig` + "pinniped-concierge-impersonation-proxy-ca-certificate", // TODO this string should come from `c.NamesConfig` c.Labels, tls.Listen, func() (http.Handler, error) { diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index aec7bcdd..de113224 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -33,6 +33,7 @@ const ( // TODO don't hard code "pinniped-concierge-" in these strings. It should be constructed from the env app name. impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" impersonationProxyTLSSecretName = "pinniped-concierge-impersonation-proxy-tls-serving-certificate" //nolint:gosec // this is not a credential + impersonationProxyCASecretName = "pinniped-concierge-impersonation-proxy-ca-certificate" impersonationProxyLoadBalancerName = "pinniped-concierge-impersonation-proxy-load-balancer" ) @@ -160,21 +161,30 @@ func TestImpersonationProxy(t *testing.T) { }) } - // Wait for ca data to be available at the secret location. + // Check that the controller generated a CA. Get the CA data so we can use it as a client. + // TODO We should be getting the CA data from the CredentialIssuer's status instead, once that is implemented. var caSecret *corev1.Secret - require.Eventually(t, - func() bool { - caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) - return caSecret != nil && caSecret.Data["ca.crt"] != nil - }, 5*time.Minute, 250*time.Millisecond) + require.Eventually(t, func() bool { + caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName, metav1.GetOptions{}) + return err == nil && caSecret != nil && caSecret.Data["ca.crt"] != nil + }, 10*time.Second, 250*time.Millisecond) + caCertPEM := caSecret.Data["ca.crt"] + + // Check that the generated TLS cert Secret was created by the controller. + // This could take a while if we are waiting for the load balancer to get an IP or hostname assigned to it, and it + // should be fast when we are not waiting for a load balancer (e.g. on kind). + require.Eventually(t, func() bool { + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + return err == nil + }, 5*time.Minute, 250*time.Millisecond) // Create an impersonation proxy client with that CA data to use for the rest of this test. // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. var impersonationProxyClient *kubernetes.Clientset if env.HasCapability(library.HasExternalLoadBalancerProvider) { - impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, caSecret.Data["ca.crt"]) + impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, caCertPEM) } else { - impersonationProxyClient = impersonationProxyViaSquidClient(caSecret.Data["ca.crt"]) + impersonationProxyClient = impersonationProxyViaSquidClient(caCertPEM) } // Test that the user can perform basic actions through the client with their username and group membership @@ -381,7 +391,7 @@ func TestImpersonationProxy(t *testing.T) { // Check that the generated TLS cert Secret was deleted by the controller. require.Eventually(t, func() bool { - caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) return k8serrors.IsNotFound(err) }, 10*time.Second, 250*time.Millisecond) } From ac404af48f79cb9e97918d70cd8ab083a783e643 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 1 Mar 2021 17:03:05 -0800 Subject: [PATCH 056/203] Add .DS_Store files to .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3a5c3201..88dd571f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,11 @@ # Dependency directories (remove the comment below to include it) # vendor/ -# goland +# GoLand .idea # Intermediate files used by Tilt /hack/lib/tilt/build + +# MacOS Desktop Services Store +.DS_Store From 41140766f032718fbab8935bd739b4909ee377de Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 1 Mar 2021 17:53:26 -0800 Subject: [PATCH 057/203] Add integration test which demonstrates double impersonation We don't support using the impersonate headers through the impersonation proxy yet, so this integration test is a negative test which asserts that we get an error. --- .../concierge_impersonation_proxy_test.go | 104 +++++++++++------- test/library/client.go | 2 +- 2 files changed, 66 insertions(+), 40 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index de113224..437a3f0d 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -33,7 +33,7 @@ const ( // TODO don't hard code "pinniped-concierge-" in these strings. It should be constructed from the env app name. impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" impersonationProxyTLSSecretName = "pinniped-concierge-impersonation-proxy-tls-serving-certificate" //nolint:gosec // this is not a credential - impersonationProxyCASecretName = "pinniped-concierge-impersonation-proxy-ca-certificate" + impersonationProxyCASecretName = "pinniped-concierge-impersonation-proxy-ca-certificate" //nolint:gosec // this is not a credential impersonationProxyLoadBalancerName = "pinniped-concierge-impersonation-proxy-load-balancer" ) @@ -58,31 +58,37 @@ func TestImpersonationProxy(t *testing.T) { // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) - impersonationProxyViaSquidClient := func(caData []byte) *kubernetes.Clientset { - t.Helper() - kubeconfig := &rest.Config{ - Host: fmt.Sprintf("https://%s", proxyServiceEndpoint), + impersonationProxyRestConfig := func(host string, caData []byte, doubleImpersonateUser string) *rest.Config { + config := rest.Config{ + Host: host, TLSClientConfig: rest.TLSClientConfig{Insecure: caData == nil, CAData: caData}, BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), - Proxy: func(req *http.Request) (*url.URL, error) { - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil - }, + } + if doubleImpersonateUser != "" { + config.Impersonate = rest.ImpersonationConfig{UserName: doubleImpersonateUser} + } + return &config + } + + impersonationProxyViaSquidClient := func(caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { + t.Helper() + host := fmt.Sprintf("https://%s", proxyServiceEndpoint) + kubeconfig := impersonationProxyRestConfig(host, caData, doubleImpersonateUser) + kubeconfig.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil } impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") return impersonationProxyClient } - impersonationProxyViaLoadBalancerClient := func(host string, caData []byte) *kubernetes.Clientset { + impersonationProxyViaLoadBalancerClient := func(host string, caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { t.Helper() - kubeconfig := &rest.Config{ - Host: fmt.Sprintf("https://%s", host), - TLSClientConfig: rest.TLSClientConfig{Insecure: caData == nil, CAData: caData}, - BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), - } + host = fmt.Sprintf("https://%s", host) + kubeconfig := impersonationProxyRestConfig(host, caData, doubleImpersonateUser) impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") return impersonationProxyClient @@ -129,7 +135,7 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. - _, err = impersonationProxyViaSquidClient(nil).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidClient(nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableViaSquidError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer). @@ -168,7 +174,7 @@ func TestImpersonationProxy(t *testing.T) { caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName, metav1.GetOptions{}) return err == nil && caSecret != nil && caSecret.Data["ca.crt"] != nil }, 10*time.Second, 250*time.Millisecond) - caCertPEM := caSecret.Data["ca.crt"] + impersonationProxyCACertPEM := caSecret.Data["ca.crt"] // Check that the generated TLS cert Secret was created by the controller. // This could take a while if we are waiting for the load balancer to get an IP or hostname assigned to it, and it @@ -182,9 +188,9 @@ func TestImpersonationProxy(t *testing.T) { // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. var impersonationProxyClient *kubernetes.Clientset if env.HasCapability(library.HasExternalLoadBalancerProvider) { - impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, caCertPEM) + impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, impersonationProxyCACertPEM, "") } else { - impersonationProxyClient = impersonationProxyViaSquidClient(caCertPEM) + impersonationProxyClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "") } // Test that the user can perform basic actions through the client with their username and group membership @@ -216,26 +222,13 @@ func TestImpersonationProxy(t *testing.T) { }) // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding( - t, - rbacv1.Subject{ - Kind: rbacv1.UserKind, - APIGroup: rbacv1.GroupName, - Name: env.TestUser.ExpectedUsername, - }, - rbacv1.RoleRef{ - Kind: "ClusterRole", - APIGroup: rbacv1.GroupName, - Name: "cluster-admin", - }, + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespace.Name, - Verb: "create", - Group: "", - Version: "v1", - Resource: "configmaps", + Namespace: namespace.Name, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", }) // Create and start informer to exercise the "watch" verb for us. @@ -354,6 +347,39 @@ func TestImpersonationProxy(t *testing.T) { require.Len(t, listResult.Items, 0) }) + t.Run("double impersonation is blocked", func(t *testing.T) { + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: env.ConciergeNamespace, Verb: "get", Group: "", Version: "v1", Resource: "secrets", + }) + + // Make a client which will send requests through the impersonation proxy and will also add + // impersonate headers to the request. + var doubleImpersonationClient *kubernetes.Clientset + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + doubleImpersonationClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, impersonationProxyCACertPEM, "other-user-to-impersonate") + } else { + doubleImpersonationClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "other-user-to-impersonate") + } + + // We already know that this Secret exists because we checked above. Now see that we can get it through + // the impersonation proxy without any impersonation headers on the request. + _, err = impersonationProxyClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + require.NoError(t, err) + + // Now we'll see what happens when we add an impersonation header to the request. This should generate a + // request similar to the one above, except that it will have an impersonation header. + _, err = doubleImpersonationClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + // Double impersonation is not supported yet, so we should get an error. + expectedErr := fmt.Sprintf("the server rejected our request for an unknown reason (get secrets %s)", impersonationProxyTLSSecretName) + require.EqualError(t, err, expectedErr) + }) + // Update configuration to force the proxy to disabled mode configMap := configMapForConfig(t, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { @@ -384,7 +410,7 @@ func TestImpersonationProxy(t *testing.T) { require.Eventually(t, func() bool { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. - _, err = impersonationProxyViaSquidClient(nil).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidClient(nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return err.Error() == serviceUnavailableViaSquidError }, 20*time.Second, 500*time.Millisecond) } diff --git a/test/library/client.go b/test/library/client.go index e4fb40a9..bfb92ee8 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -426,7 +426,7 @@ func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldHaveAccessTo *authorizationv1.ResourceAttributes) { t.Helper() client := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() RequireEventuallyWithoutError(t, func() (bool, error) { From a75c2194bcea677bb1204a9e404b4c4cc82d69d9 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 2 Mar 2021 09:31:24 -0800 Subject: [PATCH 058/203] Read the names of the impersonation-related resources from the config They were previously temporarily hardcoded. Now they are set at deploy time via the static ConfigMap in deployment.yaml --- deploy/concierge/deployment.yaml | 4 + internal/config/concierge/config.go | 33 +++-- internal/config/concierge/config_test.go | 140 ++++++++++++++++-- internal/config/concierge/types.go | 10 +- .../controllermanager/prepare_controllers.go | 14 +- .../concierge_impersonation_proxy_test.go | 71 +++++---- 6 files changed, 211 insertions(+), 61 deletions(-) diff --git a/deploy/concierge/deployment.yaml b/deploy/concierge/deployment.yaml index 8e3b67f7..41119243 100644 --- a/deploy/concierge/deployment.yaml +++ b/deploy/concierge/deployment.yaml @@ -42,6 +42,10 @@ data: servingCertificateSecret: (@= defaultResourceNameWithSuffix("api-tls-serving-certificate") @) credentialIssuer: (@= defaultResourceNameWithSuffix("config") @) apiService: (@= defaultResourceNameWithSuffix("api") @) + impersonationConfigMap: (@= defaultResourceNameWithSuffix("impersonation-proxy-config") @) + impersonationLoadBalancerService: (@= defaultResourceNameWithSuffix("impersonation-proxy-load-balancer") @) + impersonationTLSCertificateSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-tls-serving-certificate") @) + impersonationCACertificateSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-ca-certificate") @) labels: (@= json.encode(labels()).rstrip() @) kubeCertAgent: namePrefix: (@= defaultResourceNameWithSuffix("kube-cert-agent-") @) diff --git a/internal/config/concierge/config.go b/internal/config/concierge/config.go index 88d8fe06..b6720cea 100644 --- a/internal/config/concierge/config.go +++ b/internal/config/concierge/config.go @@ -96,17 +96,28 @@ func maybeSetKubeCertAgentDefaults(cfg *KubeCertAgentSpec) { func validateNames(names *NamesConfigSpec) error { missingNames := []string{} if names == nil { - missingNames = append(missingNames, "servingCertificateSecret", "credentialIssuer", "apiService") - } else { - if names.ServingCertificateSecret == "" { - missingNames = append(missingNames, "servingCertificateSecret") - } - if names.CredentialIssuer == "" { - missingNames = append(missingNames, "credentialIssuer") - } - if names.APIService == "" { - missingNames = append(missingNames, "apiService") - } + names = &NamesConfigSpec{} + } + if names.ServingCertificateSecret == "" { + missingNames = append(missingNames, "servingCertificateSecret") + } + if names.CredentialIssuer == "" { + missingNames = append(missingNames, "credentialIssuer") + } + if names.APIService == "" { + missingNames = append(missingNames, "apiService") + } + if names.ImpersonationConfigMap == "" { + missingNames = append(missingNames, "impersonationConfigMap") + } + if names.ImpersonationLoadBalancerService == "" { + missingNames = append(missingNames, "impersonationLoadBalancerService") + } + if names.ImpersonationTLSCertificateSecret == "" { + missingNames = append(missingNames, "impersonationTLSCertificateSecret") + } + if names.ImpersonationCACertificateSecret == "" { + missingNames = append(missingNames, "impersonationCACertificateSecret") } if len(missingNames) > 0 { return constable.Error("missing required names: " + strings.Join(missingNames, ", ")) diff --git a/internal/config/concierge/config_test.go b/internal/config/concierge/config_test.go index 4073d96d..5b8177ff 100644 --- a/internal/config/concierge/config_test.go +++ b/internal/config/concierge/config_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/plog" ) func TestFromPath(t *testing.T) { @@ -21,7 +22,7 @@ func TestFromPath(t *testing.T) { wantError string }{ { - name: "Happy", + name: "Fully filled out", yaml: here.Doc(` --- discovery: @@ -36,13 +37,18 @@ func TestFromPath(t *testing.T) { credentialIssuer: pinniped-config apiService: pinniped-api kubeCertAgentPrefix: kube-cert-agent-prefix + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value labels: myLabelKey1: myLabelValue1 myLabelKey2: myLabelValue2 - KubeCertAgent: + kubeCertAgent: namePrefix: kube-cert-agent-name-prefix- image: kube-cert-agent-image imagePullSecrets: [kube-cert-agent-image-pull-secret] + logLevel: debug `), wantConfig: &Config{ DiscoveryInfo: DiscoveryInfoSpec{ @@ -56,9 +62,13 @@ func TestFromPath(t *testing.T) { }, APIGroupSuffix: stringPtr("some.suffix.com"), NamesConfig: NamesConfigSpec{ - ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", - CredentialIssuer: "pinniped-config", - APIService: "pinniped-api", + ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", + CredentialIssuer: "pinniped-config", + APIService: "pinniped-api", + ImpersonationConfigMap: "impersonationConfigMap-value", + ImpersonationLoadBalancerService: "impersonationLoadBalancerService-value", + ImpersonationTLSCertificateSecret: "impersonationTLSCertificateSecret-value", + ImpersonationCACertificateSecret: "impersonationCACertificateSecret-value", }, Labels: map[string]string{ "myLabelKey1": "myLabelValue1", @@ -69,6 +79,7 @@ func TestFromPath(t *testing.T) { Image: stringPtr("kube-cert-agent-image"), ImagePullSecrets: []string{"kube-cert-agent-image-pull-secret"}, }, + LogLevel: plog.LevelDebug, }, }, { @@ -79,6 +90,10 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantConfig: &Config{ DiscoveryInfo: DiscoveryInfoSpec{ @@ -92,9 +107,13 @@ func TestFromPath(t *testing.T) { }, }, NamesConfig: NamesConfigSpec{ - ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", - CredentialIssuer: "pinniped-config", - APIService: "pinniped-api", + ServingCertificateSecret: "pinniped-concierge-api-tls-serving-certificate", + CredentialIssuer: "pinniped-config", + APIService: "pinniped-api", + ImpersonationConfigMap: "impersonationConfigMap-value", + ImpersonationLoadBalancerService: "impersonationLoadBalancerService-value", + ImpersonationTLSCertificateSecret: "impersonationTLSCertificateSecret-value", + ImpersonationCACertificateSecret: "impersonationCACertificateSecret-value", }, Labels: map[string]string{}, KubeCertAgentConfig: KubeCertAgentSpec{ @@ -104,9 +123,11 @@ func TestFromPath(t *testing.T) { }, }, { - name: "Empty", - yaml: here.Doc(``), - wantError: "validate names: missing required names: servingCertificateSecret, credentialIssuer, apiService", + name: "Empty", + yaml: here.Doc(``), + wantError: "validate names: missing required names: servingCertificateSecret, credentialIssuer, " + + "apiService, impersonationConfigMap, impersonationLoadBalancerService, " + + "impersonationTLSCertificateSecret, impersonationCACertificateSecret", }, { name: "Missing apiService name", @@ -115,6 +136,10 @@ func TestFromPath(t *testing.T) { names: servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate names: missing required names: apiService", }, @@ -125,6 +150,10 @@ func TestFromPath(t *testing.T) { names: servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate names: missing required names: credentialIssuer", }, @@ -135,9 +164,82 @@ func TestFromPath(t *testing.T) { names: credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate names: missing required names: servingCertificateSecret", }, + { + name: "Missing impersonationConfigMap name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + `), + wantError: "validate names: missing required names: impersonationConfigMap", + }, + { + name: "Missing impersonationLoadBalancerService name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + `), + wantError: "validate names: missing required names: impersonationLoadBalancerService", + }, + { + name: "Missing impersonationTLSCertificateSecret name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + `), + wantError: "validate names: missing required names: impersonationTLSCertificateSecret", + }, + { + name: "Missing impersonationCACertificateSecret name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + `), + wantError: "validate names: missing required names: impersonationCACertificateSecret", + }, + { + name: "Missing several required names", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationLoadBalancerService: impersonationLoadBalancerService-value + `), + wantError: "validate names: missing required names: impersonationConfigMap, " + + "impersonationTLSCertificateSecret, impersonationCACertificateSecret", + }, { name: "InvalidDurationRenewBefore", yaml: here.Doc(` @@ -150,6 +252,10 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate api: durationSeconds cannot be smaller than renewBeforeSeconds", }, @@ -165,6 +271,10 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate api: renewBefore must be positive", }, @@ -180,6 +290,10 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate api: renewBefore must be positive", }, @@ -196,6 +310,10 @@ func TestFromPath(t *testing.T) { servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate credentialIssuer: pinniped-config apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value `), wantError: "validate apiGroupSuffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", }, diff --git a/internal/config/concierge/types.go b/internal/config/concierge/types.go index 1f402a34..2a151274 100644 --- a/internal/config/concierge/types.go +++ b/internal/config/concierge/types.go @@ -33,9 +33,13 @@ type APIConfigSpec struct { // NamesConfigSpec configures the names of some Kubernetes resources for the Concierge. type NamesConfigSpec struct { - ServingCertificateSecret string `json:"servingCertificateSecret"` - CredentialIssuer string `json:"credentialIssuer"` - APIService string `json:"apiService"` + ServingCertificateSecret string `json:"servingCertificateSecret"` + CredentialIssuer string `json:"credentialIssuer"` + APIService string `json:"apiService"` + ImpersonationConfigMap string `json:"impersonationConfigMap"` + ImpersonationLoadBalancerService string `json:"impersonationLoadBalancerService"` + ImpersonationTLSCertificateSecret string `json:"impersonationTLSCertificateSecret"` + ImpersonationCACertificateSecret string `json:"impersonationCACertificateSecret"` } // ServingCertificateConfigSpec contains the configuration knobs for the API's diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index ce63fc91..de1e5a30 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -292,20 +292,24 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { WithController( impersonatorconfig.NewImpersonatorConfigController( c.ServerInstallationInfo.Namespace, - "pinniped-concierge-impersonation-proxy-config", // TODO this string should come from `c.NamesConfig` + c.NamesConfig.ImpersonationConfigMap, client.Kubernetes, informers.installationNamespaceK8s.Core().V1().ConfigMaps(), informers.installationNamespaceK8s.Core().V1().Services(), informers.installationNamespaceK8s.Core().V1().Secrets(), controllerlib.WithInformer, controllerlib.WithInitialEvent, - "pinniped-concierge-impersonation-proxy-load-balancer", // TODO this string should come from `c.NamesConfig` - "pinniped-concierge-impersonation-proxy-tls-serving-certificate", // TODO this string should come from `c.NamesConfig` - "pinniped-concierge-impersonation-proxy-ca-certificate", // TODO this string should come from `c.NamesConfig` + c.NamesConfig.ImpersonationLoadBalancerService, + c.NamesConfig.ImpersonationTLSCertificateSecret, + c.NamesConfig.ImpersonationCACertificateSecret, c.Labels, tls.Listen, func() (http.Handler, error) { - impersonationProxyHandler, err := impersonator.New(c.AuthenticatorCache, c.LoginJSONDecoder, klogr.New().WithName("impersonation-proxy")) + impersonationProxyHandler, err := impersonator.New( + c.AuthenticatorCache, + c.LoginJSONDecoder, + klogr.New().WithName("impersonation-proxy"), + ) if err != nil { return nil, fmt.Errorf("could not create impersonation proxy: %w", err) } diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 437a3f0d..ab67037b 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -29,14 +29,6 @@ import ( "go.pinniped.dev/test/library" ) -const ( - // TODO don't hard code "pinniped-concierge-" in these strings. It should be constructed from the env app name. - impersonationProxyConfigMapName = "pinniped-concierge-impersonation-proxy-config" - impersonationProxyTLSSecretName = "pinniped-concierge-impersonation-proxy-tls-serving-certificate" //nolint:gosec // this is not a credential - impersonationProxyCASecretName = "pinniped-concierge-impersonation-proxy-ca-certificate" //nolint:gosec // this is not a credential - impersonationProxyLoadBalancerName = "pinniped-concierge-impersonation-proxy-load-balancer" -) - // Note that this test supports being run on all of our integration test cluster types: // - load balancers not supported, has squid proxy (e.g. kind) // - load balancers supported, has squid proxy (e.g. EKS) @@ -94,11 +86,11 @@ func TestImpersonationProxy(t *testing.T) { return impersonationProxyClient } - oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName, metav1.GetOptions{}) + oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName(env), metav1.GetOptions{}) if !k8serrors.IsNotFound(err) { require.NoError(t, err) // other errors aside from NotFound are unexpected t.Logf("stashing a pre-existing configmap %s", oldConfigMap.Name) - require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName, metav1.DeleteOptions{})) + require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{})) } impersonationProxyLoadBalancerIngress := "" @@ -106,13 +98,14 @@ func TestImpersonationProxy(t *testing.T) { if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test // Check that load balancer has been created. library.RequireEventuallyWithoutError(t, func() (bool, error) { - return hasImpersonationProxyLoadBalancerService(ctx, adminClient, env.ConciergeNamespace) + return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) }, 10*time.Second, 500*time.Millisecond) + // TODO this information should come from the CredentialIssuer status once that is implemented // Wait for the load balancer to get an ingress and make a note of its address. var ingress *corev1.LoadBalancerIngress library.RequireEventuallyWithoutError(t, func() (bool, error) { - ingress, err = getImpersonationProxyLoadBalancerIngress(ctx, adminClient, env.ConciergeNamespace) + ingress, err = getImpersonationProxyLoadBalancerIngress(ctx, env, adminClient) if err != nil { return false, err } @@ -131,7 +124,7 @@ func TestImpersonationProxy(t *testing.T) { // Check that no load balancer has been created. library.RequireNeverWithoutError(t, func() (bool, error) { - return hasImpersonationProxyLoadBalancerService(ctx, adminClient, env.ConciergeNamespace) + return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. @@ -139,7 +132,7 @@ func TestImpersonationProxy(t *testing.T) { require.EqualError(t, err, serviceUnavailableViaSquidError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer). - configMap := configMapForConfig(t, impersonator.Config{ + configMap := configMapForConfig(t, env, impersonator.Config{ Mode: impersonator.ModeEnabled, Endpoint: proxyServiceEndpoint, TLS: nil, @@ -152,8 +145,8 @@ func TestImpersonationProxy(t *testing.T) { t.Cleanup(func() { ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName) - err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName, metav1.DeleteOptions{}) + t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName(env)) + err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{}) require.NoError(t, err) if len(oldConfigMap.Data) != 0 { @@ -171,7 +164,7 @@ func TestImpersonationProxy(t *testing.T) { // TODO We should be getting the CA data from the CredentialIssuer's status instead, once that is implemented. var caSecret *corev1.Secret require.Eventually(t, func() bool { - caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName, metav1.GetOptions{}) + caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) return err == nil && caSecret != nil && caSecret.Data["ca.crt"] != nil }, 10*time.Second, 250*time.Millisecond) impersonationProxyCACertPEM := caSecret.Data["ca.crt"] @@ -180,7 +173,7 @@ func TestImpersonationProxy(t *testing.T) { // This could take a while if we are waiting for the load balancer to get an IP or hostname assigned to it, and it // should be fast when we are not waiting for a load balancer (e.g. on kind). require.Eventually(t, func() bool { - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) return err == nil }, 5*time.Minute, 250*time.Millisecond) @@ -209,7 +202,7 @@ func TestImpersonationProxy(t *testing.T) { // Try more Kube API verbs through the impersonation proxy. t.Run("watching all the basic verbs", func(t *testing.T) { - // Create a namespace, because it will be easier to exercise deletecollection if we have a namespace. + // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, }, metav1.CreateOptions{}) @@ -369,19 +362,19 @@ func TestImpersonationProxy(t *testing.T) { // We already know that this Secret exists because we checked above. Now see that we can get it through // the impersonation proxy without any impersonation headers on the request. - _, err = impersonationProxyClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + _, err = impersonationProxyClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.NoError(t, err) // Now we'll see what happens when we add an impersonation header to the request. This should generate a // request similar to the one above, except that it will have an impersonation header. - _, err = doubleImpersonationClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + _, err = doubleImpersonationClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) // Double impersonation is not supported yet, so we should get an error. - expectedErr := fmt.Sprintf("the server rejected our request for an unknown reason (get secrets %s)", impersonationProxyTLSSecretName) + expectedErr := fmt.Sprintf("the server rejected our request for an unknown reason (get secrets %s)", impersonationProxyTLSSecretName(env)) require.EqualError(t, err, expectedErr) }) // Update configuration to force the proxy to disabled mode - configMap := configMapForConfig(t, impersonator.Config{Mode: impersonator.ModeDisabled}) + configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { t.Logf("creating configmap %s", configMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) @@ -396,7 +389,7 @@ func TestImpersonationProxy(t *testing.T) { // The load balancer should not exist after we disable the impersonation proxy. // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). library.RequireEventuallyWithoutError(t, func() (bool, error) { - hasService, err := hasImpersonationProxyLoadBalancerService(ctx, adminClient, env.ConciergeNamespace) + hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) return !hasService, err }, time.Minute, 500*time.Millisecond) } @@ -417,17 +410,17 @@ func TestImpersonationProxy(t *testing.T) { // Check that the generated TLS cert Secret was deleted by the controller. require.Eventually(t, func() bool { - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName, metav1.GetOptions{}) + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) return k8serrors.IsNotFound(err) }, 10*time.Second, 250*time.Millisecond) } -func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigMap { +func configMapForConfig(t *testing.T, env *library.TestEnv, config impersonator.Config) corev1.ConfigMap { configString, err := yaml.Marshal(config) require.NoError(t, err) configMap := corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ - Name: impersonationProxyConfigMapName, + Name: impersonationProxyConfigMapName(env), }, Data: map[string]string{ "config.yaml": string(configString), @@ -435,8 +428,8 @@ func configMapForConfig(t *testing.T, config impersonator.Config) corev1.ConfigM return configMap } -func hasImpersonationProxyLoadBalancerService(ctx context.Context, client kubernetes.Interface, namespace string) (bool, error) { - service, err := client.CoreV1().Services(namespace).Get(ctx, impersonationProxyLoadBalancerName, metav1.GetOptions{}) +func hasImpersonationProxyLoadBalancerService(ctx context.Context, env *library.TestEnv, client kubernetes.Interface) (bool, error) { + service, err := client.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) if k8serrors.IsNotFound(err) { return false, nil } @@ -446,8 +439,8 @@ func hasImpersonationProxyLoadBalancerService(ctx context.Context, client kubern return service.Spec.Type == corev1.ServiceTypeLoadBalancer, nil } -func getImpersonationProxyLoadBalancerIngress(ctx context.Context, client kubernetes.Interface, namespace string) (*corev1.LoadBalancerIngress, error) { - service, err := client.CoreV1().Services(namespace).Get(ctx, impersonationProxyLoadBalancerName, metav1.GetOptions{}) +func getImpersonationProxyLoadBalancerIngress(ctx context.Context, env *library.TestEnv, client kubernetes.Interface) (*corev1.LoadBalancerIngress, error) { + service, err := client.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) if err != nil { return nil, err } @@ -460,3 +453,19 @@ func getImpersonationProxyLoadBalancerIngress(ctx context.Context, client kubern } return &ingresses[0], nil } + +func impersonationProxyConfigMapName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-config" +} + +func impersonationProxyTLSSecretName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-tls-serving-certificate" +} + +func impersonationProxyCASecretName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-ca-certificate" +} + +func impersonationProxyLoadBalancerName(env *library.TestEnv) string { + return env.ConciergeAppName + "-impersonation-proxy-load-balancer" +} From 4c68050706fb0dfff4f7ae03ec318676aba07c3d Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 2 Mar 2021 14:56:54 -0800 Subject: [PATCH 059/203] Allow all headers besides impersonation-* through impersonation proxy --- .../concierge/impersonator/impersonator.go | 38 ++++++----------- .../impersonator/impersonator_test.go | 42 ++++++++++++------- 2 files changed, 38 insertions(+), 42 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index e4be50c4..c189daf0 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "regexp" "strings" "time" @@ -27,16 +28,8 @@ import ( "go.pinniped.dev/internal/kubeclient" ) -// allowedHeaders are the set of HTTP headers that are allowed to be forwarded through the impersonation proxy. -//nolint: gochecknoglobals -var allowedHeaders = []string{ - "Accept", - "Accept-Encoding", - "User-Agent", - "Connection", - "Upgrade", - "Content-Type", -} +// nolint: gochecknoglobals +var impersonateHeaderRegex = regexp.MustCompile("Impersonate-.*") type proxy struct { cache *authncache.Cache @@ -157,17 +150,9 @@ type httpError struct { func (e *httpError) Error() string { return e.message } func ensureNoImpersonationHeaders(r *http.Request) error { - if _, ok := r.Header[transport.ImpersonateUserHeader]; ok { - return fmt.Errorf("%q header already exists", transport.ImpersonateUserHeader) - } - - if _, ok := r.Header[transport.ImpersonateGroupHeader]; ok { - return fmt.Errorf("%q header already exists", transport.ImpersonateGroupHeader) - } - - for header := range r.Header { - if strings.HasPrefix(header, transport.ImpersonateUserExtraHeaderPrefix) { - return fmt.Errorf("%q header already exists", transport.ImpersonateUserExtraHeaderPrefix) + for key := range r.Header { + if impersonateHeaderRegex.MatchString(key) { + return fmt.Errorf("%q header already exists", key) } } @@ -209,11 +194,12 @@ func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header //nolint:bodyclose // We return a nil http.Response above, so there is nothing to close. _, _ = transport.NewImpersonatingRoundTripper(impersonateConfig, impersonateHeaderSpy).RoundTrip(fakeReq) - // Copy over the allowed header values from the original request to the new request. - for _, header := range allowedHeaders { - values := requestHeaders.Values(header) - for i := range values { - newHeaders.Add(header, values[i]) + // Copy over all headers except the Authorization header from the original request to the new request. + for key := range requestHeaders { + if key != "Authorization" { + for _, val := range requestHeaders.Values(key) { + newHeaders.Add(key, val) + } } } diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 3e590ccf..7a7124b2 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -132,7 +132,16 @@ func TestImpersonator(t *testing.T) { request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}), wantHTTPBody: "impersonation header already exists\n", wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Extra-\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Extra-something\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + }, + { + name: "Impersonate-* header already in request", + request: newRequest(map[string][]string{ + "Impersonate-Something": {"some-newfangled-impersonate-header"}, + }), + wantHTTPBody: "impersonation header already exists\n", + wantHTTPStatus: http.StatusBadRequest, + wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Something\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, }, { name: "missing authorization header", @@ -198,15 +207,15 @@ func TestImpersonator(t *testing.T) { { name: "token validates", request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, - "User-Agent": {"test-user-agent"}, - "Accept": {"some-accepted-format"}, - "Accept-Encoding": {"some-accepted-encoding"}, - "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` - "Upgrade": {"some-upgrade"}, - "Content-Type": {"some-type"}, - "Content-Length": {"some-length"}, - "Malicious-Header": {"test-header-value-1"}, // this header should not be forwarded to the Kube API server + "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through }), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { userInfo := user.DefaultInfo{ @@ -230,6 +239,7 @@ func TestImpersonator(t *testing.T) { "Connection": {"Upgrade"}, "Upgrade": {"some-upgrade"}, "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, @@ -238,9 +248,8 @@ func TestImpersonator(t *testing.T) { { name: "token validates and the kube API request returns an error", request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, - "Malicious-Header": {"test-header-value-1"}, - "User-Agent": {"test-user-agent"}, + "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, + "User-Agent": {"test-user-agent"}, }), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { userInfo := user.DefaultInfo{ @@ -269,9 +278,9 @@ func TestImpersonator(t *testing.T) { name: "token validates with custom api group", apiGroupOverride: customAPIGroup, request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}, - "Malicious-Header": {"test-header-value-1"}, - "User-Agent": {"test-user-agent"}, + "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}, + "Other-Header": {"test-header-value-1"}, + "User-Agent": {"test-user-agent"}, }), expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { userInfo := user.DefaultInfo{ @@ -291,6 +300,7 @@ func TestImpersonator(t *testing.T) { "Impersonate-Group": {"test-group-1", "test-group-2"}, "Impersonate-User": {"test-user"}, "User-Agent": {"test-user-agent"}, + "Other-Header": {"test-header-value-1"}, }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, From 84cc42b2caf1aca6836c249c3b58986c557f806b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 2 Mar 2021 12:23:32 -0800 Subject: [PATCH 060/203] Remove `tls` field from the impersonator config - Decided that we're not going to implement this now, although we may decide to add it in the future --- internal/concierge/impersonator/config.go | 18 ------------------ internal/concierge/impersonator/config_test.go | 15 ++------------- .../concierge_impersonation_proxy_test.go | 1 - 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go index 20d97647..a7bb8c2c 100644 --- a/internal/concierge/impersonator/config.go +++ b/internal/concierge/impersonator/config.go @@ -27,20 +27,6 @@ const ( ConfigMapDataKey = "config.yaml" ) -// When specified, both CertificateAuthoritySecretName and TLSSecretName are required. They may be specified to -// both point at the same Secret or to point at different Secrets. -type TLSConfig struct { - // CertificateAuthoritySecretName contains the name of a namespace-local Secret resource. The corresponding Secret - // must contain a key called "ca.crt" whose value is the CA certificate which clients should trust when connecting - // to the impersonation proxy. - CertificateAuthoritySecretName string `json:"certificateAuthoritySecretName"` - - // TLSSecretName contains the name of a namespace-local Secret resource. The corresponding Secret must be of type - // "kubernetes.io/tls" and contain keys called "tls.crt" and "tls.key" whose values are the TLS certificate and - // private key that will be used by the impersonation proxy to serve its endpoints. - TLSSecretName string `json:"tlsSecretName"` -} - type Config struct { // Enable or disable the impersonation proxy. Optional. Defaults to ModeAuto. Mode Mode `json:"mode,omitempty"` @@ -53,10 +39,6 @@ type Config struct { // for clients to use from outside the cluster. E.g. myhost.mycompany.com:8443. Clients should assume that they should // connect via HTTPS to this service. Endpoint string `json:"endpoint,omitempty"` - - // The TLS configuration of the impersonation proxy's endpoints. Optional. When not specified, a CA and TLS - // certificate will be automatically created based on the Endpoint setting. - TLS *TLSConfig `json:"tls,omitempty"` } func NewConfig() *Config { diff --git a/internal/concierge/impersonator/config_test.go b/internal/concierge/impersonator/config_test.go index 9b12b6ba..2e96a9af 100644 --- a/internal/concierge/impersonator/config_test.go +++ b/internal/concierge/impersonator/config_test.go @@ -33,20 +33,13 @@ func TestConfigFromConfigMap(t *testing.T) { Data: map[string]string{ "config.yaml": here.Doc(` mode: enabled - endpoint: https://proxy.example.com:8443/ - tls: - certificateAuthoritySecretName: my-ca-crt - tlsSecretName: my-tls-certificate-and-key + endpoint: proxy.example.com:8443 `), }, }, wantConfig: &Config{ Mode: "enabled", - Endpoint: "https://proxy.example.com:8443/", - TLS: &TLSConfig{ - CertificateAuthoritySecretName: "my-ca-crt", - TLSSecretName: "my-tls-certificate-and-key", - }, + Endpoint: "proxy.example.com:8443", }, }, { @@ -61,7 +54,6 @@ func TestConfigFromConfigMap(t *testing.T) { wantConfig: &Config{ Mode: "auto", Endpoint: "", - TLS: nil, }, }, { @@ -76,7 +68,6 @@ func TestConfigFromConfigMap(t *testing.T) { wantConfig: &Config{ Mode: "enabled", Endpoint: "", - TLS: nil, }, }, { @@ -91,7 +82,6 @@ func TestConfigFromConfigMap(t *testing.T) { wantConfig: &Config{ Mode: "disabled", Endpoint: "", - TLS: nil, }, }, { @@ -106,7 +96,6 @@ func TestConfigFromConfigMap(t *testing.T) { wantConfig: &Config{ Mode: "auto", Endpoint: "", - TLS: nil, }, }, { diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index ab67037b..f10f87af 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -135,7 +135,6 @@ func TestImpersonationProxy(t *testing.T) { configMap := configMapForConfig(t, env, impersonator.Config{ Mode: impersonator.ModeEnabled, Endpoint: proxyServiceEndpoint, - TLS: nil, }) t.Logf("creating configmap %s", configMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) From 1ad2c385091e0140de8a4327bce4fd98ad7cb459 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 2 Mar 2021 14:48:58 -0800 Subject: [PATCH 061/203] Impersonation controller updates CredentialIssuer on every call to Sync - This commit does not include the updates that we plan to make to the `status.strategies[].frontend` field of the CredentialIssuer. That will come in a future commit. --- .../impersonatorconfig/impersonator_config.go | 173 +++++++++--- .../impersonator_config_test.go | 255 +++++++++++++++--- .../controllermanager/prepare_controllers.go | 5 +- 3 files changed, 354 insertions(+), 79 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 661fd086..2200f968 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -20,14 +20,18 @@ import ( v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/intstr" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" + "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/clusterhost" "go.pinniped.dev/internal/concierge/impersonator" pinnipedcontroller "go.pinniped.dev/internal/controller" + "go.pinniped.dev/internal/controller/issuerconfig" "go.pinniped.dev/internal/controllerlib" "go.pinniped.dev/internal/plog" ) @@ -40,21 +44,31 @@ const ( caCrtKey = "ca.crt" caKeyKey = "ca.key" appLabelKey = "app" + + // TODO move these to the api package after resolving an upcoming merge. + PendingStrategyReason = v1alpha1.StrategyReason("Pending") + ErrorDuringSetupStrategyReason = v1alpha1.StrategyReason("ErrorDuringSetup") ) type impersonatorConfigController struct { namespace string configMapResourceName string - k8sClient kubernetes.Interface - configMapsInformer corev1informers.ConfigMapInformer - servicesInformer corev1informers.ServiceInformer - secretsInformer corev1informers.SecretInformer + credentialIssuerResourceName string generatedLoadBalancerServiceName string tlsSecretName string caSecretName string - labels map[string]string - startTLSListenerFunc StartTLSListenerFunc - httpHandlerFactory func() (http.Handler, error) + + k8sClient kubernetes.Interface + pinnipedAPIClient pinnipedclientset.Interface + + configMapsInformer corev1informers.ConfigMapInformer + servicesInformer corev1informers.ServiceInformer + secretsInformer corev1informers.SecretInformer + + labels map[string]string + clock clock.Clock + startTLSListenerFunc StartTLSListenerFunc + httpHandlerFactory func() (http.Handler, error) server *http.Server hasControlPlaneNodes *bool @@ -67,7 +81,9 @@ type StartTLSListenerFunc func(network, listenAddress string, config *tls.Config func NewImpersonatorConfigController( namespace string, configMapResourceName string, + credentialIssuerResourceName string, k8sClient kubernetes.Interface, + pinnipedAPIClient pinnipedclientset.Interface, configMapsInformer corev1informers.ConfigMapInformer, servicesInformer corev1informers.ServiceInformer, secretsInformer corev1informers.SecretInformer, @@ -77,6 +93,7 @@ func NewImpersonatorConfigController( tlsSecretName string, caSecretName string, labels map[string]string, + clock clock.Clock, startTLSListenerFunc StartTLSListenerFunc, httpHandlerFactory func() (http.Handler, error), ) controllerlib.Controller { @@ -86,14 +103,17 @@ func NewImpersonatorConfigController( Syncer: &impersonatorConfigController{ namespace: namespace, configMapResourceName: configMapResourceName, - k8sClient: k8sClient, - configMapsInformer: configMapsInformer, - servicesInformer: servicesInformer, - secretsInformer: secretsInformer, + credentialIssuerResourceName: credentialIssuerResourceName, generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, tlsSecretName: tlsSecretName, caSecretName: caSecretName, + k8sClient: k8sClient, + pinnipedAPIClient: pinnipedAPIClient, + configMapsInformer: configMapsInformer, + servicesInformer: servicesInformer, + secretsInformer: secretsInformer, labels: labels, + clock: clock, startTLSListenerFunc: startTLSListenerFunc, httpHandlerFactory: httpHandlerFactory, }, @@ -126,11 +146,37 @@ func NewImpersonatorConfigController( func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error { plog.Debug("Starting impersonatorConfigController Sync") - ctx := syncCtx.Context + strategy, err := c.doSync(syncCtx.Context) + + if err != nil { + strategy = &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: ErrorDuringSetupStrategyReason, + Message: err.Error(), + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + } + + updateStrategyErr := c.updateStrategy(syncCtx.Context, strategy) + if updateStrategyErr != nil { + plog.Error("error while updating the CredentialIssuer status", err) + if err == nil { + err = updateStrategyErr + } + } + + if err == nil { + plog.Debug("Successfully finished impersonatorConfigController Sync") + } + return err +} + +func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.CredentialIssuerStrategy, error) { config, err := c.loadImpersonationProxyConfiguration() if err != nil { - return err + return nil, err } // Make a live API call to avoid the cost of having an informer watch all node changes on the cluster, @@ -140,7 +186,7 @@ func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error if c.hasControlPlaneNodes == nil { hasControlPlaneNodes, err := clusterhost.New(c.k8sClient).HasControlPlaneNodes(ctx) if err != nil { - return err + return nil, err } c.hasControlPlaneNodes = &hasControlPlaneNodes plog.Debug("Queried for control plane nodes", "foundControlPlaneNodes", hasControlPlaneNodes) @@ -148,39 +194,38 @@ func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error if c.shouldHaveImpersonator(config) { if err = c.ensureImpersonatorIsStarted(); err != nil { - return err + return nil, err } } else { if err = c.ensureImpersonatorIsStopped(); err != nil { - return err + return nil, err } } if c.shouldHaveLoadBalancer(config) { if err = c.ensureLoadBalancerIsStarted(ctx); err != nil { - return err + return nil, err } } else { if err = c.ensureLoadBalancerIsStopped(ctx); err != nil { - return err + return nil, err } } + waitingForLoadBalancer := false if c.shouldHaveTLSSecret(config) { var impersonationCA *certauthority.CA if impersonationCA, err = c.ensureCASecretIsCreated(ctx); err != nil { - return err + return nil, err } - if err = c.ensureTLSSecret(ctx, config, impersonationCA); err != nil { - return err + if waitingForLoadBalancer, err = c.ensureTLSSecret(ctx, config, impersonationCA); err != nil { + return nil, err } } else if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { - return err + return nil, err } - plog.Debug("Successfully finished impersonatorConfigController Sync") - - return nil + return c.doSyncResult(waitingForLoadBalancer, config), nil } func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*impersonator.Config, error) { @@ -212,7 +257,19 @@ func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*i } func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { - return (config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes) || config.Mode == impersonator.ModeEnabled + return c.enabledByAutoMode(config) || config.Mode == impersonator.ModeEnabled +} + +func (c *impersonatorConfigController) enabledByAutoMode(config *impersonator.Config) bool { + return config.Mode == impersonator.ModeAuto && !*c.hasControlPlaneNodes +} + +func (c *impersonatorConfigController) disabledByAutoMode(config *impersonator.Config) bool { + return config.Mode == impersonator.ModeAuto && *c.hasControlPlaneNodes +} + +func (c *impersonatorConfigController) disabledExplicitly(config *impersonator.Config) bool { + return config.Mode == impersonator.ModeDisabled } func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonator.Config) bool { @@ -223,6 +280,10 @@ func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator. return c.shouldHaveImpersonator(config) } +func (c *impersonatorConfigController) updateStrategy(ctx context.Context, strategy *v1alpha1.CredentialIssuerStrategy) error { + return issuerconfig.UpdateStrategy(ctx, c.credentialIssuerResourceName, c.labels, c.pinnipedAPIClient, *strategy) +} + func (c *impersonatorConfigController) loadBalancerExists() (bool, error) { _, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) notFound := k8serrors.IsNotFound(err) @@ -342,17 +403,17 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.C return c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) } -func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, config *impersonator.Config, ca *certauthority.CA) error { +func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, config *impersonator.Config, ca *certauthority.CA) (bool, error) { secretFromInformer, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) notFound := k8serrors.IsNotFound(err) if !notFound && err != nil { - return err + return false, err } if !notFound { secretWasDeleted, err := c.deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx, config, ca, secretFromInformer) if err != nil { - return err + return false, err } // If it was deleted by the above call, then set it to nil. This allows us to avoid waiting // for the informer cache to update before deciding to proceed to create the new Secret below. @@ -457,35 +518,36 @@ func certHostnameAndIPMatchDesiredState(desiredIP net.IP, actualIPs []net.IP, de return false } -func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret, ca *certauthority.CA) error { +func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret, ca *certauthority.CA) (bool, error) { if secret != nil { err := c.loadTLSCertFromSecret(secret) if err != nil { - return err + return false, err } - return nil + return false, nil } ip, hostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) if err != nil { - return err + return false, err } if !nameIsReady { // Sync will get called again when the load balancer is updated with its ingress info, so this is not an error. - return nil + // Return "true" meaning that we are waiting for the load balancer. + return true, nil } newTLSSecret, err := c.createNewTLSSecret(ctx, ca, ip, hostname) if err != nil { - return err + return false, err } err = c.loadTLSCertFromSecret(newTLSSecret) if err != nil { - return err + return false, err } - return nil + return false, nil } func (c *impersonatorConfigController) ensureCASecretIsCreated(ctx context.Context) (*certauthority.CA, error) { @@ -672,6 +734,43 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return nil } +func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, config *impersonator.Config) *v1alpha1.CredentialIssuerStrategy { + switch { + case waitingForLoadBalancer: + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: PendingStrategyReason, + Message: "waiting for load balancer Service to be assigned IP or hostname", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + case c.disabledExplicitly(config): + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.DisabledStrategyReason, + Message: "impersonation proxy was explicitly disabled by configuration", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + case c.disabledByAutoMode(config): + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.DisabledStrategyReason, + Message: "automatically determined that impersonation proxy should be disabled", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + default: + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.ListeningStrategyReason, + Message: "impersonation proxy is ready to accept client connections", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + } + } +} + func (c *impersonatorConfigController) setTLSCert(cert *tls.Certificate) { c.tlsCertMutex.Lock() defer c.tlsCertMutex.Unlock() diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 3b779179..a1c27a90 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -26,12 +26,15 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/clock" kubeinformers "k8s.io/client-go/informers" corev1informers "k8s.io/client-go/informers/core/v1" kubernetesfake "k8s.io/client-go/kubernetes/fake" coretesting "k8s.io/client-go/testing" "k8s.io/client-go/tools/cache" + "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/controllerlib" "go.pinniped.dev/internal/testutil" @@ -86,6 +89,8 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { _ = NewImpersonatorConfigController( installedInNamespace, configMapResourceName, + "", + nil, nil, configMapsInformer, servicesInformer, @@ -98,6 +103,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { nil, nil, nil, + nil, ) configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) servicesInformerFilter = observableWithInformerOption.GetFilterForInformer(servicesInformer) @@ -273,6 +279,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" + const credentialIssuerResourceName = "some-credential-issuer-resource-name" const loadBalancerServiceName = "some-service-resource-name" const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential const caSecretName = "some-ca-secret-name" @@ -284,6 +291,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var subject controllerlib.Controller var kubeAPIClient *kubernetesfake.Clientset + var pinnipedAPIClient *pinnipedfake.Clientset var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory var timeoutContext context.Context @@ -294,6 +302,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var startTLSListenerUponCloseError error var httpHandlerFactoryFuncError error var startedTLSListener net.Listener + var frozenNow time.Time var startTLSListenerFunc = func(network, listenAddress string, config *tls.Config) (net.Listener, error) { startTLSListenerFuncWasCalled++ @@ -423,7 +432,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { subject = NewImpersonatorConfigController( installedInNamespace, configMapResourceName, + credentialIssuerResourceName, kubeAPIClient, + pinnipedAPIClient, kubeInformers.Core().V1().ConfigMaps(), kubeInformers.Core().V1().Services(), kubeInformers.Core().V1().Secrets(), @@ -433,6 +444,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { tlsSecretName, caSecretName, labels, + clock.NewFakeClock(frozenNow), startTLSListenerFunc, func() (http.Handler, error) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { @@ -652,6 +664,74 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ) } + var newSuccessStrategy = func() v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.ListeningStrategyReason, + Message: "impersonation proxy is ready to accept client connections", + LastUpdateTime: metav1.NewTime(frozenNow), + } + } + + var newAutoDisabledStrategy = func() v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.DisabledStrategyReason, + Message: "automatically determined that impersonation proxy should be disabled", + LastUpdateTime: metav1.NewTime(frozenNow), + } + } + + var newManuallyDisabledStrategy = func() v1alpha1.CredentialIssuerStrategy { + s := newAutoDisabledStrategy() + s.Message = "impersonation proxy was explicitly disabled by configuration" + return s + } + + var newPendingStrategy = func() v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: PendingStrategyReason, + Message: "waiting for load balancer Service to be assigned IP or hostname", + LastUpdateTime: metav1.NewTime(frozenNow), + } + } + + var newErrorStrategy = func(msg string) v1alpha1.CredentialIssuerStrategy { + return v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: ErrorDuringSetupStrategyReason, + Message: msg, + LastUpdateTime: metav1.NewTime(frozenNow), + } + } + + var requireCredentialIssuer = func(expectedStrategy v1alpha1.CredentialIssuerStrategy) { + // Rather than looking at the specific API actions on pinnipedAPIClient, we just look + // at the final result here. + // This is because the implementation is using a helper from another package to create + // and update the CredentialIssuer, and the specific API actions performed by that + // implementation are pretty complex and are already tested by its own unit tests. + // As long as we get the final result that we wanted then we are happy for the purposes + // of this test. + credentialIssuerObj, err := pinnipedAPIClient.Tracker().Get( + schema.GroupVersionResource{ + Group: v1alpha1.SchemeGroupVersion.Group, + Version: v1alpha1.SchemeGroupVersion.Version, + Resource: "credentialissuers", + }, "", credentialIssuerResourceName, + ) + r.NoError(err) + credentialIssuer, ok := credentialIssuerObj.(*v1alpha1.CredentialIssuer) + r.True(ok, "should have been able to cast this obj to CredentialIssuer: %v", credentialIssuerObj) + r.Equal(labels, credentialIssuer.Labels) + r.Equal([]v1alpha1.CredentialIssuerStrategy{expectedStrategy}, credentialIssuer.Status.Strategies) + } + var requireLoadBalancerWasCreated = func(action coretesting.Action) { createAction, ok := action.(coretesting.CreateAction) r.True(ok, "should have been able to cast this action to CreateAction: %v", action) @@ -738,6 +818,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { kubeinformers.WithNamespace(installedInNamespace), ) kubeAPIClient = kubernetesfake.NewSimpleClientset() + pinnipedAPIClient = pinnipedfake.NewSimpleClientset() + frozenNow = time.Date(2021, time.March, 2, 7, 42, 0, 0, time.Local) }) it.After(func() { @@ -747,7 +829,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the ConfigMap does not yet exist in the installation namespace or it was deleted (defaults to auto mode)", func() { it.Before(func() { - addImpersonatorConfigMapToTracker("some-other-configmap", "foo: bar", kubeInformerClient) + addImpersonatorConfigMapToTracker("some-other-unrelated-configmap", "foo: bar", kubeInformerClient) }) when("there are visible control plane nodes", func() { @@ -761,6 +843,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) + requireCredentialIssuer(newAutoDisabledStrategy()) }) }) @@ -780,6 +863,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[1]) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newAutoDisabledStrategy()) }) }) @@ -796,10 +880,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) }) }) - when("there are not visible control plane nodes and a load balancer already exists without an IP", func() { + when("there are not visible control plane nodes and a load balancer already exists without an IP/hostname", func() { it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeInformerClient) @@ -813,6 +898,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireCredentialIssuer(newPendingStrategy()) }) }) @@ -830,6 +916,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireCredentialIssuer(newPendingStrategy()) }) }) @@ -847,6 +934,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireCredentialIssuer(newErrorStrategy("could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name")) }) }) @@ -865,6 +953,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, "127.0.0.123", map[string]string{"127.0.0.123:443": testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) it("keeps the secret around after resync", func() { @@ -876,6 +965,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -895,6 +985,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) it("keeps the secret around after resync", func() { @@ -906,6 +997,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -925,6 +1017,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) it("keeps the secret around after resync", func() { @@ -936,6 +1029,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -960,6 +1054,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -983,6 +1078,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newErrorStrategy("error on delete")) }) }) @@ -1010,10 +1106,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - r.EqualError(runControllerSync(), - "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name") + errString := "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name" + r.EqualError(runControllerSync(), errString) r.Len(kubeAPIClient.Actions(), 1) // no new actions requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newErrorStrategy(errString)) }) }) }) @@ -1036,6 +1133,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerWasNeverStarted() requireNodesListed(kubeAPIClient.Actions()[0]) r.Len(kubeAPIClient.Actions(), 1) + requireCredentialIssuer(newAutoDisabledStrategy()) }) }) @@ -1052,6 +1150,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) }) }) @@ -1068,6 +1167,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerWasNeverStarted() requireNodesListed(kubeAPIClient.Actions()[0]) r.Len(kubeAPIClient.Actions(), 1) + requireCredentialIssuer(newManuallyDisabledStrategy()) }) }) @@ -1078,25 +1178,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("control-plane", kubeAPIClient) }) - it("starts the impersonator", func() { - startInformersAndController() - r.NoError(runControllerSync()) - requireTLSServerIsRunningWithoutCerts() - }) - - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") - startInformersAndController() - r.EqualError(runControllerSync(), "tls error") - }) - - it("starts the load balancer", func() { + it("starts the impersonator and creates a load balancer", func() { startInformersAndController() r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + }) + + it("returns an error when the tls listener fails to start", func() { + startTLSListenerFuncError = errors.New("tls error") + startInformersAndController() + r.EqualError(runControllerSync(), "tls error") + requireCredentialIssuer(newErrorStrategy("tls error")) }) }) @@ -1108,24 +1205,21 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addLoadBalancerServiceToTracker(loadBalancerServiceName, kubeAPIClient) }) - it("starts the impersonator", func() { + it("starts the impersonator without creating a load balancer", func() { startInformersAndController() r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 2) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) }) it("returns an error when the tls listener fails to start", func() { startTLSListenerFuncError = errors.New("tls error") startInformersAndController() r.EqualError(runControllerSync(), "tls error") - }) - - it("does not start the load balancer", func() { - startInformersAndController() - r.NoError(runControllerSync()) - r.Len(kubeAPIClient.Actions(), 2) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireCredentialIssuer(newErrorStrategy("tls error")) }) }) @@ -1150,10 +1244,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) }) - when("we have a hostname specified for the endpoint", func() { + when("the configmap has a hostname specified for the endpoint", func() { const fakeHostname = "fake.example.com" it.Before(func() { configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) @@ -1161,7 +1256,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("worker", kubeAPIClient) }) - it("starts the impersonator, generates a valid cert for the hostname", func() { + it("starts the impersonator, generates a valid cert for the specified hostname", func() { startInformersAndController() r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) @@ -1170,10 +1265,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) }) - when("endpoint is IP address with a port", func() { + when("the configmap has a endpoint which is an IP address with a port", func() { const fakeIPWithPort = "127.0.0.1:3000" it.Before(func() { configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeIPWithPort) @@ -1181,7 +1277,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("worker", kubeAPIClient) }) - it("starts the impersonator, generates a valid cert for the hostname", func() { + it("starts the impersonator, generates a valid cert for the specified IP address", func() { startInformersAndController() r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) @@ -1190,10 +1286,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeIPWithPort. requireTLSServerIsRunning(ca, fakeIPWithPort, map[string]string{fakeIPWithPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) }) - when("endpoint is hostname with a port", func() { + when("the configmap has a endpoint which is a hostname with a port", func() { const fakeHostnameWithPort = "fake.example.com:3000" it.Before(func() { configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostnameWithPort) @@ -1201,7 +1298,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addNodeWithRoleToTracker("worker", kubeAPIClient) }) - it("starts the impersonator, generates a valid cert for the hostname", func() { + it("starts the impersonator, generates a valid cert for the specified hostname", func() { startInformersAndController() r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) @@ -1210,10 +1307,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostnameWithPort. requireTLSServerIsRunning(ca, fakeHostnameWithPort, map[string]string{fakeHostnameWithPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) }) - when("switching from ip address endpoint to hostname endpoint and back to ip address", func() { + when("switching the configmap from ip address endpoint to hostname endpoint and back to ip address", func() { const fakeHostname = "fake.example.com" const fakeIP = "127.0.0.42" var hostnameYAML = fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) @@ -1232,6 +1330,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1249,6 +1348,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], ca) // reuses the old CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch. deleteSecretFromTracker(tlsSecretName, kubeInformerClient) @@ -1265,6 +1365,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[6], ca) // reuses the old CA again // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -1285,6 +1386,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch for the CA Secret. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1300,6 +1402,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -1320,6 +1423,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch for the CA Secret. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") @@ -1337,6 +1441,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // created using the new CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -1355,6 +1460,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch for the CA Secret. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") @@ -1381,6 +1487,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], caCrt) // created using the updated CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(caCrt, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) }) when("deleting the TLS cert due to mismatched CA results in an error", func() { @@ -1397,6 +1504,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Error(runControllerSync(), "error on tls secret delete") r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) // tried to delete cert but failed + requireCredentialIssuer(newErrorStrategy("error on tls secret delete")) }) }) }) @@ -1417,6 +1525,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1431,6 +1540,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsNoLongerRunning() r.Len(kubeAPIClient.Actions(), 4) requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[3]) + requireCredentialIssuer(newManuallyDisabledStrategy()) deleteServiceFromTracker(loadBalancerServiceName, kubeInformerClient) waitForServiceToBeDeleted(kubeInformers.Core().V1().Services(), loadBalancerServiceName) @@ -1442,6 +1552,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 5) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[4]) + requireCredentialIssuer(newPendingStrategy()) }) when("there is an error while shutting down the server", func() { @@ -1459,6 +1570,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.EqualError(runControllerSync(), "fake server close error") requireTLSServerIsNoLongerRunning() + requireCredentialIssuer(newErrorStrategy("fake server close error")) }) }) }) @@ -1480,6 +1592,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1496,6 +1609,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[3]) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) // the Secret was deleted because it contained a cert with the wrong IP requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") @@ -1507,6 +1621,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 5) // no new actions while it is waiting for the load balancer's ingress requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) // Update the ingress of the LB in the informer's client and run Sync again. fakeIP := "127.0.0.123" @@ -1517,6 +1632,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // reuses the existing CA // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[5], kubeInformerClient, "3") @@ -1532,6 +1648,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[6]) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[7]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[8], ca) // recreated because the endpoint was updated, reused the old CA + requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) }) }) @@ -1549,6 +1667,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1559,7 +1678,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time requireTLSServerIsRunningWithoutCerts() // still running - r.Len(kubeAPIClient.Actions(), 3) // no new API calls + requireCredentialIssuer(newPendingStrategy()) + r.Len(kubeAPIClient.Actions(), 3) // no new API calls }) it("creates certs from the ip address listed on the load balancer", func() { @@ -1570,6 +1690,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") @@ -1585,6 +1706,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") @@ -1594,6 +1716,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started again r.Len(kubeAPIClient.Actions(), 4) // no more actions requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running + requireCredentialIssuer(newSuccessStrategy()) }) it("creates certs from the hostname listed on the load balancer", func() { @@ -1605,6 +1728,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") @@ -1620,6 +1744,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now + requireCredentialIssuer(newSuccessStrategy()) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") @@ -1629,6 +1754,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time r.Len(kubeAPIClient.Actions(), 4) // no more actions requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running + requireCredentialIssuer(newSuccessStrategy()) }) }) @@ -1636,6 +1762,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns an error", func() { startInformersAndController() r.EqualError(runControllerSync(), "no nodes found") + requireCredentialIssuer(newErrorStrategy("no nodes found")) requireTLSServerWasNeverStarted() }) }) @@ -1649,6 +1776,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns an error", func() { startInformersAndController() r.EqualError(runControllerSync(), "some factory error") + requireCredentialIssuer(newErrorStrategy("some factory error")) requireTLSServerWasNeverStarted() }) }) @@ -1660,7 +1788,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns an error", func() { startInformersAndController() - r.EqualError(runControllerSync(), "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config") + errString := "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) requireTLSServerWasNeverStarted() }) }) @@ -1668,14 +1798,16 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there is an error creating the load balancer", func() { it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) - startInformersAndController() kubeAPIClient.PrependReactor("create", "services", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on create") }) }) - it("exits with an error", func() { + it("returns an error", func() { + startInformersAndController() r.EqualError(runControllerSync(), "error on create") + requireCredentialIssuer(newErrorStrategy("error on create")) + requireTLSServerIsRunningWithoutCerts() }) }) @@ -1695,6 +1827,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator without certs and returns an error", func() { startInformersAndController() r.EqualError(runControllerSync(), "error on tls secret create") + requireCredentialIssuer(newErrorStrategy("error on tls secret create")) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1719,6 +1852,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator without certs and returns an error", func() { startInformersAndController() r.EqualError(runControllerSync(), "error on ca secret create") + requireCredentialIssuer(newErrorStrategy("error on ca secret create")) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1735,7 +1869,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("starts the impersonator without certs and returns an error", func() { startInformersAndController() - r.EqualError(runControllerSync(), "could not load CA: tls: failed to find any PEM data in certificate input") + errString := "could not load CA: tls: failed to find any PEM data in certificate input" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1756,6 +1892,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the impersonator, deletes the loadbalancer, returns an error", func() { r.EqualError(runControllerSync(), "error on delete") + requireCredentialIssuer(newErrorStrategy("error on delete")) requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1791,6 +1928,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) // deleted the bad cert requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1802,7 +1940,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { startInformersAndController() - r.EqualError(runControllerSync(), "PEM data represented an invalid cert, but got error while deleting it: error on delete") + errString := "PEM data represented an invalid cert, but got error while deleting it: error on delete" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1835,6 +1975,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1846,7 +1987,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { startInformersAndController() - r.EqualError(runControllerSync(), "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete") + errString := "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1880,6 +2023,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireCredentialIssuer(newSuccessStrategy()) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1891,7 +2035,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("tries to delete the invalid cert, starts the impersonator without certs, and returns an error", func() { startInformersAndController() - r.EqualError(runControllerSync(), "cert had an invalid private key, but got error while deleting it: error on delete") + errString := "cert had an invalid private key, but got error while deleting it: error on delete" + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1900,5 +2046,32 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) }) + + when("there is an error while creating or updating the CredentialIssuer status", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + pinnipedAPIClient.PrependReactor("create", "credentialissuers", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on create") + }) + }) + + it("returns the error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "could not create or update credentialissuer: create failed: error on create") + }) + + when("there is also a more fundamental error while starting the impersonator", func() { + it.Before(func() { + kubeAPIClient.PrependReactor("create", "services", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("error on service creation") + }) + }) + + it("returns the more fundamental error instead of the CredentialIssuer error", func() { + startInformersAndController() + r.EqualError(runControllerSync(), "error on service creation") + }) + }) + }) }, spec.Parallel(), spec.Report(report.Terminal{})) } diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 3ba97940..d727a1bc 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -277,12 +277,14 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { singletonWorker, ). - // The impersonation proxy configuration controllers dynamically configure the impersonation proxy feature. + // The impersonator configuration controller dynamically configures the impersonation proxy feature. WithController( impersonatorconfig.NewImpersonatorConfigController( c.ServerInstallationInfo.Namespace, c.NamesConfig.ImpersonationConfigMap, + c.NamesConfig.CredentialIssuer, client.Kubernetes, + client.PinnipedConcierge, informers.installationNamespaceK8s.Core().V1().ConfigMaps(), informers.installationNamespaceK8s.Core().V1().Services(), informers.installationNamespaceK8s.Core().V1().Secrets(), @@ -292,6 +294,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { c.NamesConfig.ImpersonationTLSCertificateSecret, c.NamesConfig.ImpersonationCACertificateSecret, c.Labels, + clock.RealClock{}, tls.Listen, func() (http.Handler, error) { impersonationProxyHandler, err := impersonator.New( From 8bf03257f448395b026a7571b319f2f65eb4034b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 2 Mar 2021 15:27:54 -0800 Subject: [PATCH 062/203] Add new impersonation-related constants to api types and run codegen --- .../config/v1alpha1/types_credentialissuer.go.tmpl | 7 ++++++- cmd/pinniped/cmd/kubeconfig.go | 9 ++++----- .../concierge/config/v1alpha1/types_credentialissuer.go | 7 ++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 7 ++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 7 ++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 7 ++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 7 ++++++- .../controller/impersonatorconfig/impersonator_config.go | 8 ++------ .../impersonatorconfig/impersonator_config_test.go | 4 ++-- 9 files changed, 44 insertions(+), 19 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index 340a402b..ab804f99 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -19,6 +19,7 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") @@ -26,6 +27,10 @@ const ( SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 34e5605b..c1c819bf 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -250,8 +250,7 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe var conciergeCABundleData []byte // Autodiscover the --concierge-mode. - if flags.concierge.mode == modeUnknown { - + if flags.concierge.mode == modeUnknown { //nolint:nestif for _, strategy := range credentialIssuer.Status.Strategies { fe := strategy.Frontend if strategy.Status != configv1alpha1.SuccessStrategyStatus || fe == nil { @@ -475,9 +474,9 @@ func copyCurrentClusterFromExistingKubeConfig(currentKubeConfig clientcmdapi.Con if currentContextNameOverride != "" { contextName = currentContextNameOverride } - context := currentKubeConfig.Contexts[contextName] - if context == nil { + ctx := currentKubeConfig.Contexts[contextName] + if ctx == nil { return nil, fmt.Errorf("no such context %q", contextName) } - return currentKubeConfig.Clusters[context.Cluster], nil + return currentKubeConfig.Clusters[ctx.Cluster], nil } diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index 340a402b..ab804f99 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -19,6 +19,7 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") @@ -26,6 +27,10 @@ const ( SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index 340a402b..ab804f99 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -19,6 +19,7 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") @@ -26,6 +27,10 @@ const ( SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index 340a402b..ab804f99 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -19,6 +19,7 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") @@ -26,6 +27,10 @@ const ( SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index 340a402b..ab804f99 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -19,6 +19,7 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") @@ -26,6 +27,10 @@ const ( SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") diff --git a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go index 340a402b..ab804f99 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package v1alpha1 @@ -19,6 +19,7 @@ type StrategyReason string const ( KubeClusterSigningCertificateStrategyType = StrategyType("KubeClusterSigningCertificate") + ImpersonationProxyStrategyType = StrategyType("ImpersonationProxy") TokenCredentialRequestAPIFrontendType = FrontendType("TokenCredentialRequestAPI") ImpersonationProxyFrontendType = FrontendType("ImpersonationProxy") @@ -26,6 +27,10 @@ const ( SuccessStrategyStatus = StrategyStatus("Success") ErrorStrategyStatus = StrategyStatus("Error") + ListeningStrategyReason = StrategyReason("Listening") + PendingStrategyReason = StrategyReason("Pending") + DisabledStrategyReason = StrategyReason("Disabled") + ErrorDuringSetupStrategyReason = StrategyReason("ErrorDuringSetup") CouldNotFetchKeyStrategyReason = StrategyReason("CouldNotFetchKey") CouldNotGetClusterInfoStrategyReason = StrategyReason("CouldNotGetClusterInfo") FetchedKeyStrategyReason = StrategyReason("FetchedKey") diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 2200f968..28542362 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -44,10 +44,6 @@ const ( caCrtKey = "ca.crt" caKeyKey = "ca.key" appLabelKey = "app" - - // TODO move these to the api package after resolving an upcoming merge. - PendingStrategyReason = v1alpha1.StrategyReason("Pending") - ErrorDuringSetupStrategyReason = v1alpha1.StrategyReason("ErrorDuringSetup") ) type impersonatorConfigController struct { @@ -153,7 +149,7 @@ func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error strategy = &v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.ErrorStrategyStatus, - Reason: ErrorDuringSetupStrategyReason, + Reason: v1alpha1.ErrorDuringSetupStrategyReason, Message: err.Error(), LastUpdateTime: metav1.NewTime(c.clock.Now()), } @@ -740,7 +736,7 @@ func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, return &v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.ErrorStrategyStatus, - Reason: PendingStrategyReason, + Reason: v1alpha1.PendingStrategyReason, Message: "waiting for load balancer Service to be assigned IP or hostname", LastUpdateTime: metav1.NewTime(c.clock.Now()), } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index a1c27a90..12eefda9 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -694,7 +694,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.ErrorStrategyStatus, - Reason: PendingStrategyReason, + Reason: v1alpha1.PendingStrategyReason, Message: "waiting for load balancer Service to be assigned IP or hostname", LastUpdateTime: metav1.NewTime(frozenNow), } @@ -704,7 +704,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.ErrorStrategyStatus, - Reason: ErrorDuringSetupStrategyReason, + Reason: v1alpha1.ErrorDuringSetupStrategyReason, Message: msg, LastUpdateTime: metav1.NewTime(frozenNow), } From 27daf0a2fe13d446a5079c0f0893a9a76de7ed12 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 2 Mar 2021 15:49:01 -0800 Subject: [PATCH 063/203] Increase timeout for creating load balancer in impersonation proxy test --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index f10f87af..a1f7cba8 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -99,7 +99,7 @@ func TestImpersonationProxy(t *testing.T) { // Check that load balancer has been created. library.RequireEventuallyWithoutError(t, func() (bool, error) { return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) - }, 10*time.Second, 500*time.Millisecond) + }, 30*time.Second, 500*time.Millisecond) // TODO this information should come from the CredentialIssuer status once that is implemented // Wait for the load balancer to get an ingress and make a note of its address. From 454f35ccd672bd72191e68a1d3795ac3581d85f0 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 2 Mar 2021 16:00:49 -0800 Subject: [PATCH 064/203] Edit a comment on a type and run codegen --- apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl | 2 +- .../config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.17/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.18/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.19/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.20/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index ab804f99..78ed1384 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -101,7 +101,7 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index 13d71b72..5ae6b3fb 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -93,8 +93,8 @@ spec: Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: diff --git a/generated/1.17/README.adoc b/generated/1.17/README.adoc index 1b43c089..b062b7f4 100644 --- a/generated/1.17/README.adoc +++ b/generated/1.17/README.adoc @@ -347,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index ab804f99..78ed1384 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -101,7 +101,7 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 13d71b72..5ae6b3fb 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -93,8 +93,8 @@ spec: Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: diff --git a/generated/1.18/README.adoc b/generated/1.18/README.adoc index 54aa378a..18ec63f7 100644 --- a/generated/1.18/README.adoc +++ b/generated/1.18/README.adoc @@ -347,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index ab804f99..78ed1384 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -101,7 +101,7 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 13d71b72..5ae6b3fb 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -93,8 +93,8 @@ spec: Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: diff --git a/generated/1.19/README.adoc b/generated/1.19/README.adoc index 69fc53f1..612620fe 100644 --- a/generated/1.19/README.adoc +++ b/generated/1.19/README.adoc @@ -347,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index ab804f99..78ed1384 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -101,7 +101,7 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 13d71b72..5ae6b3fb 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -93,8 +93,8 @@ spec: Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: diff --git a/generated/1.20/README.adoc b/generated/1.20/README.adoc index d666180e..97b02d4e 100644 --- a/generated/1.20/README.adoc +++ b/generated/1.20/README.adoc @@ -347,7 +347,7 @@ Status of a credential issuer. |=== | Field | Description | *`server`* __string__ | Server is the Kubernetes API server URL. -| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the Kubernetes API server CA bundle. +| *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. |=== diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index ab804f99..78ed1384 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -101,7 +101,7 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 13d71b72..5ae6b3fb 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -93,8 +93,8 @@ spec: Concierge. This field is only set when Type is "TokenCredentialRequestAPI". properties: certificateAuthorityData: - description: CertificateAuthorityData is the Kubernetes - API server CA bundle. + description: CertificateAuthorityData is the base64-encoded + Kubernetes API server CA bundle. minLength: 1 type: string server: diff --git a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go index ab804f99..78ed1384 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -101,7 +101,7 @@ type TokenCredentialRequestAPIInfo struct { // +kubebuilder:validation:Pattern=`^https://|^http://` Server string `json:"server"` - // CertificateAuthorityData is the Kubernetes API server CA bundle. + // CertificateAuthorityData is the base64-encoded Kubernetes API server CA bundle. // +kubebuilder:validation:MinLength=1 CertificateAuthorityData string `json:"certificateAuthorityData"` } From d3599c541baf9c92d91a6bf581a4432b63ffa378 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 2 Mar 2021 16:51:35 -0800 Subject: [PATCH 065/203] Fill in the `frontend` field of CredentialIssuer status for impersonator --- .../impersonatorconfig/impersonator_config.go | 28 +++++- .../impersonator_config_test.go | 91 ++++++++++--------- 2 files changed, 75 insertions(+), 44 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 28542362..260a3155 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" "encoding/pem" "errors" "fmt" @@ -209,8 +210,8 @@ func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.Cr } waitingForLoadBalancer := false + var impersonationCA *certauthority.CA if c.shouldHaveTLSSecret(config) { - var impersonationCA *certauthority.CA if impersonationCA, err = c.ensureCASecretIsCreated(ctx); err != nil { return nil, err } @@ -221,7 +222,7 @@ func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.Cr return nil, err } - return c.doSyncResult(waitingForLoadBalancer, config), nil + return c.doSyncResult(waitingForLoadBalancer, config, impersonationCA), nil } func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*impersonator.Config, error) { @@ -730,7 +731,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return nil } -func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, config *impersonator.Config) *v1alpha1.CredentialIssuerStrategy { +func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, config *impersonator.Config, ca *certauthority.CA) *v1alpha1.CredentialIssuerStrategy { switch { case waitingForLoadBalancer: return &v1alpha1.CredentialIssuerStrategy{ @@ -757,12 +758,33 @@ func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, LastUpdateTime: metav1.NewTime(c.clock.Now()), } default: + var endpointName string + if config.Endpoint != "" { + endpointName = config.Endpoint + } else { + desiredIP, desiredHostname, _, _ := c.findTLSCertificateNameFromLoadBalancer() + switch { + case desiredIP != nil: + endpointName = desiredIP.String() + case desiredHostname != "": + endpointName = desiredHostname + default: + endpointName = "" // this shouldn't actually happen in practice + } + } return &v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.SuccessStrategyStatus, Reason: v1alpha1.ListeningStrategyReason, Message: "impersonation proxy is ready to accept client connections", LastUpdateTime: metav1.NewTime(c.clock.Now()), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &v1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://" + endpointName, + CertificateAuthorityData: base64.StdEncoding.EncodeToString(ca.Bundle()), + }, + }, } } } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 12eefda9..59034f06 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -8,6 +8,7 @@ import ( "crypto/tls" "crypto/x509" "crypto/x509/pkix" + "encoding/base64" "encoding/pem" "errors" "fmt" @@ -664,13 +665,20 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ) } - var newSuccessStrategy = func() v1alpha1.CredentialIssuerStrategy { + var newSuccessStrategy = func(endpoint string, ca []byte) v1alpha1.CredentialIssuerStrategy { return v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.SuccessStrategyStatus, Reason: v1alpha1.ListeningStrategyReason, Message: "impersonation proxy is ready to accept client connections", LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &v1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://" + endpoint, + CertificateAuthorityData: base64.StdEncoding.EncodeToString(ca), + }, + }, } } @@ -681,6 +689,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { Reason: v1alpha1.DisabledStrategyReason, Message: "automatically determined that impersonation proxy should be disabled", LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: nil, } } @@ -697,6 +706,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { Reason: v1alpha1.PendingStrategyReason, Message: "waiting for load balancer Service to be assigned IP or hostname", LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: nil, } } @@ -707,6 +717,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { Reason: v1alpha1.ErrorDuringSetupStrategyReason, Message: msg, LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: nil, } } @@ -939,10 +950,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) when("there are not visible control plane nodes and a load balancer already exists with multiple ips", func() { + const fakeIP = "127.0.0.123" it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) - addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeInformerClient) - addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "127.0.0.123"}, {IP: "127.0.0.456"}}, kubeAPIClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}, {IP: "127.0.0.456"}}, kubeInformerClient) + addLoadBalancerServiceWithIngressToTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}, {IP: "127.0.0.456"}}, kubeAPIClient) startInformersAndController() r.NoError(runControllerSync()) }) @@ -952,20 +964,19 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) - requireTLSServerIsRunning(ca, "127.0.0.123", map[string]string{"127.0.0.123:443": testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) - }) + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) - it("keeps the secret around after resync", func() { // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + // keeps the secret around after resync r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) }) }) @@ -985,19 +996,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) - }) + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) - it("keeps the secret around after resync", func() { // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + // keeps the secret around after resync r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) }) }) @@ -1017,19 +1027,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) - }) + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) - it("keeps the secret around after resync", func() { // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + // keeps the secret around after resync r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) }) }) @@ -1054,7 +1063,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) }) }) @@ -1150,7 +1159,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) }) }) }) @@ -1244,7 +1253,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) }) }) @@ -1265,7 +1274,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) }) }) @@ -1286,7 +1295,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeIPWithPort. requireTLSServerIsRunning(ca, fakeIPWithPort, map[string]string{fakeIPWithPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeIPWithPort, ca)) }) }) @@ -1307,7 +1316,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostnameWithPort. requireTLSServerIsRunning(ca, fakeHostnameWithPort, map[string]string{fakeHostnameWithPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostnameWithPort, ca)) }) }) @@ -1330,7 +1339,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1348,7 +1357,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], ca) // reuses the old CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch. deleteSecretFromTracker(tlsSecretName, kubeInformerClient) @@ -1365,7 +1374,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[6], ca) // reuses the old CA again // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) }) }) @@ -1386,7 +1395,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch for the CA Secret. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1402,7 +1411,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) }) }) @@ -1423,7 +1432,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch for the CA Secret. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") @@ -1441,7 +1450,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // created using the new CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) }) }) @@ -1460,7 +1469,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch for the CA Secret. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") @@ -1487,7 +1496,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[4], caCrt) // created using the updated CA // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(caCrt, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, caCrt)) }) when("deleting the TLS cert due to mismatched CA results in an error", func() { @@ -1592,7 +1601,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") @@ -1632,7 +1641,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // reuses the existing CA // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[5], kubeInformerClient, "3") @@ -1649,7 +1658,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[7]) requireTLSSecretWasCreated(kubeAPIClient.Actions()[8], ca) // recreated because the endpoint was updated, reused the old CA requireTLSServerIsRunning(ca, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) }) }) }) @@ -1706,7 +1715,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") @@ -1716,7 +1725,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started again r.Len(kubeAPIClient.Actions(), 4) // no more actions requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) }) it("creates certs from the hostname listed on the load balancer", func() { @@ -1744,7 +1753,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(hostname, ca)) // Simulate the informer cache's background update from its watch. addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") @@ -1754,7 +1763,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time r.Len(kubeAPIClient.Actions(), 4) // no more actions requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(hostname, ca)) }) }) @@ -1928,7 +1937,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) // deleted the bad cert requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1975,7 +1984,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) }) when("there is an error while the invalid cert is being deleted", func() { @@ -2023,7 +2032,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) // deleted the bad cert requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) - requireCredentialIssuer(newSuccessStrategy()) + requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) }) when("there is an error while the invalid cert is being deleted", func() { From 730092f39c5c17a3ad5dd8596913032c59aea2bf Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 09:22:35 -0800 Subject: [PATCH 066/203] impersonator_config.go: refactor to clean up cert name handling --- internal/concierge/impersonator/config.go | 4 + .../concierge/impersonator/config_test.go | 7 + .../impersonatorconfig/impersonator_config.go | 138 +++++++++--------- .../impersonator_config_test.go | 3 +- 4 files changed, 79 insertions(+), 73 deletions(-) diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go index a7bb8c2c..60ab59bd 100644 --- a/internal/concierge/impersonator/config.go +++ b/internal/concierge/impersonator/config.go @@ -41,6 +41,10 @@ type Config struct { Endpoint string `json:"endpoint,omitempty"` } +func (c *Config) HasEndpoint() bool { + return c.Endpoint != "" +} + func NewConfig() *Config { return &Config{Mode: ModeAuto} } diff --git a/internal/concierge/impersonator/config_test.go b/internal/concierge/impersonator/config_test.go index 2e96a9af..b734dba9 100644 --- a/internal/concierge/impersonator/config_test.go +++ b/internal/concierge/impersonator/config_test.go @@ -18,6 +18,13 @@ func TestNewConfig(t *testing.T) { require.Equal(t, &Config{Mode: ModeAuto}, NewConfig()) } +func TestHasEndpoint(t *testing.T) { + configWithoutEndpoint := Config{} + configWithEndpoint := Config{Endpoint: "something"} + require.False(t, configWithoutEndpoint.HasEndpoint()) + require.True(t, configWithEndpoint.HasEndpoint()) +} + func TestConfigFromConfigMap(t *testing.T) { tests := []struct { name string diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 260a3155..ccac388f 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -170,6 +170,22 @@ func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error return err } +type certNameInfo struct { + // ready will be true when the certificate name information is known. + // ready will be false when it is pending because we are waiting for a load balancer to get assigned an ip/hostname. + // When false, the other fields in this struct should not be considered meaningful and may be zero values. + ready bool + + // The IP address or hostname which was selected to be used as the name in the cert. + // Either selectedIP or selectedHostname will be set, but not both. + selectedIP net.IP + selectedHostname string + + // The name of the endpoint to which a client should connect to talk to the impersonator. + // This may be a hostname or an IP, and may include a port number. + clientEndpoint string +} + func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.CredentialIssuerStrategy, error) { config, err := c.loadImpersonationProxyConfiguration() if err != nil { @@ -209,20 +225,24 @@ func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.Cr } } - waitingForLoadBalancer := false + nameInfo, err := c.findDesiredTLSCertificateName(config) + if err != nil { + return nil, err + } + var impersonationCA *certauthority.CA if c.shouldHaveTLSSecret(config) { if impersonationCA, err = c.ensureCASecretIsCreated(ctx); err != nil { return nil, err } - if waitingForLoadBalancer, err = c.ensureTLSSecret(ctx, config, impersonationCA); err != nil { + if err = c.ensureTLSSecret(ctx, nameInfo, impersonationCA); err != nil { return nil, err } } else if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { return nil, err } - return c.doSyncResult(waitingForLoadBalancer, config, impersonationCA), nil + return c.doSyncResult(nameInfo, config, impersonationCA), nil } func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*impersonator.Config, error) { @@ -270,7 +290,7 @@ func (c *impersonatorConfigController) disabledExplicitly(config *impersonator.C } func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonator.Config) bool { - return c.shouldHaveImpersonator(config) && config.Endpoint == "" + return c.shouldHaveImpersonator(config) && !config.HasEndpoint() } func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { @@ -400,17 +420,17 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.C return c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) } -func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, config *impersonator.Config, ca *certauthority.CA) (bool, error) { +func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, nameInfo *certNameInfo, ca *certauthority.CA) error { secretFromInformer, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) notFound := k8serrors.IsNotFound(err) if !notFound && err != nil { - return false, err + return err } if !notFound { - secretWasDeleted, err := c.deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx, config, ca, secretFromInformer) + secretWasDeleted, err := c.deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx, nameInfo, ca, secretFromInformer) if err != nil { - return false, err + return err } // If it was deleted by the above call, then set it to nil. This allows us to avoid waiting // for the informer cache to update before deciding to proceed to create the new Secret below. @@ -419,10 +439,10 @@ func (c *impersonatorConfigController) ensureTLSSecret(ctx context.Context, conf } } - return c.ensureTLSSecretIsCreatedAndLoaded(ctx, config, secretFromInformer, ca) + return c.ensureTLSSecretIsCreatedAndLoaded(ctx, nameInfo, secretFromInformer, ca) } -func (c *impersonatorConfigController) deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx context.Context, config *impersonator.Config, ca *certauthority.CA, secret *v1.Secret) (bool, error) { +func (c *impersonatorConfigController) deleteTLSSecretWhenCertificateDoesNotMatchDesiredState(ctx context.Context, nameInfo *certNameInfo, ca *certauthority.CA, secret *v1.Secret) (bool, error) { certPEM := secret.Data[v1.TLSCertKey] block, _ := pem.Decode(certPEM) if block == nil { @@ -471,11 +491,7 @@ func (c *impersonatorConfigController) deleteTLSSecretWhenCertificateDoesNotMatc return true, nil } - desiredIP, desiredHostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) - if err != nil { - return false, err - } - if !nameIsReady { + if !nameInfo.ready { // We currently have a secret but we are waiting for a load balancer to be assigned an ingress, so // our current secret must be old/unwanted. if err = c.ensureTLSSecretIsRemoved(ctx); err != nil { @@ -487,14 +503,14 @@ func (c *impersonatorConfigController) deleteTLSSecretWhenCertificateDoesNotMatc actualIPs := actualCertFromSecret.IPAddresses actualHostnames := actualCertFromSecret.DNSNames plog.Info("Checking TLS certificate names", - "desiredIP", desiredIP, - "desiredHostname", desiredHostname, + "desiredIP", nameInfo.selectedIP, + "desiredHostname", nameInfo.selectedHostname, "actualIPs", actualIPs, "actualHostnames", actualHostnames, "secret", c.tlsSecretName, "namespace", c.namespace) - if certHostnameAndIPMatchDesiredState(desiredIP, actualIPs, desiredHostname, actualHostnames) { + if certHostnameAndIPMatchDesiredState(nameInfo.selectedIP, actualIPs, nameInfo.selectedHostname, actualHostnames) { // The cert already matches the desired state, so there is no need to delete/recreate it. return false, nil } @@ -515,36 +531,30 @@ func certHostnameAndIPMatchDesiredState(desiredIP net.IP, actualIPs []net.IP, de return false } -func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret, ca *certauthority.CA) (bool, error) { +func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, nameInfo *certNameInfo, secret *v1.Secret, ca *certauthority.CA) error { if secret != nil { err := c.loadTLSCertFromSecret(secret) if err != nil { - return false, err + return err } - return false, nil + return nil } - ip, hostname, nameIsReady, err := c.findDesiredTLSCertificateName(config) - if err != nil { - return false, err - } - if !nameIsReady { - // Sync will get called again when the load balancer is updated with its ingress info, so this is not an error. - // Return "true" meaning that we are waiting for the load balancer. - return true, nil + if !nameInfo.ready { + return nil } - newTLSSecret, err := c.createNewTLSSecret(ctx, ca, ip, hostname) + newTLSSecret, err := c.createNewTLSSecret(ctx, ca, nameInfo.selectedIP, nameInfo.selectedHostname) if err != nil { - return false, err + return err } err = c.loadTLSCertFromSecret(newTLSSecret) if err != nil { - return false, err + return err } - return false, nil + return nil } func (c *impersonatorConfigController) ensureCASecretIsCreated(ctx context.Context) (*certauthority.CA, error) { @@ -602,55 +612,55 @@ func (c *impersonatorConfigController) createCASecret(ctx context.Context) (*cer return impersonationCA, nil } -func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *impersonator.Config) (net.IP, string, bool, error) { - if config.Endpoint != "" { +func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *impersonator.Config) (*certNameInfo, error) { + if config.HasEndpoint() { return c.findTLSCertificateNameFromEndpointConfig(config) } return c.findTLSCertificateNameFromLoadBalancer() } -func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) (net.IP, string, bool, error) { - endpointWithoutPort := strings.Split(config.Endpoint, ":")[0] +func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) (*certNameInfo, error) { + endpointMaybeWithPort := config.Endpoint + endpointWithoutPort := strings.Split(endpointMaybeWithPort, ":")[0] parsedAsIP := net.ParseIP(endpointWithoutPort) if parsedAsIP != nil { - return parsedAsIP, "", true, nil + return &certNameInfo{ready: true, selectedIP: parsedAsIP, clientEndpoint: endpointMaybeWithPort}, nil } - return nil, endpointWithoutPort, true, nil + return &certNameInfo{ready: true, selectedHostname: endpointWithoutPort, clientEndpoint: endpointMaybeWithPort}, nil } -func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() (net.IP, string, bool, error) { +func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() (*certNameInfo, error) { lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) notFound := k8serrors.IsNotFound(err) if notFound { - // Although we created the load balancer, maybe it hasn't been cached in the informer yet. // We aren't ready and will try again later in this case. - return nil, "", false, nil + return &certNameInfo{ready: false}, nil } if err != nil { - return nil, "", false, err + return nil, err } ingresses := lb.Status.LoadBalancer.Ingress if len(ingresses) == 0 || (ingresses[0].Hostname == "" && ingresses[0].IP == "") { plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", "service", c.generatedLoadBalancerServiceName, "namespace", c.namespace) - return nil, "", false, nil + return &certNameInfo{ready: false}, nil } for _, ingress := range ingresses { hostname := ingress.Hostname if hostname != "" { - return nil, hostname, true, nil + return &certNameInfo{ready: true, selectedHostname: hostname, clientEndpoint: hostname}, nil } } for _, ingress := range ingresses { ip := ingress.IP parsedIP := net.ParseIP(ip) if parsedIP != nil { - return parsedIP, "", true, nil + return &certNameInfo{ready: true, selectedIP: parsedIP, clientEndpoint: ip}, nil } } - return nil, "", false, fmt.Errorf("could not find valid IP addresses or hostnames from load balancer %s/%s", c.namespace, lb.Name) + return nil, fmt.Errorf("could not find valid IP addresses or hostnames from load balancer %s/%s", c.namespace, lb.Name) } func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ip net.IP, hostname string) (*v1.Secret, error) { @@ -731,16 +741,8 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return nil } -func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, config *impersonator.Config, ca *certauthority.CA) *v1alpha1.CredentialIssuerStrategy { +func (c *impersonatorConfigController) doSyncResult(nameInfo *certNameInfo, config *impersonator.Config, ca *certauthority.CA) *v1alpha1.CredentialIssuerStrategy { switch { - case waitingForLoadBalancer: - return &v1alpha1.CredentialIssuerStrategy{ - Type: v1alpha1.ImpersonationProxyStrategyType, - Status: v1alpha1.ErrorStrategyStatus, - Reason: v1alpha1.PendingStrategyReason, - Message: "waiting for load balancer Service to be assigned IP or hostname", - LastUpdateTime: metav1.NewTime(c.clock.Now()), - } case c.disabledExplicitly(config): return &v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, @@ -757,21 +759,15 @@ func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, Message: "automatically determined that impersonation proxy should be disabled", LastUpdateTime: metav1.NewTime(c.clock.Now()), } - default: - var endpointName string - if config.Endpoint != "" { - endpointName = config.Endpoint - } else { - desiredIP, desiredHostname, _, _ := c.findTLSCertificateNameFromLoadBalancer() - switch { - case desiredIP != nil: - endpointName = desiredIP.String() - case desiredHostname != "": - endpointName = desiredHostname - default: - endpointName = "" // this shouldn't actually happen in practice - } + case !nameInfo.ready: + return &v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.ImpersonationProxyStrategyType, + Status: v1alpha1.ErrorStrategyStatus, + Reason: v1alpha1.PendingStrategyReason, + Message: "waiting for load balancer Service to be assigned IP or hostname", + LastUpdateTime: metav1.NewTime(c.clock.Now()), } + default: return &v1alpha1.CredentialIssuerStrategy{ Type: v1alpha1.ImpersonationProxyStrategyType, Status: v1alpha1.SuccessStrategyStatus, @@ -781,7 +777,7 @@ func (c *impersonatorConfigController) doSyncResult(waitingForLoadBalancer bool, Frontend: &v1alpha1.CredentialIssuerFrontend{ Type: v1alpha1.ImpersonationProxyFrontendType, ImpersonationProxyInfo: &v1alpha1.ImpersonationProxyInfo{ - Endpoint: "https://" + endpointName, + Endpoint: "https://" + nameInfo.clientEndpoint, CertificateAuthorityData: base64.StdEncoding.EncodeToString(ca.Bundle()), }, }, diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 59034f06..3fc3d695 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -942,9 +942,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the load balancer automatically", func() { requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 2) + r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireCredentialIssuer(newErrorStrategy("could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name")) }) }) From 333a3ab4c2816afc1db59751cfd7f8de1f0f79be Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 09:37:08 -0800 Subject: [PATCH 067/203] impersonator_config_test.go: Add another unit test --- .../impersonator_config_test.go | 54 ++++++++++++++++--- 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 3fc3d695..0c0f2147 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -721,14 +721,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var requireCredentialIssuer = func(expectedStrategy v1alpha1.CredentialIssuerStrategy) { - // Rather than looking at the specific API actions on pinnipedAPIClient, we just look - // at the final result here. - // This is because the implementation is using a helper from another package to create - // and update the CredentialIssuer, and the specific API actions performed by that - // implementation are pretty complex and are already tested by its own unit tests. - // As long as we get the final result that we wanted then we are happy for the purposes - // of this test. + var getCredentialIssuer = func() *v1alpha1.CredentialIssuer { credentialIssuerObj, err := pinnipedAPIClient.Tracker().Get( schema.GroupVersionResource{ Group: v1alpha1.SchemeGroupVersion.Group, @@ -739,6 +732,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(err) credentialIssuer, ok := credentialIssuerObj.(*v1alpha1.CredentialIssuer) r.True(ok, "should have been able to cast this obj to CredentialIssuer: %v", credentialIssuerObj) + return credentialIssuer + } + + var requireCredentialIssuer = func(expectedStrategy v1alpha1.CredentialIssuerStrategy) { + // Rather than looking at the specific API actions on pinnipedAPIClient, we just look + // at the final result here. + // This is because the implementation is using a helper from another package to create + // and update the CredentialIssuer, and the specific API actions performed by that + // implementation are pretty complex and are already tested by its own unit tests. + // As long as we get the final result that we wanted then we are happy for the purposes + // of this test. + credentialIssuer := getCredentialIssuer() r.Equal(labels, credentialIssuer.Labels) r.Equal([]v1alpha1.CredentialIssuerStrategy{expectedStrategy}, credentialIssuer.Status.Strategies) } @@ -2081,5 +2086,38 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) }) + + when("there is already a CredentialIssuer", func() { + preExistingStrategy := v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.KubeClusterSigningCertificateStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.FetchedKeyStrategyReason, + Message: "happy other unrelated strategy", + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.TokenCredentialRequestAPIFrontendType, + }, + } + + it.Before(func() { + r.NoError(pinnipedAPIClient.Tracker().Add(&v1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: credentialIssuerResourceName}, + Status: v1alpha1.CredentialIssuerStatus{Strategies: []v1alpha1.CredentialIssuerStrategy{preExistingStrategy}}, + })) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("merges into the existing strategy array on the CredentialIssuer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + credentialIssuer := getCredentialIssuer() + r.Equal([]v1alpha1.CredentialIssuerStrategy{newPendingStrategy(), preExistingStrategy}, credentialIssuer.Status.Strategies) + }) + }) }, spec.Parallel(), spec.Report(report.Terminal{})) } From 0799a538dc354d9aebe98e84e175fb5cc61e262e Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 3 Mar 2021 11:11:58 -0800 Subject: [PATCH 068/203] change FromString to Parse so TargetPort parses correctly --- internal/controller/impersonatorconfig/impersonator_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index ccac388f..efbb3a27 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -385,7 +385,7 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C Type: v1.ServiceTypeLoadBalancer, Ports: []v1.ServicePort{ { - TargetPort: intstr.FromString(impersonationProxyPort), + TargetPort: intstr.Parse(impersonationProxyPort), Port: defaultHTTPSPort, Protocol: v1.ProtocolTCP, }, From f4fcb9bde63d7fa38b1962ab0a0f5e952b9b4246 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 3 Mar 2021 14:03:27 -0600 Subject: [PATCH 069/203] Sort CredentialIssuer strategies in preferred order. This updates our issuerconfig.UpdateStrategy to sort strategies according to a weighted preference. The TokenCredentialRequest API strategy is preffered, followed by impersonation proxy, followed by any other unknown types. Signed-off-by: Matt Moyer --- .../impersonator_config_test.go | 2 +- .../issuerconfig/update_strategy.go | 20 +++++++++--- .../issuerconfig/update_strategy_test.go | 31 +++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 0c0f2147..73fb91c2 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -2116,7 +2116,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) credentialIssuer := getCredentialIssuer() - r.Equal([]v1alpha1.CredentialIssuerStrategy{newPendingStrategy(), preExistingStrategy}, credentialIssuer.Status.Strategies) + r.Equal([]v1alpha1.CredentialIssuerStrategy{preExistingStrategy, newPendingStrategy()}, credentialIssuer.Status.Strategies) }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) diff --git a/internal/controller/issuerconfig/update_strategy.go b/internal/controller/issuerconfig/update_strategy.go index 669516ff..f357aed9 100644 --- a/internal/controller/issuerconfig/update_strategy.go +++ b/internal/controller/issuerconfig/update_strategy.go @@ -52,9 +52,21 @@ func mergeStrategy(configToUpdate *v1alpha1.CredentialIssuerStatus, strategy v1a } } -// TODO: sort strategies by server preference rather than alphanumerically by type. +// weights are a set of priorities for each strategy type. +//nolint: gochecknoglobals +var weights = map[v1alpha1.StrategyType]int{ + v1alpha1.KubeClusterSigningCertificateStrategyType: 2, // most preferred strategy + v1alpha1.ImpersonationProxyStrategyType: 1, + // unknown strategy types will have weight 0 by default +} + type sortableStrategies []v1alpha1.CredentialIssuerStrategy -func (s sortableStrategies) Len() int { return len(s) } -func (s sortableStrategies) Less(i, j int) bool { return s[i].Type < s[j].Type } -func (s sortableStrategies) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s sortableStrategies) Len() int { return len(s) } +func (s sortableStrategies) Less(i, j int) bool { + if wi, wj := weights[s[i].Type], weights[s[j].Type]; wi != wj { + return wi > wj + } + return s[i].Type < s[j].Type +} +func (s sortableStrategies) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/internal/controller/issuerconfig/update_strategy_test.go b/internal/controller/issuerconfig/update_strategy_test.go index b1b90429..16302499 100644 --- a/internal/controller/issuerconfig/update_strategy_test.go +++ b/internal/controller/issuerconfig/update_strategy_test.go @@ -4,9 +4,13 @@ package issuerconfig import ( + "math/rand" + "sort" "testing" + "testing/quick" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -185,3 +189,30 @@ func TestMergeStrategy(t *testing.T) { }) } } + +func TestStrategySorting(t *testing.T) { + expected := []v1alpha1.CredentialIssuerStrategy{ + {Type: v1alpha1.KubeClusterSigningCertificateStrategyType}, + {Type: v1alpha1.ImpersonationProxyStrategyType}, + {Type: "Type1"}, + {Type: "Type2"}, + {Type: "Type3"}, + } + require.NoError(t, quick.Check(func(seed int64) bool { + // Create a randomly shuffled copy of the expected output. + //nolint:gosec // this is not meant to be a secure random, just a seeded RNG for shuffling deterministically + rng := rand.New(rand.NewSource(seed)) + output := make([]v1alpha1.CredentialIssuerStrategy, len(expected)) + copy(output, expected) + rng.Shuffle( + len(output), + func(i, j int) { output[i], output[j] = output[j], output[i] }, + ) + + // Sort it using the code under test. + sort.Stable(sortableStrategies(output)) + + // Assert that it's sorted back to the expected output order. + return assert.Equal(t, expected, output) + }, nil)) +} From 57453773eacba51513074bcac0768f6a337a798f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 12:06:44 -0800 Subject: [PATCH 070/203] CONTRIBUTING.md: remove mention of Tilt, since it isn't working well --- CONTRIBUTING.md | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c00094b4..f5112435 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,36 +96,29 @@ docker build . - [`kapp`](https://carvel.dev/#getting-started) - [`kind`](https://kind.sigs.k8s.io/docs/user/quick-start) - [`kubectl`](https://kubernetes.io/docs/tasks/tools/install-kubectl/) - - [`tilt`](https://docs.tilt.dev/install.html) - [`ytt`](https://carvel.dev/#getting-started) On macOS, these tools can be installed with [Homebrew](https://brew.sh/) (assuming you have Chrome installed already): ```bash - brew install kind tilt-dev/tap/tilt k14s/tap/ytt k14s/tap/kapp kubectl chromedriver && brew cask install docker + brew install kind k14s/tap/ytt k14s/tap/kapp kubectl chromedriver && brew cask install docker ``` -1. Create a local Kubernetes cluster using `kind`: +1. Create a kind cluster, compile, create container images, and install Pinniped and supporting dependencies using: ```bash - ./hack/kind-up.sh + ./hack/prepare-for-integration-tests.sh ``` -1. Install Pinniped and supporting dependencies using `tilt`: - - ```bash - ./hack/tilt-up.sh - ``` - - Tilt will continue running and live-updating the Pinniped deployment whenever the code changes. - 1. Run the Pinniped integration tests: ```bash source /tmp/integration-test-env && go test -v -count 1 ./test/integration ``` -To uninstall the test environment, run `./hack/tilt-down.sh`. +1. After making production code changes, recompile, redeploy, and run tests again by repeating the same + commands described above. If there are only test code changes, then simply run the tests again. + To destroy the local Kubernetes cluster, run `./hack/kind-down.sh`. ### Observing Tests on the Continuous Integration Environment From 7b7901af36df2dc5fa47c963c4730fe9281ef866 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 12:08:40 -0800 Subject: [PATCH 071/203] Add `-timeout 0` when describing how to run integration tests Because otherwise `go test` will panic/crash your test if it takes longer than 10 minutes, which is an annoying way for an integration test to fail since it skips all of the t.Cleanup's. --- CONTRIBUTING.md | 2 +- hack/prepare-for-integration-tests.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f5112435..5157c86c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -113,7 +113,7 @@ docker build . 1. Run the Pinniped integration tests: ```bash - source /tmp/integration-test-env && go test -v -count 1 ./test/integration + source /tmp/integration-test-env && go test -v -count 1 -timeout 0 ./test/integration ``` 1. After making production code changes, recompile, redeploy, and run tests again by repeating the same diff --git a/hack/prepare-for-integration-tests.sh b/hack/prepare-for-integration-tests.sh index bdc6e1b9..4a2a8650 100755 --- a/hack/prepare-for-integration-tests.sh +++ b/hack/prepare-for-integration-tests.sh @@ -297,7 +297,7 @@ kind_capabilities_file="$pinniped_path/test/cluster_capabilities/kind.yaml" pinniped_cluster_capability_file_content=$(cat "$kind_capabilities_file") cat </tmp/integration-test-env -# The following env vars should be set before running 'go test -v -count 1 ./test/integration' +# The following env vars should be set before running 'go test -v -count 1 -timeout 0 ./test/integration' export PINNIPED_TEST_CONCIERGE_NAMESPACE=${concierge_namespace} export PINNIPED_TEST_CONCIERGE_APP_NAME=${concierge_app_name} export PINNIPED_TEST_CONCIERGE_CUSTOM_LABELS='${concierge_custom_labels}' @@ -346,7 +346,7 @@ goland_vars=$(grep -v '^#' /tmp/integration-test-env | grep -E '^export .+=' | s log_note log_note "🚀 Ready to run integration tests! For example..." log_note " cd $pinniped_path" -log_note ' source /tmp/integration-test-env && go test -v -race -count 1 ./test/integration' +log_note ' source /tmp/integration-test-env && go test -v -race -count 1 -timeout 0 ./test/integration' log_note log_note 'Want to run integration tests in GoLand? Copy/paste this "Environment" value for GoLand run configurations:' log_note " ${goland_vars}PINNIPED_TEST_CLUSTER_CAPABILITY_FILE=${kind_capabilities_file}" From f0fc84c922d0455e2bd2399ab2a4779be2b3f206 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 12:30:21 -0800 Subject: [PATCH 072/203] Add new allowed values to field validations on CredentialIssuer The new values are used by the impersonation proxy's status. --- .../config/v1alpha1/types_credentialissuer.go.tmpl | 6 +++--- .../config.concierge.pinniped.dev_credentialissuers.yaml | 9 ++++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 6 +++--- .../config.concierge.pinniped.dev_credentialissuers.yaml | 9 ++++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 6 +++--- .../config.concierge.pinniped.dev_credentialissuers.yaml | 9 ++++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 6 +++--- .../config.concierge.pinniped.dev_credentialissuers.yaml | 9 ++++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 6 +++--- .../config.concierge.pinniped.dev_credentialissuers.yaml | 9 ++++++++- .../concierge/config/v1alpha1/types_credentialissuer.go | 6 +++--- 11 files changed, 58 insertions(+), 23 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index 78ed1384..e38b207b 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -5,16 +5,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index 5ae6b3fb..7436f6e2 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -111,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -126,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -139,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index 78ed1384..e38b207b 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -5,16 +5,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 5ae6b3fb..7436f6e2 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -111,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -126,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -139,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index 78ed1384..e38b207b 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -5,16 +5,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 5ae6b3fb..7436f6e2 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -111,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -126,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -139,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index 78ed1384..e38b207b 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -5,16 +5,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 5ae6b3fb..7436f6e2 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -111,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -126,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -139,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index 78ed1384..e38b207b 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -5,16 +5,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 5ae6b3fb..7436f6e2 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -111,6 +111,7 @@ spec: can use with a strategy. enum: - TokenCredentialRequestAPI + - ImpersonationProxy type: string required: - type @@ -126,8 +127,13 @@ spec: reason: description: Reason for the current status. enum: - - FetchedKey + - Listening + - Pending + - Disabled + - ErrorDuringSetup - CouldNotFetchKey + - CouldNotGetClusterInfo + - FetchedKey type: string status: description: Status of the attempted integration strategy. @@ -139,6 +145,7 @@ spec: description: Type of integration attempted. enum: - KubeClusterSigningCertificate + - ImpersonationProxy type: string required: - lastUpdateTime diff --git a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go index 78ed1384..e38b207b 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -5,16 +5,16 @@ package v1alpha1 import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// +kubebuilder:validation:Enum=KubeClusterSigningCertificate +// +kubebuilder:validation:Enum=KubeClusterSigningCertificate;ImpersonationProxy type StrategyType string -// +kubebuilder:validation:Enum=TokenCredentialRequestAPI +// +kubebuilder:validation:Enum=TokenCredentialRequestAPI;ImpersonationProxy type FrontendType string // +kubebuilder:validation:Enum=Success;Error type StrategyStatus string -// +kubebuilder:validation:Enum=FetchedKey;CouldNotFetchKey +// +kubebuilder:validation:Enum=Listening;Pending;Disabled;ErrorDuringSetup;CouldNotFetchKey;CouldNotGetClusterInfo;FetchedKey type StrategyReason string const ( From 666c0b0e18566af00bfc1427e8eec175d4e7bf84 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 12:53:23 -0800 Subject: [PATCH 073/203] Use CredentialIssuer for URL/CA discovery in impersonator int test --- .../concierge_impersonation_proxy_test.go | 178 +++++++++++------- 1 file changed, 115 insertions(+), 63 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index a1f7cba8..6365eff9 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -5,6 +5,7 @@ package integration import ( "context" + "encoding/base64" "fmt" "net/http" "net/url" @@ -24,6 +25,8 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/yaml" + "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/internal/testutil/impersonationtoken" "go.pinniped.dev/test/library" @@ -36,11 +39,12 @@ import ( func TestImpersonationProxy(t *testing.T) { env := library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) defer cancel() // Create a client using the admin kubeconfig. adminClient := library.NewKubernetesClientset(t) + adminConciergeClient := library.NewConciergeClientset(t) // Create a WebhookAuthenticator. authenticator := library.CreateTestWebhookAuthenticator(ctx, t) @@ -64,8 +68,7 @@ func TestImpersonationProxy(t *testing.T) { impersonationProxyViaSquidClient := func(caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { t.Helper() - host := fmt.Sprintf("https://%s", proxyServiceEndpoint) - kubeconfig := impersonationProxyRestConfig(host, caData, doubleImpersonateUser) + kubeconfig := impersonationProxyRestConfig("https://"+proxyServiceEndpoint, caData, doubleImpersonateUser) kubeconfig.Proxy = func(req *http.Request) (*url.URL, error) { proxyURL, err := url.Parse(env.Proxy) require.NoError(t, err) @@ -77,10 +80,9 @@ func TestImpersonationProxy(t *testing.T) { return impersonationProxyClient } - impersonationProxyViaLoadBalancerClient := func(host string, caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { + impersonationProxyViaLoadBalancerClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { t.Helper() - host = fmt.Sprintf("https://%s", host) - kubeconfig := impersonationProxyRestConfig(host, caData, doubleImpersonateUser) + kubeconfig := impersonationProxyRestConfig(proxyURL, caData, doubleImpersonateUser) impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") return impersonationProxyClient @@ -96,33 +98,16 @@ func TestImpersonationProxy(t *testing.T) { impersonationProxyLoadBalancerIngress := "" if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test - // Check that load balancer has been created. + // Check that load balancer has been automatically created by the impersonator's "auto" mode. library.RequireEventuallyWithoutError(t, func() (bool, error) { return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) }, 30*time.Second, 500*time.Millisecond) - - // TODO this information should come from the CredentialIssuer status once that is implemented - // Wait for the load balancer to get an ingress and make a note of its address. - var ingress *corev1.LoadBalancerIngress - library.RequireEventuallyWithoutError(t, func() (bool, error) { - ingress, err = getImpersonationProxyLoadBalancerIngress(ctx, env, adminClient) - if err != nil { - return false, err - } - return ingress != nil, nil - }, 10*time.Second, 500*time.Millisecond) - if ingress.Hostname != "" { - impersonationProxyLoadBalancerIngress = ingress.Hostname - } else { - require.NotEmpty(t, ingress.IP, "the ingress should have either a hostname or IP, but it didn't") - impersonationProxyLoadBalancerIngress = ingress.IP - } } else { require.NotEmpty(t, env.Proxy, "test cluster does not support load balancers but also doesn't have a squid proxy... "+ "this is not a supported configuration for test clusters") - // Check that no load balancer has been created. + // Check that no load balancer has been created by the impersonator's "auto" mode. library.RequireNeverWithoutError(t, func() (bool, error) { return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) }, 10*time.Second, 500*time.Millisecond) @@ -131,7 +116,7 @@ func TestImpersonationProxy(t *testing.T) { _, err = impersonationProxyViaSquidClient(nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableViaSquidError) - // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a LoadBalancer). + // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). configMap := configMapForConfig(t, env, impersonator.Config{ Mode: impersonator.ModeEnabled, Endpoint: proxyServiceEndpoint, @@ -159,29 +144,21 @@ func TestImpersonationProxy(t *testing.T) { }) } - // Check that the controller generated a CA. Get the CA data so we can use it as a client. - // TODO We should be getting the CA data from the CredentialIssuer's status instead, once that is implemented. - var caSecret *corev1.Secret - require.Eventually(t, func() bool { - caSecret, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) - return err == nil && caSecret != nil && caSecret.Data["ca.crt"] != nil - }, 10*time.Second, 250*time.Millisecond) - impersonationProxyCACertPEM := caSecret.Data["ca.crt"] - - // Check that the generated TLS cert Secret was created by the controller. - // This could take a while if we are waiting for the load balancer to get an IP or hostname assigned to it, and it - // should be fast when we are not waiting for a load balancer (e.g. on kind). - require.Eventually(t, func() bool { - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - return err == nil - }, 5*time.Minute, 250*time.Millisecond) + // At this point the impersonator should be starting/running. When it is ready, the CredentialIssuer's + // strategies array should be updated to include a successful impersonation strategy which can be used + // to discover the impersonator's URL and CA certificate. Until it has finished starting, it may not be included + // in the strategies array or it may be included in an error state. It can be in an error state for + // awhile when it is waiting for the load balancer to be assigned an ip/hostname. + impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminConciergeClient) // Create an impersonation proxy client with that CA data to use for the rest of this test. // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. var impersonationProxyClient *kubernetes.Clientset if env.HasCapability(library.HasExternalLoadBalancerProvider) { - impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, impersonationProxyCACertPEM, "") + impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyURL, impersonationProxyCACertPEM, "") } else { + // In this case, we specified the endpoint in the configmap, so check that it was reported correctly in the CredentialIssuer. + require.Equal(t, "https://"+proxyServiceEndpoint, impersonationProxyURL) impersonationProxyClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "") } @@ -359,8 +336,8 @@ func TestImpersonationProxy(t *testing.T) { doubleImpersonationClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "other-user-to-impersonate") } - // We already know that this Secret exists because we checked above. Now see that we can get it through - // the impersonation proxy without any impersonation headers on the request. + // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. + // We could use any resource for this, but we happen to know that this one should exist. _, err = impersonationProxyClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.NoError(t, err) @@ -385,12 +362,12 @@ func TestImpersonationProxy(t *testing.T) { } if env.HasCapability(library.HasExternalLoadBalancerProvider) { - // The load balancer should not exist after we disable the impersonation proxy. + // The load balancer should have been deleted when we disabled the impersonation proxy. // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). library.RequireEventuallyWithoutError(t, func() (bool, error) { hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) return !hasService, err - }, time.Minute, 500*time.Millisecond) + }, 2*time.Minute, 500*time.Millisecond) } // Check that the impersonation proxy port has shut down. @@ -407,14 +384,100 @@ func TestImpersonationProxy(t *testing.T) { }, 20*time.Second, 500*time.Millisecond) } - // Check that the generated TLS cert Secret was deleted by the controller. + // Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up + // when we disable the impersonator. require.Eventually(t, func() bool { _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) return k8serrors.IsNotFound(err) }, 10*time.Second, 250*time.Millisecond) + + // Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this + // around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when + // possible because they make their way into kubeconfigs on client machines. + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) + require.NoError(t, err) + + // At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to + // include an unsuccessful impersonation strategy saying that it was manually configured to be disabled. + requireDisabledByConfigurationStrategy(ctx, t, env, adminConciergeClient) +} + +func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient versioned.Interface) (string, []byte) { + t.Helper() + var impersonationProxyURL string + var impersonationProxyCACertPEM []byte + + t.Log("Waiting for CredentialIssuer strategy to be successful") + + library.RequireEventuallyWithoutError(t, func() (bool, error) { + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil || credentialIssuer.Status.Strategies == nil { + t.Log("Did not find any CredentialIssuer with any strategies") + return false, nil // didn't find it, but keep trying + } + for _, strategy := range credentialIssuer.Status.Strategies { + // There will be other strategy types in the list, so ignore those. + if strategy.Type == v1alpha1.ImpersonationProxyStrategyType && strategy.Status == v1alpha1.SuccessStrategyStatus { //nolint:nestif + if strategy.Frontend == nil { + return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test + } + if strategy.Frontend.ImpersonationProxyInfo == nil { + return false, fmt.Errorf("did not find an ImpersonationProxyInfo") // unexpected, fail the test + } + impersonationProxyURL = strategy.Frontend.ImpersonationProxyInfo.Endpoint + impersonationProxyCACertPEM, err = base64.StdEncoding.DecodeString(strategy.Frontend.ImpersonationProxyInfo.CertificateAuthorityData) + if err != nil { + return false, err // unexpected, fail the test + } + return true, nil // found it, continue the test! + } else if strategy.Type == v1alpha1.ImpersonationProxyStrategyType { + t.Logf("Waiting for successful impersonation proxy strategy on %s: found status %s with reason %s and message: %s", + credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) + if strategy.Reason == v1alpha1.ErrorDuringSetupStrategyReason { + // The server encountered an unexpected error while starting the impersonator, so fail the test fast. + return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) + } + } + } + t.Log("Did not find any impersonation proxy strategy on CredentialIssuer") + return false, nil // didn't find it, but keep trying + }, 10*time.Minute, 10*time.Second) + + t.Log("Found successful CredentialIssuer strategy") + return impersonationProxyURL, impersonationProxyCACertPEM +} + +func requireDisabledByConfigurationStrategy(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient versioned.Interface) { + t.Helper() + + library.RequireEventuallyWithoutError(t, func() (bool, error) { + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil || credentialIssuer.Status.Strategies == nil { + t.Log("Did not find any CredentialIssuer with any strategies") + return false, nil // didn't find it, but keep trying + } + for _, strategy := range credentialIssuer.Status.Strategies { + // There will be other strategy types in the list, so ignore those. + if strategy.Type == v1alpha1.ImpersonationProxyStrategyType && + strategy.Status == v1alpha1.ErrorStrategyStatus && + strategy.Reason == v1alpha1.DisabledStrategyReason { //nolint:nestif + return true, nil // found it, continue the test! + } else if strategy.Type == v1alpha1.ImpersonationProxyStrategyType { + t.Logf("Waiting for disabled impersonation proxy strategy on %s: found status %s with reason %s and message: %s", + credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) + if strategy.Reason == v1alpha1.ErrorDuringSetupStrategyReason { + // The server encountered an unexpected error while stopping the impersonator, so fail the test fast. + return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) + } + } + } + t.Log("Did not find any impersonation proxy strategy on CredentialIssuer") + return false, nil // didn't find it, but keep trying + }, 1*time.Minute, 500*time.Millisecond) } func configMapForConfig(t *testing.T, env *library.TestEnv, config impersonator.Config) corev1.ConfigMap { + t.Helper() configString, err := yaml.Marshal(config) require.NoError(t, err) configMap := corev1.ConfigMap{ @@ -438,21 +501,6 @@ func hasImpersonationProxyLoadBalancerService(ctx context.Context, env *library. return service.Spec.Type == corev1.ServiceTypeLoadBalancer, nil } -func getImpersonationProxyLoadBalancerIngress(ctx context.Context, env *library.TestEnv, client kubernetes.Interface) (*corev1.LoadBalancerIngress, error) { - service, err := client.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) - if err != nil { - return nil, err - } - ingresses := service.Status.LoadBalancer.Ingress - if len(ingresses) > 1 { - return nil, fmt.Errorf("didn't expect multiple ingresses, but if it happens then maybe this test needs to be adjusted") - } - if len(ingresses) == 0 { - return nil, nil - } - return &ingresses[0], nil -} - func impersonationProxyConfigMapName(env *library.TestEnv) string { return env.ConciergeAppName + "-impersonation-proxy-config" } @@ -468,3 +516,7 @@ func impersonationProxyCASecretName(env *library.TestEnv) string { func impersonationProxyLoadBalancerName(env *library.TestEnv) string { return env.ConciergeAppName + "-impersonation-proxy-load-balancer" } + +func credentialIssuerName(env *library.TestEnv) string { + return env.ConciergeAppName + "-config" +} From 1b3103c9b5a341e51b60bd1a535840bd6cb6b684 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 13:37:03 -0800 Subject: [PATCH 074/203] Remove a nolint comment to satisfy the version of the linter used in CI --- internal/concierge/impersonator/impersonator.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index c189daf0..7c88ed67 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -28,7 +28,6 @@ import ( "go.pinniped.dev/internal/kubeclient" ) -// nolint: gochecknoglobals var impersonateHeaderRegex = regexp.MustCompile("Impersonate-.*") type proxy struct { From 58607c7e8198de0374a24088702597b4c9244864 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 14:19:24 -0800 Subject: [PATCH 075/203] Update `TestCredentialIssuer` int test to ignore ImpersonationProxy type --- .../concierge_credentialissuer_test.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/integration/concierge_credentialissuer_test.go b/test/integration/concierge_credentialissuer_test.go index c7ed6e66..4a4881cb 100644 --- a/test/integration/concierge_credentialissuer_test.go +++ b/test/integration/concierge_credentialissuer_test.go @@ -64,9 +64,20 @@ func TestCredentialIssuer(t *testing.T) { // Verify the cluster strategy status based on what's expected of the test cluster's ability to share signing keys. actualStatusStrategies := actualConfigList.Items[0].Status.Strategies - require.Len(t, actualStatusStrategies, 1) - actualStatusStrategy := actualStatusStrategies[0] - require.Equal(t, configv1alpha1.KubeClusterSigningCertificateStrategyType, actualStatusStrategy.Type) + + // There should be two. One of type KubeClusterSigningCertificate and one of type ImpersonationProxy. + require.Len(t, actualStatusStrategies, 2) + + // The details of the ImpersonationProxy type is tested by a different integration test for the impersonator. + // Grab the KubeClusterSigningCertificate result so we can check it in detail below. + var actualStatusStrategy configv1alpha1.CredentialIssuerStrategy + for _, s := range actualStatusStrategies { + if s.Type == configv1alpha1.KubeClusterSigningCertificateStrategyType { + actualStatusStrategy = s + break + } + } + require.NotNil(t, actualStatusStrategy) if env.HasCapability(library.ClusterSigningKeyIsAvailable) { require.Equal(t, configv1alpha1.SuccessStrategyStatus, actualStatusStrategy.Status) From 7c9aff3278e50e5ee32b3d1a4c3654e225a3d947 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 3 Mar 2021 16:49:33 -0600 Subject: [PATCH 076/203] Allow TestE2EFullIntegration to run on clusters where only the impersonation proxy works. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 87a9646c..7fe6d351 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -41,7 +41,7 @@ import ( // TestE2EFullIntegration tests a full integration scenario that combines the supervisor, concierge, and CLI. func TestE2EFullIntegration(t *testing.T) { - env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + env := library.IntegrationEnv(t) // If anything in this test crashes, dump out the supervisor and proxy pod logs. defer library.DumpLogs(t, env.SupervisorNamespace, "") From 48f2ae9eb4ea36ab3585de2aef72a5c7b79ca033 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 15:17:45 -0800 Subject: [PATCH 077/203] Fix a typo in concierge_impersonation_proxy_test.go --- test/integration/concierge_impersonation_proxy_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 6365eff9..d4b6bb50 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -95,8 +95,6 @@ func TestImpersonationProxy(t *testing.T) { require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{})) } - impersonationProxyLoadBalancerIngress := "" - if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test // Check that load balancer has been automatically created by the impersonator's "auto" mode. library.RequireEventuallyWithoutError(t, func() (bool, error) { @@ -331,7 +329,7 @@ func TestImpersonationProxy(t *testing.T) { // impersonate headers to the request. var doubleImpersonationClient *kubernetes.Clientset if env.HasCapability(library.HasExternalLoadBalancerProvider) { - doubleImpersonationClient = impersonationProxyViaLoadBalancerClient(impersonationProxyLoadBalancerIngress, impersonationProxyCACertPEM, "other-user-to-impersonate") + doubleImpersonationClient = impersonationProxyViaLoadBalancerClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate") } else { doubleImpersonationClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "other-user-to-impersonate") } From 9c1c760f561163f34aef3333698890cf72111569 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 16:23:07 -0800 Subject: [PATCH 078/203] Always clean up the ConfigMap at the end of the impersonator int test Signed-off-by: Margo Crawford --- .../concierge_impersonation_proxy_test.go | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index d4b6bb50..b52c662f 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -94,6 +94,28 @@ func TestImpersonationProxy(t *testing.T) { t.Logf("stashing a pre-existing configmap %s", oldConfigMap.Name) require.NoError(t, adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{})) } + // At the end of the test, clean up the ConfigMap. + t.Cleanup(func() { + ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Delete any version that was created by this test. + t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName(env)) + err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{}) + if !k8serrors.IsNotFound(err) { + require.NoError(t, err) // only not found errors are acceptable + } + + // Only recreate it if it already existed at the start of this test. + if len(oldConfigMap.Data) != 0 { + t.Log(oldConfigMap) + oldConfigMap.UID = "" // cant have a UID yet + oldConfigMap.ResourceVersion = "" + t.Logf("restoring a pre-existing configmap %s", oldConfigMap.Name) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, oldConfigMap, metav1.CreateOptions{}) + require.NoError(t, err) + } + }) if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test // Check that load balancer has been automatically created by the impersonator's "auto" mode. @@ -122,24 +144,6 @@ func TestImpersonationProxy(t *testing.T) { t.Logf("creating configmap %s", configMap.Name) _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) require.NoError(t, err) - - // At the end of the test, clean up the ConfigMap. - t.Cleanup(func() { - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName(env)) - err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{}) - require.NoError(t, err) - - if len(oldConfigMap.Data) != 0 { - t.Log(oldConfigMap) - oldConfigMap.UID = "" // cant have a UID yet - oldConfigMap.ResourceVersion = "" - t.Logf("restoring a pre-existing configmap %s", oldConfigMap.Name) - _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, oldConfigMap, metav1.CreateOptions{}) - require.NoError(t, err) - } - }) } // At this point the impersonator should be starting/running. When it is ready, the CredentialIssuer's From 5697adc36ae8e273018998a4d60592b13279f52b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 3 Mar 2021 17:24:10 -0800 Subject: [PATCH 079/203] Revert "Allow TestE2EFullIntegration to run on clusters where only the impersonation proxy works." This reverts commit 7c9aff3278e50e5ee32b3d1a4c3654e225a3d947. --- test/integration/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 7fe6d351..87a9646c 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -41,7 +41,7 @@ import ( // TestE2EFullIntegration tests a full integration scenario that combines the supervisor, concierge, and CLI. func TestE2EFullIntegration(t *testing.T) { - env := library.IntegrationEnv(t) + env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) // If anything in this test crashes, dump out the supervisor and proxy pod logs. defer library.DumpLogs(t, env.SupervisorNamespace, "") From 03f09c6870fc3c903e9b9b8fca81b005ebca0faa Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Wed, 3 Mar 2021 16:49:33 -0600 Subject: [PATCH 080/203] Allow TestE2EFullIntegration to run on clusters where only the impersonation proxy works (again). This time, don't use the Squid proxy if the cluster supports real external load balancers (as in EKS/GKE/AKS). Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 87a9646c..35f67e1d 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -41,7 +41,7 @@ import ( // TestE2EFullIntegration tests a full integration scenario that combines the supervisor, concierge, and CLI. func TestE2EFullIntegration(t *testing.T) { - env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + env := library.IntegrationEnv(t) // If anything in this test crashes, dump out the supervisor and proxy pod logs. defer library.DumpLogs(t, env.SupervisorNamespace, "") @@ -157,6 +157,8 @@ func TestE2EFullIntegration(t *testing.T) { ) require.Equal(t, "", stderr) + t.Logf("test kubeconfig:\n%s\n\n", kubeconfigYAML) + restConfig := library.NewRestConfigFromKubeconfig(t, kubeconfigYAML) require.NotNil(t, restConfig.ExecProvider) require.Equal(t, []string{"login", "oidc"}, restConfig.ExecProvider.Args[:2]) @@ -171,7 +173,9 @@ func TestE2EFullIntegration(t *testing.T) { // Run "kubectl get namespaces" which should trigger a browser login via the plugin. start := time.Now() kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) - kubectlCmd.Env = append(os.Environ(), env.ProxyEnv()...) + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + kubectlCmd.Env = append(os.Environ(), env.ProxyEnv()...) + } stderrPipe, err := kubectlCmd.StderrPipe() require.NoError(t, err) stdoutPipe, err := kubectlCmd.StdoutPipe() @@ -279,7 +283,9 @@ func TestE2EFullIntegration(t *testing.T) { // Run kubectl again, which should work with no browser interaction. kubectlCmd2 := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) - kubectlCmd2.Env = append(os.Environ(), env.ProxyEnv()...) + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + kubectlCmd2.Env = append(os.Environ(), env.ProxyEnv()...) + } start = time.Now() kubectlOutput2, err := kubectlCmd2.CombinedOutput() require.NoError(t, err) From ddd1d29e5dde570909e323cdf0dc75fb334d9599 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 12:24:57 -0600 Subject: [PATCH 081/203] Fix "pinniped get kubeconfig" strategy detection to pick the _first_ working strategy. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 14 +++++----- cmd/pinniped/cmd/kubeconfig_test.go | 40 ++++++++++++++++++++--------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index c1c819bf..562d3190 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -251,23 +251,25 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // Autodiscover the --concierge-mode. if flags.concierge.mode == modeUnknown { //nolint:nestif + + strategyLoop: for _, strategy := range credentialIssuer.Status.Strategies { - fe := strategy.Frontend - if strategy.Status != configv1alpha1.SuccessStrategyStatus || fe == nil { + if strategy.Status != configv1alpha1.SuccessStrategyStatus || strategy.Frontend == nil { continue } - - switch fe.Type { + switch strategy.Frontend.Type { case configv1alpha1.TokenCredentialRequestAPIFrontendType: flags.concierge.mode = modeTokenCredentialRequestAPI + break strategyLoop case configv1alpha1.ImpersonationProxyFrontendType: flags.concierge.mode = modeImpersonationProxy - flags.concierge.endpoint = fe.ImpersonationProxyInfo.Endpoint + flags.concierge.endpoint = strategy.Frontend.ImpersonationProxyInfo.Endpoint var err error - conciergeCABundleData, err = base64.StdEncoding.DecodeString(fe.ImpersonationProxyInfo.CertificateAuthorityData) + conciergeCABundleData, err = base64.StdEncoding.DecodeString(strategy.Frontend.ImpersonationProxyInfo.CertificateAuthorityData) if err != nil { return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) } + break strategyLoop default: // Skip any unknown frontend types. } diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index c5427e35..79fdb2dd 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -788,20 +788,36 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{{ - Type: "SomeType", - Status: configv1alpha1.SuccessStrategyStatus, - Reason: "SomeReason", - Message: "Some message", - LastUpdateTime: metav1.Now(), - Frontend: &configv1alpha1.CredentialIssuerFrontend{ - Type: configv1alpha1.ImpersonationProxyFrontendType, - ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ - Endpoint: "https://impersonation-proxy-endpoint.test", - CertificateAuthorityData: "dGVzdC1jb25jaWVyZ2UtY2E=", + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + { + Type: "SomeType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.test", + CertificateAuthorityData: "dGVzdC1jb25jaWVyZ2UtY2E=", + }, }, }, - }}, + { + Type: "SomeOtherType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeOtherReason", + Message: "Some other message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://some-other-impersonation-endpoint", + CertificateAuthorityData: "dGVzdC1jb25jaWVyZ2UtY2E=", + }, + }, + }, + }, }, }, &conciergev1alpha1.JWTAuthenticator{ From 9a0f75980d8b62a81fa52c9f543727e84c9d7502 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 12:35:29 -0600 Subject: [PATCH 082/203] Set a special proxy environment just for the "pinniped login oidc" command in the E2E test. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 35f67e1d..ba77759b 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -26,6 +26,7 @@ import ( authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" authv1alpha "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" @@ -162,6 +163,17 @@ func TestE2EFullIntegration(t *testing.T) { restConfig := library.NewRestConfigFromKubeconfig(t, kubeconfigYAML) require.NotNil(t, restConfig.ExecProvider) require.Equal(t, []string{"login", "oidc"}, restConfig.ExecProvider.Args[:2]) + + // If there is a proxy, we always want the "pinniped login oidc" command to use it, even if the + // parent kubectl process is connecting to an external load balancer and not using the proxy. + if env.Proxy != "" { + restConfig.ExecProvider.Env = append(restConfig.ExecProvider.Env, + clientcmdapi.ExecEnvVar{Name: "http_proxy", Value: env.Proxy}, + clientcmdapi.ExecEnvVar{Name: "https_proxy", Value: env.Proxy}, + clientcmdapi.ExecEnvVar{Name: "no_proxy", Value: "127.0.0.1"}, + ) + } + kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) From 9dfbe60253e5c8bb5c86e8d042b83ca4f74f54ee Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 14:41:20 -0600 Subject: [PATCH 083/203] Do the kubeconfig proxy environment injection, but actually render back out the YAML. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 10 ++------- test/library/env.go | 41 +++++++++++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index ba77759b..69a17b95 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -26,7 +26,6 @@ import ( authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" authv1alpha "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" @@ -166,13 +165,8 @@ func TestE2EFullIntegration(t *testing.T) { // If there is a proxy, we always want the "pinniped login oidc" command to use it, even if the // parent kubectl process is connecting to an external load balancer and not using the proxy. - if env.Proxy != "" { - restConfig.ExecProvider.Env = append(restConfig.ExecProvider.Env, - clientcmdapi.ExecEnvVar{Name: "http_proxy", Value: env.Proxy}, - clientcmdapi.ExecEnvVar{Name: "https_proxy", Value: env.Proxy}, - clientcmdapi.ExecEnvVar{Name: "no_proxy", Value: "127.0.0.1"}, - ) - } + kubeconfigYAML = env.InjectProxyEnvIntoKubeconfig(kubeconfigYAML) + t.Logf("test kubeconfig after proxy environment addition:\n%s\n\n", kubeconfigYAML) kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) diff --git a/test/library/env.go b/test/library/env.go index 0cf28a42..ba2b3b2c 100644 --- a/test/library/env.go +++ b/test/library/env.go @@ -10,6 +10,8 @@ import ( "testing" "github.com/stretchr/testify/require" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/yaml" auth1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" @@ -67,10 +69,47 @@ type TestOIDCUpstream struct { // ProxyEnv returns a set of environment variable strings (e.g., to combine with os.Environ()) which set up the configured test HTTP proxy. func (e *TestEnv) ProxyEnv() []string { + vars := e.proxyVars() + if vars == nil { + return nil + } + res := make([]string, 0, len(vars)) + for k, v := range vars { + res = append(res, k+"="+v) + } + return res +} + +func (e *TestEnv) InjectProxyEnvIntoKubeconfig(kubeconfigYAML string) string { + proxyVars := e.proxyVars() + if proxyVars == nil { + return kubeconfigYAML + } + + kubeconfig, err := clientcmd.Load([]byte(kubeconfigYAML)) + require.NoError(e.t, err) + for i := range kubeconfig.AuthInfos { + if exec := kubeconfig.AuthInfos[i].Exec; exec != nil { + for k, v := range proxyVars { + exec.Env = append(exec.Env, clientcmdapi.ExecEnvVar{Name: k, Value: v}) + } + } + } + + newYAML, err := clientcmd.Write(*kubeconfig) + require.NoError(t, err) + return string(newYAML) +} + +func (e *TestEnv) proxyVars() map[string] { if e.Proxy == "" { return nil } - return []string{"http_proxy=" + e.Proxy, "https_proxy=" + e.Proxy, "no_proxy=127.0.0.1"} + return map[string]string{ + "http_proxy": e.Proxy, + "https_proxy": e.Proxy, + "no_proxy": "127.0.0.1", + } } // IntegrationEnv gets the integration test environment from OS environment variables. This From 7146cb3880e88650d09724e0a58e744b578cbc55 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 15:02:42 -0600 Subject: [PATCH 084/203] Remove old debug-make-impersonation-token command. Signed-off-by: Matt Moyer --- cmd/debug-make-impersonation-token/main.go | 41 ---------------------- 1 file changed, 41 deletions(-) delete mode 100644 cmd/debug-make-impersonation-token/main.go diff --git a/cmd/debug-make-impersonation-token/main.go b/cmd/debug-make-impersonation-token/main.go deleted file mode 100644 index 972ab27e..00000000 --- a/cmd/debug-make-impersonation-token/main.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package main - -import ( - "encoding/base64" - "encoding/json" - "fmt" - "os" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" -) - -func main() { - reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: os.Getenv("PINNIPED_TEST_CONCIERGE_NAMESPACE"), - }, - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginv1alpha1.GroupName + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: os.Getenv("PINNIPED_TEST_USER_TOKEN"), - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, - Kind: os.Getenv("PINNIPED_AUTHENTICATOR_KIND"), - Name: os.Getenv("PINNIPED_AUTHENTICATOR_NAME"), - }, - }, - }) - if err != nil { - panic(err) - } - fmt.Println(base64.StdEncoding.EncodeToString(reqJSON)) -} From 274e6281a8f20ea1ee7cfbfe08dbae0f3bb8e4cc Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 15:21:17 -0600 Subject: [PATCH 085/203] Whoops, missed these fixes in test/library/env.go. Signed-off-by: Matt Moyer --- test/library/env.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/library/env.go b/test/library/env.go index ba2b3b2c..d78f7d8b 100644 --- a/test/library/env.go +++ b/test/library/env.go @@ -97,11 +97,11 @@ func (e *TestEnv) InjectProxyEnvIntoKubeconfig(kubeconfigYAML string) string { } newYAML, err := clientcmd.Write(*kubeconfig) - require.NoError(t, err) + require.NoError(e.t, err) return string(newYAML) } -func (e *TestEnv) proxyVars() map[string] { +func (e *TestEnv) proxyVars() map[string]string { if e.Proxy == "" { return nil } From 34e15f03c3da118b7c84b9fe524e003e9ac32ab4 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 15:17:42 -0600 Subject: [PATCH 086/203] Simplify const declarations in flag_types.go. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/flag_types.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index ad682a1d..73c8cfd1 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -16,9 +16,9 @@ type conciergeMode int var _ flag.Value = new(conciergeMode) const ( - modeUnknown conciergeMode = iota - modeTokenCredentialRequestAPI conciergeMode = iota - modeImpersonationProxy conciergeMode = iota + modeUnknown conciergeMode = iota + modeTokenCredentialRequestAPI + modeImpersonationProxy ) func (c *conciergeMode) String() string { From d24cf4b8a787ad8c4f51b164be8b8a256447879d Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 16:05:56 -0600 Subject: [PATCH 087/203] Go back to testing entirely through the proxy, but add a retry loop during the first connection. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 69a17b95..501e7655 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -26,6 +26,7 @@ import ( authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/client-go/rest" authv1alpha "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" @@ -162,12 +163,6 @@ func TestE2EFullIntegration(t *testing.T) { restConfig := library.NewRestConfigFromKubeconfig(t, kubeconfigYAML) require.NotNil(t, restConfig.ExecProvider) require.Equal(t, []string{"login", "oidc"}, restConfig.ExecProvider.Args[:2]) - - // If there is a proxy, we always want the "pinniped login oidc" command to use it, even if the - // parent kubectl process is connecting to an external load balancer and not using the proxy. - kubeconfigYAML = env.InjectProxyEnvIntoKubeconfig(kubeconfigYAML) - t.Logf("test kubeconfig after proxy environment addition:\n%s\n\n", kubeconfigYAML) - kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) @@ -179,9 +174,7 @@ func TestE2EFullIntegration(t *testing.T) { // Run "kubectl get namespaces" which should trigger a browser login via the plugin. start := time.Now() kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) - if !env.HasCapability(library.HasExternalLoadBalancerProvider) { - kubectlCmd.Env = append(os.Environ(), env.ProxyEnv()...) - } + kubectlCmd.Env = append(os.Environ(), env.ProxyEnv()...) stderrPipe, err := kubectlCmd.StderrPipe() require.NoError(t, err) stdoutPipe, err := kubectlCmd.StdoutPipe() @@ -276,6 +269,19 @@ func TestE2EFullIntegration(t *testing.T) { require.NoError(t, err) require.Equal(t, "you have been logged in and may now close this tab", msg) + // Verify that we can actually reach the endpoint in the kubeconfig. + restClient, err := rest.RESTClientFor(restConfig) + require.NoError(t, err) + require.Eventually(t, func() bool { + var status int + _, err := restClient.Get().AbsPath("/version").Do(ctx).StatusCode(&status).Raw() + if status == 200 || status == 403 { + return true + } + t.Logf("attempted to connect to API server's /version endpoint, got response code %d with error %v", status, err) + return false + }, 5*time.Minute, time.Second) + // Expect the CLI to output a list of namespaces in JSON format. t.Logf("waiting for kubectl to output namespace list JSON") var kubectlOutput string @@ -289,9 +295,7 @@ func TestE2EFullIntegration(t *testing.T) { // Run kubectl again, which should work with no browser interaction. kubectlCmd2 := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) - if !env.HasCapability(library.HasExternalLoadBalancerProvider) { - kubectlCmd2.Env = append(os.Environ(), env.ProxyEnv()...) - } + kubectlCmd2.Env = append(os.Environ(), env.ProxyEnv()...) start = time.Now() kubectlOutput2, err := kubectlCmd2.CombinedOutput() require.NoError(t, err) From 6a8f377781ee821b928e513ae4f111ba8dfd12a1 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 16:16:03 -0600 Subject: [PATCH 088/203] Fix a linter warning. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 562d3190..6a4e8327 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -251,7 +251,6 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // Autodiscover the --concierge-mode. if flags.concierge.mode == modeUnknown { //nolint:nestif - strategyLoop: for _, strategy := range credentialIssuer.Status.Strategies { if strategy.Status != configv1alpha1.SuccessStrategyStatus || strategy.Frontend == nil { From 165fce67af44815bfad2999e62fa96b0b1d6ae58 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 16:23:39 -0600 Subject: [PATCH 089/203] Use the unversioned REST client for this check. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 501e7655..672df23e 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -270,7 +270,7 @@ func TestE2EFullIntegration(t *testing.T) { require.Equal(t, "you have been logged in and may now close this tab", msg) // Verify that we can actually reach the endpoint in the kubeconfig. - restClient, err := rest.RESTClientFor(restConfig) + restClient, err := rest.UnversionedRESTClientFor(restConfig) require.NoError(t, err) require.Eventually(t, func() bool { var status int From 16163b989b74f2df68787e3c94f0d86d51e0c15d Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 17:18:24 -0600 Subject: [PATCH 090/203] Use regular http.Client in this test. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 37 +++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 672df23e..fb7b8b90 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -6,11 +6,14 @@ import ( "bufio" "bytes" "context" + "crypto/tls" + "crypto/x509" "crypto/x509/pkix" "encoding/base64" "errors" "fmt" "io/ioutil" + "net/http" "net/url" "os" "os/exec" @@ -26,7 +29,6 @@ import ( authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/client-go/rest" authv1alpha "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" @@ -270,16 +272,33 @@ func TestE2EFullIntegration(t *testing.T) { require.Equal(t, "you have been logged in and may now close this tab", msg) // Verify that we can actually reach the endpoint in the kubeconfig. - restClient, err := rest.UnversionedRESTClientFor(restConfig) - require.NoError(t, err) require.Eventually(t, func() bool { - var status int - _, err := restClient.Get().AbsPath("/version").Do(ctx).StatusCode(&status).Raw() - if status == 200 || status == 403 { - return true + kubeconfigCA := x509.NewCertPool() + require.True(t, kubeconfigCA.AppendCertsFromPEM(restConfig.TLSClientConfig.CAData), "expected to load kubeconfig CA") + + // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{RootCAs: kubeconfigCA}, + Proxy: func(req *http.Request) (*url.URL, error) { + if env.Proxy == "" { + t.Logf("passing request for %s with no proxy", req.URL) + return nil, nil + } + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + }, + }, } - t.Logf("attempted to connect to API server's /version endpoint, got response code %d with error %v", status, err) - return false + resp, err := httpClient.Get(restConfig.Host) + if err != nil { + t.Logf("could not connect to the API server at %q: %v", restConfig.Host, err) + return false + } + t.Logf("got %d response from API server at %q", resp.StatusCode, restConfig.Host) + return resp.StatusCode == 200 || resp.StatusCode == 403 }, 5*time.Minute, time.Second) // Expect the CLI to output a list of namespaces in JSON format. From fea626b6547eaf3e9c4f90094b837cff56278e2c Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 4 Mar 2021 17:19:59 -0600 Subject: [PATCH 091/203] Remove this proxy-related test code that we ended up not needing. Signed-off-by: Matt Moyer --- test/library/env.go | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/test/library/env.go b/test/library/env.go index d78f7d8b..0cf28a42 100644 --- a/test/library/env.go +++ b/test/library/env.go @@ -10,8 +10,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "k8s.io/client-go/tools/clientcmd" - clientcmdapi "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/yaml" auth1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" @@ -69,47 +67,10 @@ type TestOIDCUpstream struct { // ProxyEnv returns a set of environment variable strings (e.g., to combine with os.Environ()) which set up the configured test HTTP proxy. func (e *TestEnv) ProxyEnv() []string { - vars := e.proxyVars() - if vars == nil { - return nil - } - res := make([]string, 0, len(vars)) - for k, v := range vars { - res = append(res, k+"="+v) - } - return res -} - -func (e *TestEnv) InjectProxyEnvIntoKubeconfig(kubeconfigYAML string) string { - proxyVars := e.proxyVars() - if proxyVars == nil { - return kubeconfigYAML - } - - kubeconfig, err := clientcmd.Load([]byte(kubeconfigYAML)) - require.NoError(e.t, err) - for i := range kubeconfig.AuthInfos { - if exec := kubeconfig.AuthInfos[i].Exec; exec != nil { - for k, v := range proxyVars { - exec.Env = append(exec.Env, clientcmdapi.ExecEnvVar{Name: k, Value: v}) - } - } - } - - newYAML, err := clientcmd.Write(*kubeconfig) - require.NoError(e.t, err) - return string(newYAML) -} - -func (e *TestEnv) proxyVars() map[string]string { if e.Proxy == "" { return nil } - return map[string]string{ - "http_proxy": e.Proxy, - "https_proxy": e.Proxy, - "no_proxy": "127.0.0.1", - } + return []string{"http_proxy=" + e.Proxy, "https_proxy=" + e.Proxy, "no_proxy=127.0.0.1"} } // IntegrationEnv gets the integration test environment from OS environment variables. This From 9eb97e2683798337cc89af0bff830da73685f204 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 4 Mar 2021 13:52:34 -0800 Subject: [PATCH 092/203] Use Eventually when making tls connections and avoid resource version 0 - Use `Eventually` when making tls connections because the production code's handling of starting and stopping the TLS server port has some async behavior. - Don't use resource version "0" because that has special meaning in the informer libraries. --- .../impersonator_config_test.go | 109 ++++++++++-------- 1 file changed, 61 insertions(+), 48 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 73fb91c2..75161d55 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -15,12 +15,14 @@ import ( "io/ioutil" "net" "net/http" + "regexp" "strings" "testing" "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -286,6 +288,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { const caSecretName = "some-ca-secret-name" const localhostIP = "127.0.0.1" const httpsPort = ":443" + const fakeServerResponseBody = "hello, world!" var labels = map[string]string{"app": "app-name", "other-key": "other-value"} var r *require.Assertions @@ -367,14 +370,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { url := "https://" + addr req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) - resp, err := client.Do(req) + var resp *http.Response + assert.Eventually(t, func() bool { + resp, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose + return err == nil + }, 5*time.Second, 50*time.Millisecond) r.NoError(err) r.Equal(http.StatusOK, resp.StatusCode) body, err := ioutil.ReadAll(resp.Body) r.NoError(resp.Body.Close()) r.NoError(err) - r.Equal("hello world", string(body)) + r.Equal(fakeServerResponseBody, string(body)) } var requireTLSServerIsRunningWithoutCerts = func() { @@ -386,20 +393,33 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { url := "https://" + testServerAddr() req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) - _, err = client.Do(req) //nolint:bodyclose + expectedErrorRegex := "Get .*: remote error: tls: unrecognized name" + expectedErrorRegexCompiled, err := regexp.Compile(expectedErrorRegex) + r.NoError(err) + assert.Eventually(t, func() bool { + _, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose + return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) + }, 5*time.Second, 50*time.Millisecond) r.Error(err) - r.Regexp("Get .*: remote error: tls: unrecognized name", err.Error()) + r.Regexp(expectedErrorRegex, err.Error()) } var requireTLSServerIsNoLongerRunning = func() { r.Greater(startTLSListenerFuncWasCalled, 0) - _, err := tls.Dial( - startedTLSListener.Addr().Network(), - testServerAddr(), - &tls.Config{InsecureSkipVerify: true}, //nolint:gosec - ) + var err error + expectedErrorRegex := "dial tcp .*: connect: connection refused" + expectedErrorRegexCompiled, err := regexp.Compile(expectedErrorRegex) + r.NoError(err) + assert.Eventually(t, func() bool { + _, err = tls.Dial( + startedTLSListener.Addr().Network(), + testServerAddr(), + &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + ) + return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) + }, 5*time.Second, 50*time.Millisecond) r.Error(err) - r.Regexp(`dial tcp .*: connect: connection refused`, err.Error()) + r.Regexp(expectedErrorRegex, err.Error()) } var requireTLSServerWasNeverStarted = func() { @@ -449,7 +469,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startTLSListenerFunc, func() (http.Handler, error) { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, err := fmt.Fprintf(w, "hello world") + _, err := fmt.Fprint(w, fakeServerResponseBody) r.NoError(err) }), httpHandlerFactoryFuncError }, @@ -475,10 +495,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: resourceName, Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", }, Data: map[string]string{ "config.yaml": configYAML, @@ -511,10 +527,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: resourceName, Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", }, Data: data, } @@ -564,13 +576,16 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { createdSecret, ok := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) - r.True(ok, "should have been able to cast this action to CreateAction: %v", action) + r.True(ok, "should have been able to cast this action's object to Secret: %v", action) + createdSecret = createdSecret.DeepCopy() createdSecret.ResourceVersion = resourceVersion r.NoError(client.Tracker().Add(createdSecret)) } var addServiceFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { - createdService := action.(coretesting.CreateAction).GetObject().(*corev1.Service) + createdService, ok := action.(coretesting.CreateAction).GetObject().(*corev1.Service) + r.True(ok, "should have been able to cast this action's object to Service: %v", action) + createdService = createdService.DeepCopy() createdService.ResourceVersion = resourceVersion r.NoError(client.Tracker().Add(createdService)) } @@ -580,10 +595,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: resourceName, Namespace: installedInNamespace, - // Note that this seems to be ignored by the informer during initial creation, so actually - // the informer will see this as resource version "". Leaving it here to express the intent - // that the initial version is version 0. - ResourceVersion: "0", }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, @@ -606,7 +617,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var addSecretToTrackers = func(secret *corev1.Secret, clients ...*kubernetesfake.Clientset) { for _, client := range clients { - r.NoError(client.Tracker().Add(secret)) + secretCopy := secret.DeepCopy() + r.NoError(client.Tracker().Add(secretCopy)) } } @@ -618,6 +630,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ) r.NoError(err) service := serviceObj.(*corev1.Service) + service = service.DeepCopy() service.ResourceVersion = newResourceVersion service.Status = corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingresses}} r.NoError(client.Tracker().Update( @@ -972,10 +985,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // keeps the secret around after resync r.NoError(runControllerSync()) @@ -1003,10 +1016,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // keeps the secret around after resync r.NoError(runControllerSync()) @@ -1034,10 +1047,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") // keeps the secret around after resync r.NoError(runControllerSync()) @@ -1706,13 +1719,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time @@ -1744,13 +1757,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "0") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "0") - - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient, "1") + addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time @@ -1760,8 +1773,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(hostname, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time @@ -1925,7 +1938,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { Namespace: installedInNamespace, }, Data: map[string][]byte{ - // "aGVsbG8gd29ybGQK" is "hello world" base64 encoded + // "aGVsbG8gd29ybGQK" is "hello world" base64 encoded which is not a valid cert corev1.TLSCertKey: []byte("-----BEGIN CERTIFICATE-----\naGVsbG8gd29ybGQK\n-----END CERTIFICATE-----\n"), }, } From b102aa89916b19f77b543c8c8961b58f1bf2c86d Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 4 Mar 2021 15:36:51 -0800 Subject: [PATCH 093/203] In unit test, wait for obj from informer instead of resource version In impersonator_config_test.go, instead of waiting for the resource version to appear in the informers, wait for the actual object to appear. This is an attempt to resolve flaky failures that only happen in CI, but it also cleans up the test a bit by avoiding inventing fake resource version numbers all over the test. Signed-off-by: Monis Khan --- .../impersonator_config_test.go | 275 ++++++++---------- 1 file changed, 117 insertions(+), 158 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 75161d55..60fe1982 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -15,6 +15,7 @@ import ( "io/ioutil" "net" "net/http" + "reflect" "regexp" "strings" "testing" @@ -25,21 +26,19 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/clock" kubeinformers "k8s.io/client-go/informers" - corev1informers "k8s.io/client-go/informers/core/v1" kubernetesfake "k8s.io/client-go/kubernetes/fake" coretesting "k8s.io/client-go/testing" - "k8s.io/client-go/tools/cache" "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" ) @@ -374,7 +373,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { assert.Eventually(t, func() bool { resp, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose return err == nil - }, 5*time.Second, 50*time.Millisecond) + }, 5*time.Second, 5*time.Millisecond) r.NoError(err) r.Equal(http.StatusOK, resp.StatusCode) @@ -399,7 +398,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { assert.Eventually(t, func() bool { _, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) - }, 5*time.Second, 50*time.Millisecond) + }, 5*time.Second, 5*time.Millisecond) r.Error(err) r.Regexp(expectedErrorRegex, err.Error()) } @@ -417,7 +416,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { &tls.Config{InsecureSkipVerify: true}, //nolint:gosec ) return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) - }, 5*time.Second, 50*time.Millisecond) + }, 5*time.Second, 5*time.Millisecond) r.Error(err) r.Regexp(expectedErrorRegex, err.Error()) } @@ -426,26 +425,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(0, startTLSListenerFuncWasCalled) } - var waitForInformerCacheToSeeResourceVersion = func(informer cache.SharedIndexInformer, wantVersion string) { - r.Eventually(func() bool { - return informer.LastSyncResourceVersion() == wantVersion - }, 10*time.Second, time.Millisecond) - } - - var waitForServiceToBeDeleted = func(informer corev1informers.ServiceInformer, name string) { - r.Eventually(func() bool { - _, err := informer.Lister().Services(installedInNamespace).Get(name) - return k8serrors.IsNotFound(err) - }, 10*time.Second, time.Millisecond) - } - - var waitForSecretToBeDeleted = func(informer corev1informers.SecretInformer, name string) { - r.Eventually(func() bool { - _, err := informer.Lister().Secrets(installedInNamespace).Get(name) - return k8serrors.IsNotFound(err) - }, 10*time.Second, time.Millisecond) - } - // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. var startInformersAndController = func() { @@ -503,25 +482,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(client.Tracker().Add(impersonatorConfigMap)) } - var updateImpersonatorConfigMapInTracker = func(resourceName, configYAML string, client *kubernetesfake.Clientset, newResourceVersion string) { - configMapObj, err := client.Tracker().Get( - schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, - installedInNamespace, - resourceName, - ) - r.NoError(err) - configMap := configMapObj.(*corev1.ConfigMap) - configMap.ResourceVersion = newResourceVersion - configMap.Data = map[string]string{ - "config.yaml": configYAML, - } - r.NoError(client.Tracker().Update( - schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, - configMap, - installedInNamespace, - )) - } - var newSecretWithData = func(resourceName string, data map[string][]byte) *corev1.Secret { return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -574,22 +534,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return newSecretWithData(resourceName, newTLSCertSecretData(ca, []string{"foo", "bar"}, ip)) } - var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { - createdSecret, ok := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) - r.True(ok, "should have been able to cast this action's object to Secret: %v", action) - createdSecret = createdSecret.DeepCopy() - createdSecret.ResourceVersion = resourceVersion - r.NoError(client.Tracker().Add(createdSecret)) - } - - var addServiceFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { - createdService, ok := action.(coretesting.CreateAction).GetObject().(*corev1.Service) - r.True(ok, "should have been able to cast this action's object to Service: %v", action) - createdService = createdService.DeepCopy() - createdService.ResourceVersion = resourceVersion - r.NoError(client.Tracker().Add(createdService)) - } - var newLoadBalancerService = func(resourceName string, status corev1.ServiceStatus) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -603,6 +547,74 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } + // Anytime an object is added/updated/deleted in the informer's client *after* the informer is started, then we + // need to wait for the informer's cache to asynchronously pick up that change from its "watch". + // If an object is added to the informer's client *before* the informer is started, then waiting is + // not needed because the informer's initial "list" will pick up the object. + var waitForObjectToAppearInInformer = func(obj kubeclient.Object, informer controllerlib.InformerGetter) { + r.Eventually(func() bool { + gotObj, exists, err := informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + obj.GetName()) + return err == nil && exists && reflect.DeepEqual(gotObj.(kubeclient.Object), obj) + }, 10*time.Second, 5*time.Millisecond) + } + + // See comment for waitForObjectToAppearInInformer above. + var waitForObjectToBeDeletedFromInformer = func(resourceName string, informer controllerlib.InformerGetter) { + r.Eventually(func() bool { + _, exists, err := informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + resourceName) + return err == nil && !exists + }, 10*time.Second, 5*time.Millisecond) + } + + var addObjectToInformerAndWait = func(obj kubeclient.Object, informer controllerlib.InformerGetter) { + r.NoError(kubeInformerClient.Tracker().Add(obj)) + waitForObjectToAppearInInformer(obj, informer) + } + + var addObjectFromCreateActionToInformerAndWait = func(action coretesting.Action, informer controllerlib.InformerGetter) { + createdObject, ok := action.(coretesting.CreateAction).GetObject().(kubeclient.Object) + r.True(ok, "should have been able to cast this action's object to kubeclient.Object: %v", action) + addObjectToInformerAndWait(createdObject, informer) + } + + var updateImpersonatorConfigMapInInformerAndWait = func(resourceName, configYAML string, informer controllerlib.InformerGetter) { + configMapObj, err := kubeInformerClient.Tracker().Get( + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + installedInNamespace, + resourceName, + ) + r.NoError(err) + configMap := configMapObj.(*corev1.ConfigMap) + configMap = configMap.DeepCopy() // don't edit the original from the tracker + configMap.Data = map[string]string{ + "config.yaml": configYAML, + } + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "configmaps"}, + configMap, + installedInNamespace, + )) + waitForObjectToAppearInInformer(configMap, informer) + } + + var updateLoadBalancerServiceInInformerAndWait = func(resourceName string, ingresses []corev1.LoadBalancerIngress, informer controllerlib.InformerGetter) { + serviceObj, err := kubeInformerClient.Tracker().Get( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + installedInNamespace, + resourceName, + ) + r.NoError(err) + service := serviceObj.(*corev1.Service) + service = service.DeepCopy() // don't edit the original from the tracker + service.Status = corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingresses}} + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + service, + installedInNamespace, + )) + waitForObjectToAppearInInformer(service, informer) + } + var addLoadBalancerServiceToTracker = func(resourceName string, client *kubernetesfake.Clientset) { loadBalancerService := newLoadBalancerService(resourceName, corev1.ServiceStatus{}) r.NoError(client.Tracker().Add(loadBalancerService)) @@ -622,24 +634,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var updateLoadBalancerServiceInTracker = func(resourceName string, ingresses []corev1.LoadBalancerIngress, client *kubernetesfake.Clientset, newResourceVersion string) { - serviceObj, err := client.Tracker().Get( - schema.GroupVersionResource{Version: "v1", Resource: "services"}, - installedInNamespace, - resourceName, - ) - r.NoError(err) - service := serviceObj.(*corev1.Service) - service = service.DeepCopy() - service.ResourceVersion = newResourceVersion - service.Status = corev1.ServiceStatus{LoadBalancer: corev1.LoadBalancerStatus{Ingress: ingresses}} - r.NoError(client.Tracker().Update( - schema.GroupVersionResource{Version: "v1", Resource: "services"}, - service, - installedInNamespace, - )) - } - var deleteServiceFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { r.NoError(client.Tracker().Delete( schema.GroupVersionResource{Version: "v1", Resource: "services"}, @@ -985,10 +979,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // keeps the secret around after resync r.NoError(runControllerSync()) @@ -1016,10 +1008,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // keeps the secret around after resync r.NoError(runControllerSync()) @@ -1047,10 +1037,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // keeps the secret around after resync r.NoError(runControllerSync()) @@ -1129,8 +1117,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: "not-an-ip"}}, kubeInformers.Core().V1().Services()) errString := "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name" r.EqualError(runControllerSync(), errString) @@ -1359,14 +1346,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // Switch the endpoint config to a hostname. - updateImpersonatorConfigMapInTracker(configMapResourceName, hostnameYAML, kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, hostnameYAML, kubeInformers.Core().V1().ConfigMaps()) r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 5) @@ -1378,12 +1362,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Simulate the informer cache's background update from its watch. deleteSecretFromTracker(tlsSecretName, kubeInformerClient) - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[4], kubeInformerClient, "3") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "3") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[4], kubeInformers.Core().V1().Secrets()) // Switch the endpoint config back to an IP. - updateImpersonatorConfigMapInTracker(configMapResourceName, ipAddressYAML, kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, ipAddressYAML, kubeInformers.Core().V1().ConfigMaps()) r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 7) @@ -1415,8 +1397,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch for the CA Secret. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) // Delete the TLS Secret that was just created from the Kube API server. Note that we never // simulated it getting added to the informer cache, so we don't need to remove it from there. @@ -1452,8 +1433,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch for the CA Secret. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // Delete the CA Secret that was just created from the Kube API server. Note that we never // simulated it getting added to the informer cache, so we don't need to remove it from there. @@ -1489,8 +1469,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) // Simulate the informer cache's background update from its watch for the CA Secret. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // Simulate someone updating the CA Secret out of band, e.g. when a human edits it with kubectl. // Delete the CA Secret that was just created from the Kube API server. Note that we never @@ -1500,9 +1479,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { anotherCA := newCA() newCASecret := newActualCASecret(anotherCA, caSecretName) caCrt = newCASecret.Data["ca.crt"] - newCASecret.ResourceVersion = "2" - addSecretToTrackers(newCASecret, kubeInformerClient, kubeAPIClient) - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addSecretToTrackers(newCASecret, kubeAPIClient) + addObjectToInformerAndWait(newCASecret, kubeInformers.Core().V1().Secrets()) }) it("deletes the old TLS cert and makes a new TLS cert using the new CA", func() { @@ -1554,13 +1532,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + // Update the configmap. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: disabled", kubeInformers.Core().V1().ConfigMaps()) r.NoError(runControllerSync()) requireTLSServerIsNoLongerRunning() @@ -1569,10 +1545,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newManuallyDisabledStrategy()) deleteServiceFromTracker(loadBalancerServiceName, kubeInformerClient) - waitForServiceToBeDeleted(kubeInformers.Core().V1().Services(), loadBalancerServiceName) + waitForObjectToBeDeletedFromInformer(loadBalancerServiceName, kubeInformers.Core().V1().Services()) - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") + // Update the configmap again. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: enabled", kubeInformers.Core().V1().ConfigMaps()) r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() @@ -1591,8 +1567,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) requireTLSServerIsRunningWithoutCerts() - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: disabled", kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + // Update the configmap. + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: disabled", kubeInformers.Core().V1().ConfigMaps()) r.EqualError(runControllerSync(), "fake server close error") requireTLSServerIsNoLongerRunning() @@ -1621,14 +1597,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // Switch to "enabled" mode without an "endpoint", so a load balancer is needed now. - updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: enabled", kubeInformers.Core().V1().ConfigMaps()) r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 5) @@ -1638,10 +1611,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Services()) deleteSecretFromTracker(tlsSecretName, kubeInformerClient) - waitForSecretToBeDeleted(kubeInformers.Core().V1().Secrets(), tlsSecretName) + waitForObjectToBeDeletedFromInformer(tlsSecretName, kubeInformers.Core().V1().Secrets()) // The controller should be waiting for the load balancer's ingress to become available. r.NoError(runControllerSync()) @@ -1651,8 +1623,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Update the ingress of the LB in the informer's client and run Sync again. fakeIP := "127.0.0.123" - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}}, kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: fakeIP}}, kubeInformers.Core().V1().Services()) r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 6) requireTLSSecretWasCreated(kubeAPIClient.Actions()[5], ca) // reuses the existing CA @@ -1661,13 +1632,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[5], kubeInformerClient, "3") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "3") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[5], kubeInformers.Core().V1().Secrets()) // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) - updateImpersonatorConfigMapInTracker(configMapResourceName, configMapYAML, kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") + updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, configMapYAML, kubeInformers.Core().V1().ConfigMaps()) r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 9) @@ -1696,10 +1665,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time @@ -1719,13 +1686,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformers.Core().V1().Services()) r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time @@ -1735,8 +1699,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Secrets()) r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started again @@ -1757,13 +1720,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newPendingStrategy()) // Simulate the informer cache's background update from its watch. - addServiceFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) - updateLoadBalancerServiceInTracker(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "2") + updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformers.Core().V1().Services()) r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time @@ -1773,8 +1733,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCredentialIssuer(newSuccessStrategy(hostname, ca)) // Simulate the informer cache's background update from its watch. - addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[3], kubeInformerClient, "2") - waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "2") + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Secrets()) r.NoError(runControllerSync()) r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time From d8c6894cbce4a0f8a7073f6d52965328589d1fe5 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 4 Mar 2021 17:25:43 -0800 Subject: [PATCH 094/203] All controller unit tests should not cancel context until test is over All controller unit tests were accidentally using a timeout context for the informers, instead of a cancel context which stays alive until each test is completely finished. There is no reason to risk unpredictable behavior of a timeout being reached during an individual test, even though with the previous 3 second timeout it could only be reached on a machine which is running orders of magnitude slower than usual, since each test usually runs in about 100-300 ms. Unfortunately, sometimes our CI workers might get that slow. This sparked a review of other usages of timeout contexts in other tests, and all of them were increased to a minimum value of 1 minute, under the rule of thumb that our tests will be more reliable on slow machines if they "pass fast and fail slow". --- cmd/local-user-authenticator/main_test.go | 9 ++-- .../apicerts/apiservice_updater_test.go | 13 +++--- .../controller/apicerts/certs_expirer_test.go | 4 +- .../controller/apicerts/certs_manager_test.go | 14 +++--- .../apicerts/certs_observer_test.go | 15 +++---- .../cachecleaner/cachecleaner_test.go | 3 +- .../jwtcachefiller/jwtcachefiller_test.go | 2 +- .../webhookcachefiller_test.go | 3 +- .../impersonator_config_test.go | 45 ++++++++++++------- .../kubecertagent/annotater_test.go | 14 +++--- .../controller/kubecertagent/creater_test.go | 14 +++--- .../controller/kubecertagent/deleter_test.go | 17 ++++--- .../controller/kubecertagent/execer_test.go | 12 ++--- .../federation_domain_watcher_test.go | 12 ++--- .../federation_domain_secrets_test.go | 3 +- .../generator/supervisor_secrets_test.go | 3 +- .../supervisorconfig/jwks_observer_test.go | 31 +++++++------ .../supervisorconfig/jwks_writer_test.go | 3 +- .../tls_cert_observer_test.go | 31 +++++++------ .../upstreamwatcher/upstreamwatcher_test.go | 2 +- .../garbage_collector_test.go | 30 ++++++------- .../securityheader/securityheader_test.go | 4 +- internal/secret/cache_test.go | 4 +- pkg/oidcclient/login_test.go | 4 +- test/integration/cli_test.go | 2 +- .../concierge_api_serving_certs_test.go | 2 +- .../concierge_availability_test.go | 2 +- test/integration/concierge_client_test.go | 2 +- .../concierge_credentialissuer_test.go | 2 +- .../concierge_credentialrequest_test.go | 10 ++--- .../concierge_impersonation_proxy_test.go | 2 +- .../concierge_kubecertagent_test.go | 2 +- test/integration/supervisor_login_test.go | 4 +- ...ervisor_storage_garbage_collection_test.go | 2 +- test/integration/supervisor_storage_test.go | 4 +- test/integration/whoami_test.go | 14 +++--- test/library/assertions.go | 2 +- test/library/client.go | 20 ++++----- test/library/dumplogs.go | 2 +- 39 files changed, 182 insertions(+), 182 deletions(-) diff --git a/cmd/local-user-authenticator/main_test.go b/cmd/local-user-authenticator/main_test.go index 07771971..b755f1ac 100644 --- a/cmd/local-user-authenticator/main_test.go +++ b/cmd/local-user-authenticator/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package main @@ -99,7 +99,7 @@ func TestWebhook(t *testing.T) { }, })) - secretInformer := createSecretInformer(t, kubeClient) + secretInformer := createSecretInformer(ctx, t, kubeClient) certProvider, caBundle, serverName := newCertProvider(t) w := newWebhook(certProvider, secretInformer) @@ -437,7 +437,7 @@ func TestWebhook(t *testing.T) { } } -func createSecretInformer(t *testing.T, kubeClient kubernetes.Interface) corev1informers.SecretInformer { +func createSecretInformer(ctx context.Context, t *testing.T, kubeClient kubernetes.Interface) corev1informers.SecretInformer { t.Helper() kubeInformers := kubeinformers.NewSharedInformerFactory(kubeClient, 0) @@ -448,9 +448,6 @@ func createSecretInformer(t *testing.T, kubeClient kubernetes.Interface) corev1i // informer factory before syncing it. secretInformer.Informer() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) - defer cancel() - kubeInformers.Start(ctx.Done()) informerTypesSynced := kubeInformers.WaitForCacheSync(ctx.Done()) diff --git a/internal/controller/apicerts/apiservice_updater_test.go b/internal/controller/apicerts/apiservice_updater_test.go index 8454170f..64314215 100644 --- a/internal/controller/apicerts/apiservice_updater_test.go +++ b/internal/controller/apicerts/apiservice_updater_test.go @@ -7,7 +7,6 @@ import ( "context" "errors" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -112,8 +111,8 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { var aggregatorAPIClient *aggregatorfake.Clientset var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context // Defer starting the informers until the last possible moment so that the @@ -131,7 +130,7 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -140,14 +139,14 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -155,7 +154,7 @@ func TestAPIServiceUpdaterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet a serving cert Secret in the installation namespace or it was deleted", func() { diff --git a/internal/controller/apicerts/certs_expirer_test.go b/internal/controller/apicerts/certs_expirer_test.go index 8ba490fa..f8f16595 100644 --- a/internal/controller/apicerts/certs_expirer_test.go +++ b/internal/controller/apicerts/certs_expirer_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -216,7 +216,7 @@ func TestExpirerControllerSync(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() kubeAPIClient := kubernetesfake.NewSimpleClientset() diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index d43babf6..f70f8500 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -125,8 +125,8 @@ func TestManagerControllerSync(t *testing.T) { var kubeAPIClient *kubernetesfake.Clientset var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context // Defer starting the informers until the last possible moment so that the @@ -151,7 +151,7 @@ func TestManagerControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -160,14 +160,14 @@ func TestManagerControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -175,7 +175,7 @@ func TestManagerControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet a serving cert Secret in the installation namespace or it was deleted", func() { diff --git a/internal/controller/apicerts/certs_observer_test.go b/internal/controller/apicerts/certs_observer_test.go index 452c3544..bd1540cc 100644 --- a/internal/controller/apicerts/certs_observer_test.go +++ b/internal/controller/apicerts/certs_observer_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -6,7 +6,6 @@ package apicerts import ( "context" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -104,8 +103,8 @@ func TestObserverControllerSync(t *testing.T) { var subject controllerlib.Controller var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var dynamicCertProvider dynamiccert.Provider @@ -123,7 +122,7 @@ func TestObserverControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -132,14 +131,14 @@ func TestObserverControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -147,7 +146,7 @@ func TestObserverControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet a serving cert Secret in the installation namespace or it was deleted", func() { diff --git a/internal/controller/authenticator/cachecleaner/cachecleaner_test.go b/internal/controller/authenticator/cachecleaner/cachecleaner_test.go index 4bac5be5..5683e26a 100644 --- a/internal/controller/authenticator/cachecleaner/cachecleaner_test.go +++ b/internal/controller/authenticator/cachecleaner/cachecleaner_test.go @@ -6,7 +6,6 @@ package cachecleaner import ( "context" "testing" - "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -150,7 +149,7 @@ func TestController(t *testing.T) { jwtAuthenticators := informers.Authentication().V1alpha1().JWTAuthenticators() controller := New(cache, webhooks, jwtAuthenticators, testLog) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) diff --git a/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go b/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go index 6e53876b..5a43751e 100644 --- a/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go +++ b/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go @@ -325,7 +325,7 @@ func TestController(t *testing.T) { controller := New(cache, informers.Authentication().V1alpha1().JWTAuthenticators(), testLog) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) diff --git a/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go b/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go index b2dcbf84..4d6a0744 100644 --- a/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go +++ b/internal/controller/authenticator/webhookcachefiller/webhookcachefiller_test.go @@ -11,7 +11,6 @@ import ( "net/http" "os" "testing" - "time" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,7 +92,7 @@ func TestController(t *testing.T) { controller := New(cache, informers.Authentication().V1alpha1().WebhookAuthenticators(), testLog) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() informers.Start(ctx.Done()) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 60fe1982..65ea68ff 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -297,8 +297,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var pinnipedAPIClient *pinnipedfake.Clientset var kubeInformerClient *kubernetesfake.Clientset var kubeInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var startTLSListenerFuncWasCalled int var startTLSListenerFuncError error @@ -369,11 +369,12 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { url := "https://" + addr req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) + var resp *http.Response assert.Eventually(t, func() bool { resp, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose return err == nil - }, 5*time.Second, 5*time.Millisecond) + }, 20*time.Second, 50*time.Millisecond) r.NoError(err) r.Equal(http.StatusOK, resp.StatusCode) @@ -392,13 +393,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { url := "https://" + testServerAddr() req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) + expectedErrorRegex := "Get .*: remote error: tls: unrecognized name" expectedErrorRegexCompiled, err := regexp.Compile(expectedErrorRegex) r.NoError(err) assert.Eventually(t, func() bool { _, err = client.Do(req.Clone(context.Background())) //nolint:bodyclose return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) - }, 5*time.Second, 5*time.Millisecond) + }, 20*time.Second, 50*time.Millisecond) r.Error(err) r.Regexp(expectedErrorRegex, err.Error()) } @@ -416,7 +418,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { &tls.Config{InsecureSkipVerify: true}, //nolint:gosec ) return err != nil && expectedErrorRegexCompiled.MatchString(err.Error()) - }, 5*time.Second, 5*time.Millisecond) + }, 20*time.Second, 50*time.Millisecond) r.Error(err) r.Regexp(expectedErrorRegex, err.Error()) } @@ -456,7 +458,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -465,7 +467,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -552,18 +554,29 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // If an object is added to the informer's client *before* the informer is started, then waiting is // not needed because the informer's initial "list" will pick up the object. var waitForObjectToAppearInInformer = func(obj kubeclient.Object, informer controllerlib.InformerGetter) { - r.Eventually(func() bool { - gotObj, exists, err := informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + obj.GetName()) - return err == nil && exists && reflect.DeepEqual(gotObj.(kubeclient.Object), obj) - }, 10*time.Second, 5*time.Millisecond) + var objFromInformer interface{} + var exists bool + var err error + assert.Eventually(t, func() bool { + objFromInformer, exists, err = informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + obj.GetName()) + return err == nil && exists && reflect.DeepEqual(objFromInformer.(kubeclient.Object), obj) + }, 30*time.Second, 10*time.Millisecond) + r.NoError(err) + r.True(exists, "this object should have existed in informer but didn't: %+v", obj) + r.Equal(obj, objFromInformer, "was waiting for expected to be found in informer, but found actual") } // See comment for waitForObjectToAppearInInformer above. var waitForObjectToBeDeletedFromInformer = func(resourceName string, informer controllerlib.InformerGetter) { - r.Eventually(func() bool { - _, exists, err := informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + resourceName) + var objFromInformer interface{} + var exists bool + var err error + assert.Eventually(t, func() bool { + objFromInformer, exists, err = informer.Informer().GetIndexer().GetByKey(installedInNamespace + "/" + resourceName) return err == nil && !exists - }, 10*time.Second, 5*time.Millisecond) + }, 30*time.Second, 10*time.Millisecond) + r.NoError(err) + r.False(exists, "this object should have been deleted from informer but wasn't: %s", objFromInformer) } var addObjectToInformerAndWait = func(obj kubeclient.Object, informer controllerlib.InformerGetter) { @@ -835,7 +848,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactoryWithOptions(kubeInformerClient, 0, kubeinformers.WithNamespace(installedInNamespace), @@ -846,7 +859,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() closeTLSListener() }) diff --git a/internal/controller/kubecertagent/annotater_test.go b/internal/controller/kubecertagent/annotater_test.go index bb767a47..379133fa 100644 --- a/internal/controller/kubecertagent/annotater_test.go +++ b/internal/controller/kubecertagent/annotater_test.go @@ -79,8 +79,8 @@ func TestAnnotaterControllerSync(t *testing.T) { var agentInformerClient *kubernetesfake.Clientset var agentInformers kubeinformers.SharedInformerFactory var pinnipedAPIClient *pinnipedfake.Clientset - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var controllerManagerPod, agentPod *corev1.Pod var podsGVR schema.GroupVersionResource @@ -116,7 +116,7 @@ func TestAnnotaterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: kubeSystemNamespace, @@ -125,8 +125,8 @@ func TestAnnotaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(timeoutContext.Done()) - agentInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(cancelContext.Done()) + agentInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -143,7 +143,7 @@ func TestAnnotaterControllerSync(t *testing.T) { pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( kubeSystemNamespace, agentPodNamespace, certPath, keyPath, @@ -173,7 +173,7 @@ func TestAnnotaterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is an agent pod without annotations set", func() { diff --git a/internal/controller/kubecertagent/creater_test.go b/internal/controller/kubecertagent/creater_test.go index eab57997..13dd9943 100644 --- a/internal/controller/kubecertagent/creater_test.go +++ b/internal/controller/kubecertagent/creater_test.go @@ -94,8 +94,8 @@ func TestCreaterControllerSync(t *testing.T) { var agentInformerClient *kubernetesfake.Clientset var agentInformers kubeinformers.SharedInformerFactory var pinnipedAPIClient *pinnipedfake.Clientset - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var controllerManagerPod, agentPod *corev1.Pod var podsGVR schema.GroupVersionResource @@ -135,7 +135,7 @@ func TestCreaterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: kubeSystemNamespace, @@ -144,8 +144,8 @@ func TestCreaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(timeoutContext.Done()) - agentInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(cancelContext.Done()) + agentInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -162,7 +162,7 @@ func TestCreaterControllerSync(t *testing.T) { pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( kubeSystemNamespace, agentPodNamespace, "ignored for this test", "ignored for this test", @@ -201,7 +201,7 @@ func TestCreaterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is a controller manager pod", func() { diff --git a/internal/controller/kubecertagent/deleter_test.go b/internal/controller/kubecertagent/deleter_test.go index 2a8b5721..ba5240af 100644 --- a/internal/controller/kubecertagent/deleter_test.go +++ b/internal/controller/kubecertagent/deleter_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package kubecertagent @@ -6,7 +6,6 @@ package kubecertagent import ( "context" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -57,8 +56,8 @@ func TestDeleterControllerSync(t *testing.T) { var kubeSystemInformers kubeinformers.SharedInformerFactory var agentInformerClient *kubernetesfake.Clientset var agentInformers kubeinformers.SharedInformerFactory - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var controllerManagerPod, agentPod *corev1.Pod var podsGVR schema.GroupVersionResource @@ -85,7 +84,7 @@ func TestDeleterControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: kubeSystemNamespace, @@ -94,8 +93,8 @@ func TestDeleterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(timeoutContext.Done()) - agentInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(cancelContext.Done()) + agentInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -109,7 +108,7 @@ func TestDeleterControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeAPIClient = kubernetesfake.NewSimpleClientset() @@ -139,7 +138,7 @@ func TestDeleterControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is an agent pod", func() { diff --git a/internal/controller/kubecertagent/execer_test.go b/internal/controller/kubecertagent/execer_test.go index b3789176..7d1092c9 100644 --- a/internal/controller/kubecertagent/execer_test.go +++ b/internal/controller/kubecertagent/execer_test.go @@ -146,8 +146,8 @@ func TestManagerControllerSync(t *testing.T) { var r *require.Assertions var subject controllerlib.Controller - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var pinnipedAPIClient *pinnipedfake.Clientset var kubeInformerFactory kubeinformers.SharedInformerFactory @@ -181,7 +181,7 @@ func TestManagerControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: agentPodNamespace, @@ -190,7 +190,7 @@ func TestManagerControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformerFactory.Start(timeoutContext.Done()) + kubeInformerFactory.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -228,7 +228,7 @@ func TestManagerControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) pinnipedAPIClient = pinnipedfake.NewSimpleClientset() kubeClientset = kubernetesfake.NewSimpleClientset() kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClientset, 0) @@ -253,7 +253,7 @@ func TestManagerControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there is not yet any agent pods or they were deleted", func() { diff --git a/internal/controller/supervisorconfig/federation_domain_watcher_test.go b/internal/controller/supervisorconfig/federation_domain_watcher_test.go index df4f20d9..8dda9704 100644 --- a/internal/controller/supervisorconfig/federation_domain_watcher_test.go +++ b/internal/controller/supervisorconfig/federation_domain_watcher_test.go @@ -103,8 +103,8 @@ func TestSync(t *testing.T) { var federationDomainInformerClient *pinnipedfake.Clientset var federationDomainInformers pinnipedinformers.SharedInformerFactory var pinnipedAPIClient *pinnipedfake.Clientset - var timeoutContext context.Context - var timeoutContextCancel context.CancelFunc + var cancelContext context.Context + var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context var frozenNow time.Time var providersSetter *fakeProvidersSetter @@ -124,7 +124,7 @@ func TestSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: namespace, @@ -133,7 +133,7 @@ func TestSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - federationDomainInformers.Start(timeoutContext.Done()) + federationDomainInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -143,7 +143,7 @@ func TestSync(t *testing.T) { providersSetter = &fakeProvidersSetter{} frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) federationDomainInformerClient = pinnipedfake.NewSimpleClientset() federationDomainInformers = pinnipedinformers.NewSharedInformerFactory(federationDomainInformerClient, 0) @@ -157,7 +157,7 @@ func TestSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are some valid FederationDomains in the informer", func() { diff --git a/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go b/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go index 4fc92a00..a15a2477 100644 --- a/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go +++ b/internal/controller/supervisorconfig/generator/federation_domain_secrets_test.go @@ -10,7 +10,6 @@ import ( "fmt" "sync" "testing" - "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -630,7 +629,7 @@ func TestFederationDomainSecretsControllerSync(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() pinnipedAPIClient := pinnipedfake.NewSimpleClientset() diff --git a/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go b/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go index eeeb9930..044d1cb0 100644 --- a/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go +++ b/internal/controller/supervisorconfig/generator/supervisor_secrets_test.go @@ -8,7 +8,6 @@ import ( "errors" "sync" "testing" - "time" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" @@ -412,7 +411,7 @@ func TestSupervisorSecretsControllerSync(t *testing.T) { t.Run(test.name, func(t *testing.T) { // We cannot currently run this test in parallel since it uses the global generateKey function. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() if test.generateKey != nil { diff --git a/internal/controller/supervisorconfig/jwks_observer_test.go b/internal/controller/supervisorconfig/jwks_observer_test.go index a1e7e3e3..07d6f62b 100644 --- a/internal/controller/supervisorconfig/jwks_observer_test.go +++ b/internal/controller/supervisorconfig/jwks_observer_test.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -124,16 +123,16 @@ func TestJWKSObserverControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" var ( - r *require.Assertions - subject controllerlib.Controller - pinnipedInformerClient *pinnipedfake.Clientset - kubeInformerClient *kubernetesfake.Clientset - pinnipedInformers pinnipedinformers.SharedInformerFactory - kubeInformers kubeinformers.SharedInformerFactory - timeoutContext context.Context - timeoutContextCancel context.CancelFunc - syncContext *controllerlib.Context - issuerToJWKSSetter *fakeIssuerToJWKSMapSetter + r *require.Assertions + subject controllerlib.Controller + pinnipedInformerClient *pinnipedfake.Clientset + kubeInformerClient *kubernetesfake.Clientset + pinnipedInformers pinnipedinformers.SharedInformerFactory + kubeInformers kubeinformers.SharedInformerFactory + cancelContext context.Context + cancelContextCancelFunc context.CancelFunc + syncContext *controllerlib.Context + issuerToJWKSSetter *fakeIssuerToJWKSMapSetter ) // Defer starting the informers until the last possible moment so that the @@ -149,7 +148,7 @@ func TestJWKSObserverControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -158,15 +157,15 @@ func TestJWKSObserverControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) - pinnipedInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) + pinnipedInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -184,7 +183,7 @@ func TestJWKSObserverControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are no FederationDomains and no JWKS Secrets yet", func() { diff --git a/internal/controller/supervisorconfig/jwks_writer_test.go b/internal/controller/supervisorconfig/jwks_writer_test.go index c528be90..f977c8f8 100644 --- a/internal/controller/supervisorconfig/jwks_writer_test.go +++ b/internal/controller/supervisorconfig/jwks_writer_test.go @@ -12,7 +12,6 @@ import ( "io" "io/ioutil" "testing" - "time" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -674,7 +673,7 @@ func TestJWKSWriterControllerSync(t *testing.T) { return goodKey, test.generateKeyErr } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() kubeAPIClient := kubernetesfake.NewSimpleClientset() diff --git a/internal/controller/supervisorconfig/tls_cert_observer_test.go b/internal/controller/supervisorconfig/tls_cert_observer_test.go index f24888a2..3e391942 100644 --- a/internal/controller/supervisorconfig/tls_cert_observer_test.go +++ b/internal/controller/supervisorconfig/tls_cert_observer_test.go @@ -9,7 +9,6 @@ import ( "io/ioutil" "net/url" "testing" - "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -130,16 +129,16 @@ func TestTLSCertObserverControllerSync(t *testing.T) { ) var ( - r *require.Assertions - subject controllerlib.Controller - pinnipedInformerClient *pinnipedfake.Clientset - kubeInformerClient *kubernetesfake.Clientset - pinnipedInformers pinnipedinformers.SharedInformerFactory - kubeInformers kubeinformers.SharedInformerFactory - timeoutContext context.Context - timeoutContextCancel context.CancelFunc - syncContext *controllerlib.Context - issuerTLSCertSetter *fakeIssuerTLSCertSetter + r *require.Assertions + subject controllerlib.Controller + pinnipedInformerClient *pinnipedfake.Clientset + kubeInformerClient *kubernetesfake.Clientset + pinnipedInformers pinnipedinformers.SharedInformerFactory + kubeInformers kubeinformers.SharedInformerFactory + cancelContext context.Context + cancelContextCancelFunc context.CancelFunc + syncContext *controllerlib.Context + issuerTLSCertSetter *fakeIssuerTLSCertSetter ) // Defer starting the informers until the last possible moment so that the @@ -156,7 +155,7 @@ func TestTLSCertObserverControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: installedInNamespace, @@ -165,8 +164,8 @@ func TestTLSCertObserverControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) - pinnipedInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) + pinnipedInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -179,7 +178,7 @@ func TestTLSCertObserverControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) @@ -197,7 +196,7 @@ func TestTLSCertObserverControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are no FederationDomains and no TLS Secrets yet", func() { diff --git a/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go b/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go index cba27c39..f7397352 100644 --- a/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go +++ b/internal/controller/supervisorconfig/upstreamwatcher/upstreamwatcher_test.go @@ -624,7 +624,7 @@ func TestController(t *testing.T) { controllerlib.WithInformer, ) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() pinnipedInformers.Start(ctx.Done()) diff --git a/internal/controller/supervisorstorage/garbage_collector_test.go b/internal/controller/supervisorstorage/garbage_collector_test.go index a21067ee..a5606cf8 100644 --- a/internal/controller/supervisorstorage/garbage_collector_test.go +++ b/internal/controller/supervisorstorage/garbage_collector_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package supervisorstorage @@ -108,16 +108,16 @@ func TestGarbageCollectorControllerSync(t *testing.T) { ) var ( - r *require.Assertions - subject controllerlib.Controller - kubeInformerClient *kubernetesfake.Clientset - kubeClient *kubernetesfake.Clientset - kubeInformers kubeinformers.SharedInformerFactory - timeoutContext context.Context - timeoutContextCancel context.CancelFunc - syncContext *controllerlib.Context - fakeClock *clock.FakeClock - frozenNow time.Time + r *require.Assertions + subject controllerlib.Controller + kubeInformerClient *kubernetesfake.Clientset + kubeClient *kubernetesfake.Clientset + kubeInformers kubeinformers.SharedInformerFactory + cancelContext context.Context + cancelContextCancelFunc context.CancelFunc + syncContext *controllerlib.Context + fakeClock *clock.FakeClock + frozenNow time.Time ) // Defer starting the informers until the last possible moment so that the @@ -133,7 +133,7 @@ func TestGarbageCollectorControllerSync(t *testing.T) { // Set this at the last second to support calling subject.Name(). syncContext = &controllerlib.Context{ - Context: timeoutContext, + Context: cancelContext, Name: subject.Name(), Key: controllerlib.Key{ Namespace: "", @@ -142,14 +142,14 @@ func TestGarbageCollectorControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeInformers.Start(cancelContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeClient = kubernetesfake.NewSimpleClientset() @@ -168,7 +168,7 @@ func TestGarbageCollectorControllerSync(t *testing.T) { }) it.After(func() { - timeoutContextCancel() + cancelContextCancelFunc() }) when("there are secrets without the garbage-collect-after annotation", func() { diff --git a/internal/httputil/securityheader/securityheader_test.go b/internal/httputil/securityheader/securityheader_test.go index a0688c1a..7ee7331f 100644 --- a/internal/httputil/securityheader/securityheader_test.go +++ b/internal/httputil/securityheader/securityheader_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package securityheader @@ -22,7 +22,7 @@ func TestWrap(t *testing.T) { }))) t.Cleanup(testServer.Close) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() req, err := http.NewRequestWithContext(ctx, http.MethodGet, testServer.URL, nil) diff --git a/internal/secret/cache_test.go b/internal/secret/cache_test.go index 0934bd58..0a6b500c 100644 --- a/internal/secret/cache_test.go +++ b/internal/secret/cache_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package secret @@ -68,7 +68,7 @@ func TestCacheSynchronized(t *testing.T) { c.SetStateEncoderHashKey(issuer, stateEncoderHashKey) c.SetStateEncoderBlockKey(issuer, stateEncoderBlockKey) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() eg, _ := errgroup.WithContext(ctx) diff --git a/pkg/oidcclient/login_test.go b/pkg/oidcclient/login_test.go index eaa7a1b1..8005bcaf 100644 --- a/pkg/oidcclient/login_test.go +++ b/pkg/oidcclient/login_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package oidcclient @@ -941,7 +941,7 @@ func TestHandleAuthCodeCallback(t *testing.T) { require.NoError(t, tt.opt(t)(h)) } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() resp := httptest.NewRecorder() diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index 910d1ac4..e055fccf 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -38,7 +38,7 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") // Create a test webhook configuration to use with the CLI. - ctx, cancelFunc := context.WithTimeout(context.Background(), 4*time.Minute) + ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute) defer cancelFunc() authenticator := library.CreateTestWebhookAuthenticator(ctx, t) diff --git a/test/integration/concierge_api_serving_certs_test.go b/test/integration/concierge_api_serving_certs_test.go index 78aefea6..04ee11af 100644 --- a/test/integration/concierge_api_serving_certs_test.go +++ b/test/integration/concierge_api_serving_certs_test.go @@ -78,7 +78,7 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) { kubeClient := library.NewKubernetesClientset(t) aggregatedClient := library.NewAggregatedClientset(t) conciergeClient := library.NewConciergeClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() apiServiceName := "v1alpha1.login.concierge." + env.APIGroupSuffix diff --git a/test/integration/concierge_availability_test.go b/test/integration/concierge_availability_test.go index 575e4fc2..6dc260bf 100644 --- a/test/integration/concierge_availability_test.go +++ b/test/integration/concierge_availability_test.go @@ -20,7 +20,7 @@ func TestGetDeployment(t *testing.T) { env := library.IntegrationEnv(t) client := library.NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() appDeployment, err := client.AppsV1().Deployments(env.ConciergeNamespace).Get(ctx, env.ConciergeAppName, metav1.GetOptions{}) diff --git a/test/integration/concierge_client_test.go b/test/integration/concierge_client_test.go index 1cad0de5..1dc4b6b6 100644 --- a/test/integration/concierge_client_test.go +++ b/test/integration/concierge_client_test.go @@ -59,7 +59,7 @@ func TestClient(t *testing.T) { library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() webhook := library.CreateTestWebhookAuthenticator(ctx, t) diff --git a/test/integration/concierge_credentialissuer_test.go b/test/integration/concierge_credentialissuer_test.go index 4a4881cb..abb4f8be 100644 --- a/test/integration/concierge_credentialissuer_test.go +++ b/test/integration/concierge_credentialissuer_test.go @@ -25,7 +25,7 @@ func TestCredentialIssuer(t *testing.T) { library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() t.Run("test successful CredentialIssuer", func(t *testing.T) { diff --git a/test/integration/concierge_credentialrequest_test.go b/test/integration/concierge_credentialrequest_test.go index 7a37c9c7..de48bf85 100644 --- a/test/integration/concierge_credentialrequest_test.go +++ b/test/integration/concierge_credentialrequest_test.go @@ -27,7 +27,7 @@ func TestUnsuccessfulCredentialRequest(t *testing.T) { library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() response, err := makeRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, corev1.TypedLocalObjectReference{ @@ -137,7 +137,7 @@ func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthentic // Create a testWebhook so we have a legitimate authenticator to pass to the // TokenCredentialRequest API. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) @@ -160,7 +160,7 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T // Create a testWebhook so we have a legitimate authenticator to pass to the // TokenCredentialRequest API. - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) @@ -188,7 +188,7 @@ func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheCl library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) @@ -208,7 +208,7 @@ func makeRequest(ctx context.Context, t *testing.T, spec loginv1alpha1.TokenCred client := library.NewAnonymousConciergeClientset(t) - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, &loginv1alpha1.TokenCredentialRequest{ diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index b52c662f..369930f5 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -96,7 +96,7 @@ func TestImpersonationProxy(t *testing.T) { } // At the end of the test, clean up the ConfigMap. t.Cleanup(func() { - ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel = context.WithTimeout(context.Background(), time.Minute) defer cancel() // Delete any version that was created by this test. diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 37392013..73959bf8 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -30,7 +30,7 @@ func TestKubeCertAgent(t *testing.T) { library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t) diff --git a/test/integration/supervisor_login_test.go b/test/integration/supervisor_login_test.go index 84fa8efa..9d6b6787 100644 --- a/test/integration/supervisor_login_test.go +++ b/test/integration/supervisor_login_test.go @@ -240,7 +240,7 @@ func verifyTokenResponse( nonceParam nonce.Nonce, expectedIDTokenClaims []string, ) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Verify the ID Token. @@ -310,7 +310,7 @@ func (s *localCallbackServer) waitForCallback(timeout time.Duration) *http.Reque } func doTokenExchange(t *testing.T, config *oauth2.Config, tokenResponse *oauth2.Token, httpClient *http.Client, provider *coreosoidc.Provider) { - ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Form the HTTP POST request with the parameters specified by RFC8693. diff --git a/test/integration/supervisor_storage_garbage_collection_test.go b/test/integration/supervisor_storage_garbage_collection_test.go index c89a66e4..84b82196 100644 --- a/test/integration/supervisor_storage_garbage_collection_test.go +++ b/test/integration/supervisor_storage_garbage_collection_test.go @@ -99,7 +99,7 @@ func createSecret(ctx context.Context, t *testing.T, secrets corev1client.Secret // Make sure the Secret is deleted when the test ends. t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := secrets.Delete(ctx, secret.Name, metav1.DeleteOptions{}) notFound := k8serrors.IsNotFound(err) diff --git a/test/integration/supervisor_storage_test.go b/test/integration/supervisor_storage_test.go index 51d0f70e..5edc4e70 100644 --- a/test/integration/supervisor_storage_test.go +++ b/test/integration/supervisor_storage_test.go @@ -42,13 +42,13 @@ func TestAuthorizeCodeStorage(t *testing.T) { secrets := client.CoreV1().Secrets(env.SupervisorNamespace) t.Cleanup(func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := secrets.Delete(ctx, name, metav1.DeleteOptions{}) require.NoError(t, err) }) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // get a session with most of the data filled out diff --git a/test/integration/whoami_test.go b/test/integration/whoami_test.go index de478f52..345347d8 100644 --- a/test/integration/whoami_test.go +++ b/test/integration/whoami_test.go @@ -34,7 +34,7 @@ func TestWhoAmI_Kubeadm(t *testing.T) { // we should add more robust logic around skipping clusters based on vendor _ = library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() whoAmI, err := library.NewConciergeClientset(t).IdentityV1alpha1().WhoAmIRequests(). @@ -63,7 +63,7 @@ func TestWhoAmI_Kubeadm(t *testing.T) { func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t).CoreV1() @@ -107,7 +107,7 @@ func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { return false, err } return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil - }, 30*time.Second, time.Second) + }, time.Minute, time.Second) saConfig := library.NewAnonymousClientRestConfig(t) saConfig.BearerToken = string(secret.Data[corev1.ServiceAccountTokenKey]) @@ -140,7 +140,7 @@ func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { func TestWhoAmI_ServiceAccount_TokenRequest(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t).CoreV1() @@ -258,7 +258,7 @@ func TestWhoAmI_CSR(t *testing.T) { // we should add more robust logic around skipping clusters based on vendor _ = library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() kubeClient := library.NewKubernetesClientset(t) @@ -346,7 +346,7 @@ func TestWhoAmI_CSR(t *testing.T) { func TestWhoAmI_Anonymous(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() anonymousConfig := library.NewAnonymousClientRestConfig(t) @@ -377,7 +377,7 @@ func TestWhoAmI_Anonymous(t *testing.T) { func TestWhoAmI_ImpersonateDirectly(t *testing.T) { _ = library.IntegrationEnv(t) - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() impersonationConfig := library.NewClientConfig(t) diff --git a/test/library/assertions.go b/test/library/assertions.go index 4db0ee09..90bb4661 100644 --- a/test/library/assertions.go +++ b/test/library/assertions.go @@ -88,7 +88,7 @@ func getRestartCounts(t *testing.T, namespace, labelSelector string) map[string] t.Helper() kubeClient := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) diff --git a/test/library/client.go b/test/library/client.go index bfb92ee8..1d5fd9e8 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -156,7 +156,7 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty client := NewConciergeClientset(t) webhooks := client.AuthenticationV1alpha1().WebhookAuthenticators() - createContext, cancel := context.WithTimeout(ctx, 5*time.Second) + createContext, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() webhook, err := webhooks.Create(createContext, &auth1alpha1.WebhookAuthenticator{ @@ -169,7 +169,7 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty t.Cleanup(func() { t.Helper() t.Logf("cleaning up test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) - deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := webhooks.Delete(deleteCtx, webhook.Name, metav1.DeleteOptions{}) require.NoErrorf(t, err, "could not cleanup test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) @@ -219,7 +219,7 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp client := NewConciergeClientset(t) jwtAuthenticators := client.AuthenticationV1alpha1().JWTAuthenticators() - createContext, cancel := context.WithTimeout(ctx, 5*time.Second) + createContext, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() jwtAuthenticator, err := jwtAuthenticators.Create(createContext, &auth1alpha1.JWTAuthenticator{ @@ -232,7 +232,7 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp t.Cleanup(func() { t.Helper() t.Logf("cleaning up test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) - deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := jwtAuthenticators.Delete(deleteCtx, jwtAuthenticator.Name, metav1.DeleteOptions{}) require.NoErrorf(t, err, "could not cleanup test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) @@ -255,7 +255,7 @@ func CreateTestFederationDomain(ctx context.Context, t *testing.T, issuer string t.Helper() testEnv := IntegrationEnv(t) - createContext, cancel := context.WithTimeout(ctx, 5*time.Second) + createContext, cancel := context.WithTimeout(ctx, time.Minute) defer cancel() if issuer == "" { @@ -276,7 +276,7 @@ func CreateTestFederationDomain(ctx context.Context, t *testing.T, issuer string t.Cleanup(func() { t.Helper() t.Logf("cleaning up test FederationDomain %s/%s", federationDomain.Namespace, federationDomain.Name) - deleteCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := federationDomains.Delete(deleteCtx, federationDomain.Name, metav1.DeleteOptions{}) notFound := k8serrors.IsNotFound(err) @@ -330,7 +330,7 @@ func RandHex(t *testing.T, numBytes int) string { func CreateTestSecret(t *testing.T, namespace string, baseName string, secretType corev1.SecretType, stringData map[string]string) *corev1.Secret { t.Helper() client := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() created, err := client.CoreV1().Secrets(namespace).Create(ctx, &corev1.Secret{ @@ -401,7 +401,7 @@ func CreateTestOIDCIdentityProvider(t *testing.T, spec idpv1alpha1.OIDCIdentityP func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef rbacv1.RoleRef) *rbacv1.ClusterRoleBinding { t.Helper() client := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() // Create the ClusterRoleBinding using GenerateName to get a random name. @@ -426,7 +426,7 @@ func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldHaveAccessTo *authorizationv1.ResourceAttributes) { t.Helper() client := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() RequireEventuallyWithoutError(t, func() (bool, error) { @@ -441,7 +441,7 @@ func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldH return false, err } return subjectAccessReview.Status.Allowed, nil - }, 10*time.Second, 500*time.Millisecond) + }, time.Minute, 500*time.Millisecond) } func testObjectMeta(t *testing.T, baseName string) metav1.ObjectMeta { diff --git a/test/library/dumplogs.go b/test/library/dumplogs.go index dca0d276..f2eaaa4a 100644 --- a/test/library/dumplogs.go +++ b/test/library/dumplogs.go @@ -22,7 +22,7 @@ func DumpLogs(t *testing.T, namespace string, labelSelector string) { } kubeClient := NewKubernetesClientset(t) - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() logTailLines := int64(40) From ec133b9743b6a9bfa8ea03dba99e4766da3f8db4 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 4 Mar 2021 17:44:01 -0800 Subject: [PATCH 095/203] Resolve some new linter errors --- test/integration/e2e_test.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index fb7b8b90..3e950a25 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -279,7 +279,10 @@ func TestE2EFullIntegration(t *testing.T) { // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. httpClient := &http.Client{ Transport: &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: kubeconfigCA}, + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: kubeconfigCA, + }, Proxy: func(req *http.Request) (*url.URL, error) { if env.Proxy == "" { t.Logf("passing request for %s with no proxy", req.URL) @@ -292,7 +295,9 @@ func TestE2EFullIntegration(t *testing.T) { }, }, } - resp, err := httpClient.Get(restConfig.Host) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, restConfig.Host, nil) + require.NoError(t, err) + resp, err := httpClient.Do(req) //nolint:bodyclose if err != nil { t.Logf("could not connect to the API server at %q: %v", restConfig.Host, err) return false From c3b7d210377adac4f8e93db5a4ce8cd72d66e6f2 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 08:39:48 -0600 Subject: [PATCH 096/203] Be less picky about what error code is returned here. The thing we're waiting for is mostly that DNS is resolving, the ELB is listening, and connections are making it to the proxy. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 3e950a25..f13fc823 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -303,8 +303,8 @@ func TestE2EFullIntegration(t *testing.T) { return false } t.Logf("got %d response from API server at %q", resp.StatusCode, restConfig.Host) - return resp.StatusCode == 200 || resp.StatusCode == 403 - }, 5*time.Minute, time.Second) + return resp.StatusCode < 500 + }, 5*time.Minute, 2*time.Second) // Expect the CLI to output a list of namespaces in JSON format. t.Logf("waiting for kubectl to output namespace list JSON") From d84849917633ea1d3dd451d3be9459a4d9a547e7 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 08:45:25 -0600 Subject: [PATCH 097/203] Close this HTTP response body in TestE2EFullIntegration. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index f13fc823..57056850 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -297,12 +297,13 @@ func TestE2EFullIntegration(t *testing.T) { } req, err := http.NewRequestWithContext(ctx, http.MethodGet, restConfig.Host, nil) require.NoError(t, err) - resp, err := httpClient.Do(req) //nolint:bodyclose + resp, err := httpClient.Do(req) if err != nil { t.Logf("could not connect to the API server at %q: %v", restConfig.Host, err) return false } t.Logf("got %d response from API server at %q", resp.StatusCode, restConfig.Host) + require.NoError(t, resp.Body.Close()) return resp.StatusCode < 500 }, 5*time.Minute, 2*time.Second) From 52f58477b855a6bebbac738182c3fc634a37c480 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 09:32:49 -0600 Subject: [PATCH 098/203] Wait for the ELB to become available _before_ starting the kubectl command. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 72 ++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 57056850..5973a8a9 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -173,6 +173,42 @@ func TestE2EFullIntegration(t *testing.T) { t.Log("sleeping 10s to wait for JWTAuthenticator to become initialized") time.Sleep(10 * time.Second) + // Verify that we can actually reach the endpoint in the kubeconfig. + require.Eventually(t, func() bool { + kubeconfigCA := x509.NewCertPool() + require.True(t, kubeconfigCA.AppendCertsFromPEM(restConfig.TLSClientConfig.CAData), "expected to load kubeconfig CA") + + // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: kubeconfigCA, + }, + Proxy: func(req *http.Request) (*url.URL, error) { + if env.Proxy == "" { + t.Logf("passing request for %s with no proxy", req.URL) + return nil, nil + } + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + }, + }, + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, restConfig.Host, nil) + require.NoError(t, err) + resp, err := httpClient.Do(req) + if err != nil { + t.Logf("could not connect to the API server at %q: %v", restConfig.Host, err) + return false + } + t.Logf("got %d response from API server at %q", resp.StatusCode, restConfig.Host) + require.NoError(t, resp.Body.Close()) + return resp.StatusCode < 500 + }, 5*time.Minute, 2*time.Second) + // Run "kubectl get namespaces" which should trigger a browser login via the plugin. start := time.Now() kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) @@ -271,42 +307,6 @@ func TestE2EFullIntegration(t *testing.T) { require.NoError(t, err) require.Equal(t, "you have been logged in and may now close this tab", msg) - // Verify that we can actually reach the endpoint in the kubeconfig. - require.Eventually(t, func() bool { - kubeconfigCA := x509.NewCertPool() - require.True(t, kubeconfigCA.AppendCertsFromPEM(restConfig.TLSClientConfig.CAData), "expected to load kubeconfig CA") - - // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: kubeconfigCA, - }, - Proxy: func(req *http.Request) (*url.URL, error) { - if env.Proxy == "" { - t.Logf("passing request for %s with no proxy", req.URL) - return nil, nil - } - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil - }, - }, - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, restConfig.Host, nil) - require.NoError(t, err) - resp, err := httpClient.Do(req) - if err != nil { - t.Logf("could not connect to the API server at %q: %v", restConfig.Host, err) - return false - } - t.Logf("got %d response from API server at %q", resp.StatusCode, restConfig.Host) - require.NoError(t, resp.Body.Close()) - return resp.StatusCode < 500 - }, 5*time.Minute, 2*time.Second) - // Expect the CLI to output a list of namespaces in JSON format. t.Logf("waiting for kubectl to output namespace list JSON") var kubectlOutput string From c4f6fd5b3cbcdd69b5ae23c6759dfdbefa8e832a Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 15:49:45 -0600 Subject: [PATCH 099/203] Add a bit nicer assertion helper in testutil/testlogger. This makes output that's easier to copy-paste into the test. We could also make it ignore the order of key/value pairs in the future. Signed-off-by: Matt Moyer --- internal/testutil/testlogger/testlogger.go | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/internal/testutil/testlogger/testlogger.go b/internal/testutil/testlogger/testlogger.go index f20cbc7e..51b6ead6 100644 --- a/internal/testutil/testlogger/testlogger.go +++ b/internal/testutil/testlogger/testlogger.go @@ -6,6 +6,7 @@ package testlogger import ( "bytes" + "fmt" "log" "strings" "sync" @@ -13,6 +14,7 @@ import ( "github.com/go-logr/logr" "github.com/go-logr/stdr" + "github.com/stretchr/testify/require" ) // Logger implements logr.Logger in a way that captures logs for test assertions. @@ -46,6 +48,12 @@ func (l *Logger) Lines() []string { return result } +// Expect the emitted lines to match known-good output. +func (l *Logger) Expect(expected []string) { + actual := l.Lines() + require.Equalf(l.t, expected, actual, "did not see expected log output, actual output:\n%#v", backtickStrings(actual)) +} + // syncBuffer synchronizes access to a bytes.Buffer. type syncBuffer struct { mutex sync.Mutex @@ -57,3 +65,18 @@ func (s *syncBuffer) Write(p []byte) (n int, err error) { defer s.mutex.Unlock() return s.buffer.Write(p) } + +type backtickStrings []string + +func (b backtickStrings) GoString() string { + lines := make([]string, 0, len(b)) + for _, s := range b { + if strings.Contains(s, "`") { + s = fmt.Sprintf("%#v", s) + } else { + s = fmt.Sprintf("`%s`", s) + } + lines = append(lines, fmt.Sprintf("\t%s,", s)) + } + return fmt.Sprintf("[]string{\n%s\n}", strings.Join(lines, "\n")) +} From 36bc6791428370bf2edcea0cf8a161333e6dda35 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 15:52:17 -0600 Subject: [PATCH 100/203] Add diagnostic logging to "pinniped get kubeconfig". These stderr logs should help clarify all the autodetection logic that's happening in a particular run. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 41 ++++++++--- cmd/pinniped/cmd/kubeconfig_test.go | 103 +++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 10 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 6a4e8327..3314e7a0 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -10,20 +10,22 @@ import ( "fmt" "io" "io/ioutil" + "log" "os" "strconv" "strings" "time" "github.com/coreos/go-oidc/v3/oidc" + "github.com/go-logr/logr" + "github.com/go-logr/stdr" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" + _ "k8s.io/client-go/plugin/pkg/client/auth" // Adds handlers for various dynamic auth plugins in client-go "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" - _ "k8s.io/client-go/plugin/pkg/client/auth" // Adds handlers for various dynamic auth plugins in client-go - conciergev1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" @@ -34,6 +36,7 @@ import ( type kubeconfigDeps struct { getPathToSelf func() (string, error) getClientset func(clientConfig clientcmd.ClientConfig, apiGroupSuffix string) (conciergeclientset.Interface, error) + log logr.Logger } func kubeconfigRealDeps() kubeconfigDeps { @@ -53,6 +56,7 @@ func kubeconfigRealDeps() kubeconfigDeps { } return client.PinnipedConcierge, nil }, + log: stdr.New(log.New(os.Stderr, "", 0)), } } @@ -181,7 +185,7 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar } if !flags.concierge.disabled { - credentialIssuer, err := lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer) + credentialIssuer, err := lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) if err != nil { return err } @@ -190,12 +194,13 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar clientset, flags.concierge.authenticatorType, flags.concierge.authenticatorName, + deps.log, ) if err != nil { return err } - if err := configureConcierge(credentialIssuer, authenticator, &flags, cluster, &oidcCABundle, &execConfig); err != nil { + if err := configureConcierge(credentialIssuer, authenticator, &flags, cluster, &oidcCABundle, &execConfig, deps.log); err != nil { return err } } @@ -246,7 +251,7 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig)) } -func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig) error { +func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig, log logr.Logger) error { var conciergeCABundleData []byte // Autodiscover the --concierge-mode. @@ -258,9 +263,11 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe } switch strategy.Frontend.Type { case configv1alpha1.TokenCredentialRequestAPIFrontendType: + log.Info("detected Concierge in TokenCredentialRequest API mode") flags.concierge.mode = modeTokenCredentialRequestAPI break strategyLoop case configv1alpha1.ImpersonationProxyFrontendType: + flags.concierge.mode = modeImpersonationProxy flags.concierge.endpoint = strategy.Frontend.ImpersonationProxyInfo.Endpoint var err error @@ -268,6 +275,7 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe if err != nil { return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) } + log.Info("detected Concierge in impersonation proxy mode", "endpoint", strategy.Frontend.ImpersonationProxyInfo.Endpoint) break strategyLoop default: // Skip any unknown frontend types. @@ -288,6 +296,7 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // If the --concierge-authenticator-type/--concierge-authenticator-name flags were not set explicitly, set // them to point at the discovered WebhookAuthenticator. if flags.concierge.authenticatorType == "" && flags.concierge.authenticatorName == "" { + log.Info("discovered WebhookAuthenticator", "name", auth.Name) flags.concierge.authenticatorType = "webhook" flags.concierge.authenticatorName = auth.Name } @@ -295,17 +304,20 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // If the --concierge-authenticator-type/--concierge-authenticator-name flags were not set explicitly, set // them to point at the discovered JWTAuthenticator. if flags.concierge.authenticatorType == "" && flags.concierge.authenticatorName == "" { + log.Info("discovered JWTAuthenticator", "name", auth.Name) flags.concierge.authenticatorType = "jwt" flags.concierge.authenticatorName = auth.Name } // If the --oidc-issuer flag was not set explicitly, default it to the spec.issuer field of the JWTAuthenticator. if flags.oidc.issuer == "" { + log.Info("detected OIDC issuer", "issuer", auth.Spec.Issuer) flags.oidc.issuer = auth.Spec.Issuer } // If the --oidc-request-audience flag was not set explicitly, default it to the spec.audience field of the JWTAuthenticator. if flags.oidc.requestAudience == "" { + log.Info("detected OIDC audience", "audience", auth.Spec.Audience) flags.oidc.requestAudience = auth.Spec.Audience } @@ -316,16 +328,19 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe if err != nil { return fmt.Errorf("tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator %s has invalid spec.tls.certificateAuthorityData: %w", auth.Name, err) } + log.Info("detected OIDC CA bundle", "length", len(decoded)) *oidcCABundle = string(decoded) } } if flags.concierge.endpoint == "" { + log.Info("detected concierge endpoint", "endpoint", v1Cluster.Server) flags.concierge.endpoint = v1Cluster.Server } if conciergeCABundleData == nil { if flags.concierge.caBundlePath == "" { + log.Info("detected concierge CA bundle", "length", len(v1Cluster.CertificateAuthorityData)) conciergeCABundleData = v1Cluster.CertificateAuthorityData } else { caBundleString, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) @@ -349,6 +364,7 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // If we're in impersonation proxy mode, the main server endpoint for the kubeconfig also needs to point to the proxy if flags.concierge.mode == modeImpersonationProxy { + log.Info("switching kubeconfig cluster to point at impersonation proxy endpoint", "endpoint", flags.concierge.endpoint) v1Cluster.CertificateAuthorityData = conciergeCABundleData v1Cluster.Server = flags.concierge.endpoint } @@ -383,7 +399,7 @@ func newExecKubeconfig(cluster *clientcmdapi.Cluster, execConfig *clientcmdapi.E } } -func lookupCredentialIssuer(clientset conciergeclientset.Interface, name string) (*configv1alpha1.CredentialIssuer, error) { +func lookupCredentialIssuer(clientset conciergeclientset.Interface, name string, log logr.Logger) (*configv1alpha1.CredentialIssuer, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) defer cancelFunc() @@ -403,10 +419,13 @@ func lookupCredentialIssuer(clientset conciergeclientset.Interface, name string) if len(results.Items) > 1 { return nil, fmt.Errorf("multiple CredentialIssuers were found, so the --concierge-credential-issuer flag must be specified") } - return &results.Items[0], nil + + result := &results.Items[0] + log.Info("discovered CredentialIssuer", "name", result.Name) + return result, nil } -func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authName string) (metav1.Object, error) { +func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authName string, log logr.Logger) (metav1.Object, error) { ctx, cancelFunc := context.WithTimeout(context.Background(), time.Second*20) defer cancelFunc() @@ -444,6 +463,12 @@ func lookupAuthenticator(clientset conciergeclientset.Interface, authType, authN return nil, fmt.Errorf("no authenticators were found") } if len(results) > 1 { + for _, jwtAuth := range jwtAuths.Items { + log.Info("found JWTAuthenticator", "name", jwtAuth.Name) + } + for _, webhook := range webhooks.Items { + log.Info("found WebhookAuthenticator", "name", webhook.Name) + } return nil, fmt.Errorf("multiple authenticators were found, so the --concierge-authenticator-type/--concierge-authenticator-name flags must be specified") } return results[0], nil diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 79fdb2dd..7d84d31c 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -26,6 +26,7 @@ import ( "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/internal/testutil/testlogger" ) func TestGetKubeconfig(t *testing.T) { @@ -46,6 +47,7 @@ func TestGetKubeconfig(t *testing.T) { getClientsetErr error conciergeObjects []runtime.Object conciergeReactions []kubetesting.Reactor + wantLogs []string wantError bool wantStdout string wantStderr string @@ -171,6 +173,9 @@ func TestGetKubeconfig(t *testing.T) { conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: webhookauthenticators.authentication.concierge.pinniped.dev "test-authenticator" not found @@ -186,6 +191,9 @@ func TestGetKubeconfig(t *testing.T) { conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: jwtauthenticators.authentication.concierge.pinniped.dev "test-authenticator" not found @@ -201,6 +209,9 @@ func TestGetKubeconfig(t *testing.T) { conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: invalid authenticator type "invalid", supported values are "webhook" and "jwt" @@ -214,6 +225,9 @@ func TestGetKubeconfig(t *testing.T) { conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, conciergeReactions: []kubetesting.Reactor{ &kubetesting.SimpleReactor{ Verb: "*", @@ -245,6 +259,9 @@ func TestGetKubeconfig(t *testing.T) { }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: failed to list WebhookAuthenticator objects for autodiscovery: some list error @@ -258,6 +275,9 @@ func TestGetKubeconfig(t *testing.T) { conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: no authenticators were found @@ -275,6 +295,13 @@ func TestGetKubeconfig(t *testing.T) { &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-3"}}, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator-4"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="found JWTAuthenticator" "name"="test-authenticator-1"`, + `"level"=0 "msg"="found JWTAuthenticator" "name"="test-authenticator-2"`, + `"level"=0 "msg"="found WebhookAuthenticator" "name"="test-authenticator-3"`, + `"level"=0 "msg"="found WebhookAuthenticator" "name"="test-authenticator-4"`, + }, wantError: true, wantStderr: here.Doc(` Error: multiple authenticators were found, so the --concierge-authenticator-type/--concierge-authenticator-name flags must be specified @@ -289,6 +316,9 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: could not autodiscover --concierge-mode and none was provided @@ -340,6 +370,9 @@ func TestGetKubeconfig(t *testing.T) { }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: autodiscovered Concierge CA bundle is invalid: illegal base64 data at input byte 7 @@ -372,6 +405,13 @@ func TestGetKubeconfig(t *testing.T) { }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="detected Concierge in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, + }, wantError: true, wantStderr: here.Doc(` Error: could not autodiscover --oidc-issuer and none was provided @@ -401,6 +441,12 @@ func TestGetKubeconfig(t *testing.T) { }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected OIDC issuer" "issuer"=""`, + `"level"=0 "msg"="detected OIDC audience" "audience"=""`, + }, wantError: true, wantStderr: here.Doc(` Error: tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator test-authenticator has invalid spec.tls.certificateAuthorityData: illegal base64 data at input byte 7 @@ -428,6 +474,9 @@ func TestGetKubeconfig(t *testing.T) { }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + }, wantError: true, wantStderr: here.Doc(` Error: could not read --concierge-ca-bundle: open ./does/not/exist: no such file or directory @@ -452,6 +501,12 @@ func TestGetKubeconfig(t *testing.T) { }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, + }, wantError: true, wantStderr: here.Doc(` Error: only one of --static-token and --static-token-env can be specified @@ -485,6 +540,12 @@ func TestGetKubeconfig(t *testing.T) { }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, + }, wantStdout: here.Doc(` apiVersion: v1 clusters: @@ -539,6 +600,12 @@ func TestGetKubeconfig(t *testing.T) { }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, + }, wantStdout: here.Doc(` apiVersion: v1 clusters: @@ -601,6 +668,15 @@ func TestGetKubeconfig(t *testing.T) { }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="detected OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="detected OIDC CA bundle" "length"=587`, + `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, + }, wantStdout: here.Docf(` apiVersion: v1 clusters: @@ -645,9 +721,12 @@ func TestGetKubeconfig(t *testing.T) { name: "autodetect nothing, set a bunch of options", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-credential-issuer", "test-credential-issuer", "--concierge-api-group-suffix", "tuna.io", "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", + "--concierge-endpoint", "https://concierge-endpoint.example.com", + "--concierge-ca-bundle", testConciergeCABundlePath, "--oidc-issuer", "https://example.com/issuer", "--oidc-skip-browser", "--oidc-listen-port", "1234", @@ -697,8 +776,8 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-api-group-suffix=tuna.io - --concierge-authenticator-name=test-authenticator - --concierge-authenticator-type=webhook - - --concierge-endpoint=https://fake-server-url-value - - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== + - --concierge-endpoint=https://concierge-endpoint.example.com + - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= - --concierge-mode=TokenCredentialRequestAPI - --issuer=https://example.com/issuer - --client-id=pinniped-cli @@ -739,6 +818,14 @@ func TestGetKubeconfig(t *testing.T) { }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="detected OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="detected OIDC CA bundle" "length"=587`, + `"level"=0 "msg"="switching kubeconfig cluster to point at impersonation proxy endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + }, wantStdout: here.Docf(` apiVersion: v1 clusters: @@ -831,6 +918,15 @@ func TestGetKubeconfig(t *testing.T) { }, }, }, + wantLogs: []string{ + `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="detected Concierge in impersonation proxy mode" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, + `"level"=0 "msg"="detected OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="detected OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="detected OIDC CA bundle" "length"=587`, + `"level"=0 "msg"="switching kubeconfig cluster to point at impersonation proxy endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + }, wantStdout: here.Docf(` apiVersion: v1 clusters: @@ -875,6 +971,7 @@ func TestGetKubeconfig(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + testLog := testlogger.New(t) cmd := kubeconfigCommand(kubeconfigDeps{ getPathToSelf: func() (string, error) { if tt.getPathToSelfErr != nil { @@ -897,6 +994,7 @@ func TestGetKubeconfig(t *testing.T) { } return fake, nil }, + log: testLog, }) require.NotNil(t, cmd) @@ -910,6 +1008,7 @@ func TestGetKubeconfig(t *testing.T) { } else { require.NoError(t, err) } + testLog.Expect(tt.wantLogs) require.Equal(t, tt.wantStdout, stdout.String(), "unexpected stdout") require.Equal(t, tt.wantStderr, stderr.String(), "unexpected stderr") }) From ce1b6303d975e55ea3e08919d1535446b7ce8609 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 15:53:30 -0600 Subject: [PATCH 101/203] Add an "--output" flag to "pinniped get kubeconfig". Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 14 +++++++++++++- cmd/pinniped/cmd/kubeconfig_test.go | 1 + 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 3314e7a0..63415711 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -91,6 +91,7 @@ type getKubeconfigConciergeParams struct { type getKubeconfigParams struct { kubeconfigPath string kubeconfigContextOverride string + outputPath string staticToken string staticTokenEnvName string oidc getKubeconfigOIDCParams @@ -135,13 +136,24 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.oidc.requestAudience, "oidc-request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange") f.StringVar(&flags.kubeconfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig file") f.StringVar(&flags.kubeconfigContextOverride, "kubeconfig-context", "", "Kubeconfig context name (default: current active context)") + f.StringVarP(&flags.outputPath, "output", "o", "", "Output file path (default: stdout)") mustMarkHidden(cmd, "oidc-debug-session-cache") mustMarkDeprecated(cmd, "concierge-namespace", "not needed anymore") mustMarkHidden(cmd, "concierge-namespace") - cmd.RunE = func(cmd *cobra.Command, args []string) error { return runGetKubeconfig(cmd.OutOrStdout(), deps, flags) } + cmd.RunE = func(cmd *cobra.Command, args []string) error { + if flags.outputPath != "" { + out, err := os.Create(flags.outputPath) + if err != nil { + return fmt.Errorf("could not open output file: %w", err) + } + defer func() { _ = out.Close() }() + cmd.SetOut(out) + } + return runGetKubeconfig(cmd.OutOrStdout(), deps, flags) + } return cmd } diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 7d84d31c..c5b5f1c0 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -83,6 +83,7 @@ func TestGetKubeconfig(t *testing.T) { --oidc-scopes strings OpenID Connect scopes to request during login (default [offline_access,openid,pinniped:request-audience]) --oidc-session-cache string Path to OpenID Connect session cache file --oidc-skip-browser During OpenID Connect login, skip opening the browser (just print the URL) + -o, --output string Output file path (default: stdout) --static-token string Instead of doing an OIDC-based login, specify a static token --static-token-env string Instead of doing an OIDC-based login, read a static token from the environment `), From 5d8594b285ba7176316c7710fa497e71ff19c6ff Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 16:35:42 -0600 Subject: [PATCH 102/203] Add validation step to "pinniped get kubeconfig". This adds two new flags to "pinniped get kubeconfig": --skip-validation and --timeout. By default, at the end of the kubeconfig generation process, we validate that we can reach the configured cluster. In the future this might also validate that the TokenCredentialRequest API is running, but for not it just verifies that the DNS name resolves, and the TLS connection is available on the given port. If there is an error during this check, we block and retry for up to 10 minutes. This duration can be changed with --timeout an the entire process can be skipped with --skip-validation. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 101 ++++++++++++++++++++++++++-- cmd/pinniped/cmd/kubeconfig_test.go | 8 +++ 2 files changed, 105 insertions(+), 4 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 63415711..610b1cc0 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -6,11 +6,14 @@ package cmd import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/base64" "fmt" "io" "io/ioutil" "log" + "net/http" "os" "strconv" "strings" @@ -91,6 +94,8 @@ type getKubeconfigConciergeParams struct { type getKubeconfigParams struct { kubeconfigPath string kubeconfigContextOverride string + skipValidate bool + timeout time.Duration outputPath string staticToken string staticTokenEnvName string @@ -136,6 +141,8 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.oidc.requestAudience, "oidc-request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange") f.StringVar(&flags.kubeconfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig file") f.StringVar(&flags.kubeconfigContextOverride, "kubeconfig-context", "", "Kubeconfig context name (default: current active context)") + f.BoolVar(&flags.skipValidate, "skip-validation", false, "Skip final validation of the kubeconfig (default: false)") + f.DurationVar(&flags.timeout, "timeout", 10*time.Minute, "Timeout for autodiscovery and validation") f.StringVarP(&flags.outputPath, "output", "o", "", "Output file path (default: stdout)") mustMarkHidden(cmd, "oidc-debug-session-cache") @@ -152,13 +159,16 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { defer func() { _ = out.Close() }() cmd.SetOut(out) } - return runGetKubeconfig(cmd.OutOrStdout(), deps, flags) + return runGetKubeconfig(cmd.Context(), cmd.OutOrStdout(), deps, flags) } return cmd } //nolint:funlen -func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigParams) error { +func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, flags getKubeconfigParams) error { + ctx, cancel := context.WithTimeout(ctx, flags.timeout) + defer cancel() + // Validate api group suffix and immediately return an error if it is invalid. if err := groupsuffix.Validate(flags.concierge.apiGroupSuffix); err != nil { return fmt.Errorf("invalid api group suffix: %w", err) @@ -229,7 +239,12 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if flags.staticTokenEnvName != "" { execConfig.Args = append(execConfig.Args, "--token-env="+flags.staticTokenEnvName) } - return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig)) + + kubeconfig := newExecKubeconfig(cluster, &execConfig) + if err := validateKubeconfig(ctx, flags, kubeconfig, deps.log); err != nil { + return err + } + return writeConfigAsYAML(out, kubeconfig) } // Otherwise continue to parse the OIDC-related flags and output a config that runs `pinniped login oidc`. @@ -260,7 +275,11 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar if flags.oidc.requestAudience != "" { execConfig.Args = append(execConfig.Args, "--request-audience="+flags.oidc.requestAudience) } - return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig)) + kubeconfig := newExecKubeconfig(cluster, &execConfig) + if err := validateKubeconfig(ctx, flags, kubeconfig, deps.log); err != nil { + return err + } + return writeConfigAsYAML(out, kubeconfig) } func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig, log logr.Logger) error { @@ -518,3 +537,77 @@ func copyCurrentClusterFromExistingKubeConfig(currentKubeConfig clientcmdapi.Con } return currentKubeConfig.Clusters[ctx.Cluster], nil } + +func validateKubeconfig(ctx context.Context, flags getKubeconfigParams, kubeconfig clientcmdapi.Config, log logr.Logger) error { + if flags.skipValidate { + return nil + } + + kubeContext := kubeconfig.Contexts[kubeconfig.CurrentContext] + if kubeContext == nil { + return fmt.Errorf("invalid kubeconfig (no context)") + } + cluster := kubeconfig.Clusters[kubeContext.Cluster] + if cluster == nil { + return fmt.Errorf("invalid kubeconfig (no cluster)") + } + + kubeconfigCA := x509.NewCertPool() + if !kubeconfigCA.AppendCertsFromPEM(cluster.CertificateAuthorityData) { + return fmt.Errorf("invalid kubeconfig (no certificateAuthorityData)") + } + + httpClient := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: kubeconfigCA, + }, + Proxy: http.ProxyFromEnvironment, + TLSHandshakeTimeout: 10 * time.Second, + }, + Timeout: 10 * time.Second, + } + + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + pingCluster := func() error { + reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, cluster.Server, nil) + if err != nil { + return fmt.Errorf("could not form request to validate cluster: %w", err) + } + resp, err := httpClient.Do(req) + if err != nil { + return err + } + _ = resp.Body.Close() + if resp.StatusCode >= 500 { + return fmt.Errorf("unexpected status code %d", resp.StatusCode) + } + return nil + } + + err := pingCluster() + if err == nil { + log.Info("validated connection to the cluster") + return nil + } + + log.Info("could not immediately connect to the cluster but it may be initializing, will retry until timeout") + deadline, _ := ctx.Deadline() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + err := pingCluster() + if err == nil { + return nil + } + log.Error(err, "could not connect to cluster, retrying...", "remaining", time.Until(deadline).Round(time.Second).String()) + } + } +} diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index c5b5f1c0..c9dd5b77 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -84,8 +84,10 @@ func TestGetKubeconfig(t *testing.T) { --oidc-session-cache string Path to OpenID Connect session cache file --oidc-skip-browser During OpenID Connect login, skip opening the browser (just print the URL) -o, --output string Output file path (default: stdout) + --skip-validation Skip final validation of the kubeconfig (default: false) --static-token string Instead of doing an OIDC-based login, specify a static token --static-token-env string Instead of doing an OIDC-based login, read a static token from the environment + --timeout duration Timeout for autodiscovery and validation (default 10m0s) `), }, { @@ -528,6 +530,7 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", "--static-token", "test-token", + "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ @@ -588,6 +591,7 @@ func TestGetKubeconfig(t *testing.T) { args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", "--static-token-env", "TEST_TOKEN", + "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ @@ -647,6 +651,7 @@ func TestGetKubeconfig(t *testing.T) { name: "autodetect JWT authenticator", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", + "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ @@ -735,6 +740,7 @@ func TestGetKubeconfig(t *testing.T) { "--oidc-session-cache", "/path/to/cache/dir/sessions.yaml", "--oidc-debug-session-cache", "--oidc-request-audience", "test-audience", + "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ @@ -802,6 +808,7 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-ca-bundle", testConciergeCABundlePath, "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", "--concierge-mode", "ImpersonationProxy", + "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ @@ -871,6 +878,7 @@ func TestGetKubeconfig(t *testing.T) { name: "autodetect impersonation proxy with autodetected JWT authenticator", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", + "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ From ba0dc3bf5251ca784b8e09ce259b6c48ade0fdf5 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 16:39:57 -0600 Subject: [PATCH 103/203] Remove this test retry loop since the "get kubeconfig" step should now wait. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 39 ------------------------------------ 1 file changed, 39 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 5973a8a9..a1eecc6a 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -6,14 +6,11 @@ import ( "bufio" "bytes" "context" - "crypto/tls" - "crypto/x509" "crypto/x509/pkix" "encoding/base64" "errors" "fmt" "io/ioutil" - "net/http" "net/url" "os" "os/exec" @@ -173,42 +170,6 @@ func TestE2EFullIntegration(t *testing.T) { t.Log("sleeping 10s to wait for JWTAuthenticator to become initialized") time.Sleep(10 * time.Second) - // Verify that we can actually reach the endpoint in the kubeconfig. - require.Eventually(t, func() bool { - kubeconfigCA := x509.NewCertPool() - require.True(t, kubeconfigCA.AppendCertsFromPEM(restConfig.TLSClientConfig.CAData), "expected to load kubeconfig CA") - - // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. - httpClient := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: kubeconfigCA, - }, - Proxy: func(req *http.Request) (*url.URL, error) { - if env.Proxy == "" { - t.Logf("passing request for %s with no proxy", req.URL) - return nil, nil - } - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil - }, - }, - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, restConfig.Host, nil) - require.NoError(t, err) - resp, err := httpClient.Do(req) - if err != nil { - t.Logf("could not connect to the API server at %q: %v", restConfig.Host, err) - return false - } - t.Logf("got %d response from API server at %q", resp.StatusCode, restConfig.Host) - require.NoError(t, resp.Body.Close()) - return resp.StatusCode < 500 - }, 5*time.Minute, 2*time.Second) - // Run "kubectl get namespaces" which should trigger a browser login via the plugin. start := time.Now() kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) From 4750d7d7d2ac5061e3186b239f729b116a4af459 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 16:57:24 -0600 Subject: [PATCH 104/203] The stderr from "pinniped get kubeconfig" is no longer empty. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index a1eecc6a..daad3425 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -155,8 +155,7 @@ func TestE2EFullIntegration(t *testing.T) { "--oidc-ca-bundle", testCABundlePath, "--oidc-session-cache", sessionCachePath, ) - require.Equal(t, "", stderr) - + t.Logf("stderr output from 'pinnipedget kubeconfig':\n%s\n\n", stderr) t.Logf("test kubeconfig:\n%s\n\n", kubeconfigYAML) restConfig := library.NewRestConfigFromKubeconfig(t, kubeconfigYAML) From 73419313ee7fa9647ae0f77538ad78458c42e73f Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 5 Mar 2021 16:59:43 -0600 Subject: [PATCH 105/203] Log when the validation eventually succeeds. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 610b1cc0..d0025840 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -598,16 +598,19 @@ func validateKubeconfig(ctx context.Context, flags getKubeconfigParams, kubeconf log.Info("could not immediately connect to the cluster but it may be initializing, will retry until timeout") deadline, _ := ctx.Deadline() + attempts := 0 for { select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: + attempts++ err := pingCluster() if err == nil { + log.Info("validated connection to the cluster", "attempts", attempts) return nil } - log.Error(err, "could not connect to cluster, retrying...", "remaining", time.Until(deadline).Round(time.Second).String()) + log.Error(err, "could not connect to cluster, retrying...", "attempts", attempts, "remaining", time.Until(deadline).Round(time.Second).String()) } } } From 49ec16038c40aa1fdb1ba70fa681146aa8419e09 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 5 Mar 2021 16:14:45 -0800 Subject: [PATCH 106/203] Add integration test for using "kubectl exec" through the impersonator Signed-off-by: Margo Crawford --- test/integration/cli_test.go | 5 +- .../concierge_impersonation_proxy_test.go | 90 +++++++++++++++++++ test/integration/e2e_test.go | 2 +- 3 files changed, 94 insertions(+), 3 deletions(-) diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index e055fccf..f2479d0c 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -64,7 +64,7 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { } { tt := tt t.Run(tt.name, func(t *testing.T) { - stdout, stderr := runPinnipedCLI(t, pinnipedExe, tt.args...) + stdout, stderr := runPinnipedCLI(t, nil, pinnipedExe, tt.args...) require.Equal(t, tt.expectStderr, stderr) // Even the deprecated command should now generate a kubeconfig with the new "pinniped login static" command. @@ -99,12 +99,13 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { } } -func runPinnipedCLI(t *testing.T, pinnipedExe string, args ...string) (string, string) { +func runPinnipedCLI(t *testing.T, envVars []string, pinnipedExe string, args ...string) (string, string) { t.Helper() var stdout, stderr bytes.Buffer cmd := exec.Command(pinnipedExe, args...) cmd.Stdout = &stdout cmd.Stderr = &stderr + cmd.Env = envVars require.NoErrorf(t, cmd.Run(), "stderr:\n%s\n\nstdout:\n%s\n\n", stderr.String(), stdout.String()) return stdout.String(), stderr.String() } diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 369930f5..1bdd9316 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -4,11 +4,17 @@ package integration import ( + "bytes" "context" "encoding/base64" "fmt" + "io/ioutil" "net/http" "net/url" + "os" + "os/exec" + "path/filepath" + "strings" "testing" "time" @@ -28,6 +34,7 @@ import ( "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/testutil" "go.pinniped.dev/internal/testutil/impersonationtoken" "go.pinniped.dev/test/library" ) @@ -351,6 +358,89 @@ func TestImpersonationProxy(t *testing.T) { require.EqualError(t, err, expectedErr) }) + t.Run("kubectl as a client", func(t *testing.T) { + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Verb: "get", Group: "", Version: "v1", Resource: "namespaces", + }) + + pinnipedExe := library.PinnipedCLIPath(t) + tempDir := testutil.TempDir(t) + + var envVarsWithProxy []string + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + // Only if you don't have a load balancer, use the squid proxy when it's available. + envVarsWithProxy = append(os.Environ(), env.ProxyEnv()...) + } + + // Get the kubeconfig. + getKubeConfigCmd := []string{"get", "kubeconfig", + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--oidc-skip-browser", + "--static-token", env.TestUser.Token, + // Force the use of impersonation proxy strategy, but let it auto-discover the endpoint and CA. + "--concierge-mode", "ImpersonationProxy"} + t.Log("Running:", pinnipedExe, getKubeConfigCmd) + kubeconfigYAML, getKubeConfigStderr := runPinnipedCLI(t, envVarsWithProxy, pinnipedExe, getKubeConfigCmd...) + // "pinniped get kubectl" prints some status messages to stderr + t.Log(getKubeConfigStderr) + // Make sure that the "pinniped get kubeconfig" auto-discovered the impersonation proxy and we're going to + // make our kubectl requests through the impersonation proxy. Avoid using require.Contains because the error + // message would contain credentials. + require.True(t, + strings.Contains(kubeconfigYAML, "server: "+impersonationProxyURL+"\n"), + "the generated kubeconfig did not include the expected impersonation server address: %s", + impersonationProxyURL, + ) + require.True(t, + strings.Contains(kubeconfigYAML, "- --concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(impersonationProxyCACertPEM)+"\n"), + "the generated kubeconfig did not include the base64 encoded version of this expected impersonation CA cert: %s", + impersonationProxyCACertPEM, + ) + + // Write the kubeconfig to a temp file. + kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") + require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) + + // Func to run kubeconfig commands. + kubectl := func(args ...string) (string, string, error) { + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + + allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) + //nolint:gosec // we are not performing malicious argument injection against ourselves + kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) + var stdout, stderr bytes.Buffer + kubectlCmd.Stdout = &stdout + kubectlCmd.Stderr = &stderr + kubectlCmd.Env = envVarsWithProxy + + t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " ")) + err := kubectlCmd.Run() + t.Logf("kubectl stdout output: %s", stdout.String()) + t.Logf("kubectl stderr output: %s", stderr.String()) + return stdout.String(), stderr.String(), err + } + + // Get pods in concierge namespace and pick one. + // We don't actually care which pod, just want to see that we can "exec echo" in one of them. + pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Greater(t, len(pods.Items), 0) + podName := pods.Items[0].Name + + // Try "kubectl exec" through the impersonation proxy. + echoString := "hello world" + stdout, _, err := kubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "echo", echoString) + require.NoError(t, err) + require.Equal(t, stdout, echoString+"\n") + }) + // Update configuration to force the proxy to disabled mode configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index daad3425..910eb573 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -147,7 +147,7 @@ func TestE2EFullIntegration(t *testing.T) { sessionCachePath := tempDir + "/sessions.yaml" // Run "pinniped get kubeconfig" to get a kubeconfig YAML. - kubeconfigYAML, stderr := runPinnipedCLI(t, pinnipedExe, "get", "kubeconfig", + kubeconfigYAML, stderr := runPinnipedCLI(t, nil, pinnipedExe, "get", "kubeconfig", "--concierge-api-group-suffix", env.APIGroupSuffix, "--concierge-authenticator-type", "jwt", "--concierge-authenticator-name", authenticator.Name, From 389cd3486babfc9f37d0d707c69ab2ea74ce794e Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 8 Mar 2021 11:43:56 -0600 Subject: [PATCH 107/203] Rework "pinniped get kubeconfig" so that --concierge-mode can be used even when auto-discovering other parameters. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/flag_types.go | 16 +- cmd/pinniped/cmd/flag_types_test.go | 10 ++ cmd/pinniped/cmd/kubeconfig.go | 160 +++++++++++------- cmd/pinniped/cmd/kubeconfig_test.go | 246 ++++++++++++++++++++-------- 4 files changed, 306 insertions(+), 126 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index 73c8cfd1..c0e3624d 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -7,6 +7,8 @@ import ( "flag" "fmt" "strings" + + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" ) // conciergeMode represents the method by which we should connect to the Concierge on a cluster during login. @@ -25,7 +27,7 @@ func (c *conciergeMode) String() string { switch *c { case modeImpersonationProxy: return "ImpersonationProxy" - case modeTokenCredentialRequestAPI, modeUnknown: + case modeTokenCredentialRequestAPI: return "TokenCredentialRequestAPI" default: return "TokenCredentialRequestAPI" @@ -51,3 +53,15 @@ func (c *conciergeMode) Set(s string) error { func (c *conciergeMode) Type() string { return "mode" } + +// MatchesFrontend returns true iff the flag matches the type of the provided frontend. +func (c *conciergeMode) MatchesFrontend(frontend *configv1alpha1.CredentialIssuerFrontend) bool { + switch *c { + case modeImpersonationProxy: + return frontend.Type == configv1alpha1.ImpersonationProxyFrontendType + case modeTokenCredentialRequestAPI: + return frontend.Type == configv1alpha1.TokenCredentialRequestAPIFrontendType + default: + return true + } +} diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go index 26f107b7..8c0faeda 100644 --- a/cmd/pinniped/cmd/flag_types_test.go +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -7,17 +7,25 @@ import ( "testing" "github.com/stretchr/testify/require" + + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" ) func TestConciergeModeFlag(t *testing.T) { var m conciergeMode require.Equal(t, "mode", m.Type()) require.Equal(t, modeUnknown, m) + require.NoError(t, m.Set("")) + require.Equal(t, modeUnknown, m) require.EqualError(t, m.Set("foo"), `invalid mode "foo", valid modes are TokenCredentialRequestAPI and ImpersonationProxy`) + require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) require.NoError(t, m.Set("TokenCredentialRequestAPI")) require.Equal(t, modeTokenCredentialRequestAPI, m) require.Equal(t, "TokenCredentialRequestAPI", m.String()) + require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.False(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) require.NoError(t, m.Set("tokencredentialrequestapi")) require.Equal(t, modeTokenCredentialRequestAPI, m) @@ -26,6 +34,8 @@ func TestConciergeModeFlag(t *testing.T) { require.NoError(t, m.Set("ImpersonationProxy")) require.Equal(t, modeImpersonationProxy, m) require.Equal(t, "ImpersonationProxy", m.String()) + require.False(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) require.NoError(t, m.Set("impersonationproxy")) require.Equal(t, modeImpersonationProxy, m) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index d0025840..968e15d7 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -283,43 +283,56 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f } func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig, log logr.Logger) error { - var conciergeCABundleData []byte - // Autodiscover the --concierge-mode. - if flags.concierge.mode == modeUnknown { //nolint:nestif - strategyLoop: - for _, strategy := range credentialIssuer.Status.Strategies { - if strategy.Status != configv1alpha1.SuccessStrategyStatus || strategy.Frontend == nil { - continue - } - switch strategy.Frontend.Type { - case configv1alpha1.TokenCredentialRequestAPIFrontendType: - log.Info("detected Concierge in TokenCredentialRequest API mode") - flags.concierge.mode = modeTokenCredentialRequestAPI - break strategyLoop - case configv1alpha1.ImpersonationProxyFrontendType: + frontend, err := getConciergeFrontend(credentialIssuer, flags.concierge.mode) + if err != nil { + return err + } - flags.concierge.mode = modeImpersonationProxy - flags.concierge.endpoint = strategy.Frontend.ImpersonationProxyInfo.Endpoint - var err error - conciergeCABundleData, err = base64.StdEncoding.DecodeString(strategy.Frontend.ImpersonationProxyInfo.CertificateAuthorityData) - if err != nil { - return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) - } - log.Info("detected Concierge in impersonation proxy mode", "endpoint", strategy.Frontend.ImpersonationProxyInfo.Endpoint) - break strategyLoop - default: - // Skip any unknown frontend types. - } - } - if flags.concierge.mode == modeUnknown { - // Fall back to deprecated field for backwards compatibility. - if credentialIssuer.Status.KubeConfigInfo != nil { - flags.concierge.mode = modeTokenCredentialRequestAPI - } else { - return fmt.Errorf("could not autodiscover --concierge-mode and none was provided") + // Auto-set --concierge-mode if it wasn't explicitly set. + if flags.concierge.mode == modeUnknown { + switch frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType: + log.Info("discovered Concierge operating in TokenCredentialRequest API mode") + flags.concierge.mode = modeTokenCredentialRequestAPI + case configv1alpha1.ImpersonationProxyFrontendType: + log.Info("discovered Concierge operating in impersonation proxy mode") + flags.concierge.mode = modeImpersonationProxy + } + } + + // Auto-set --concierge-endpoint if it wasn't explicitly set. + if flags.concierge.endpoint == "" { + switch frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType: + flags.concierge.endpoint = v1Cluster.Server + case configv1alpha1.ImpersonationProxyFrontendType: + flags.concierge.endpoint = frontend.ImpersonationProxyInfo.Endpoint + } + log.Info("discovered Concierge endpoint", "endpoint", flags.concierge.endpoint) + } + + // Load specified --concierge-ca-bundle or autodiscover a value. + var conciergeCABundleData []byte + if flags.concierge.caBundlePath != "" { + caBundleString, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) + if err != nil { + return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) + } + conciergeCABundleData = []byte(caBundleString) + log.Info("loaded Concierge certificate authority bundle", "roots", countCACerts(conciergeCABundleData)) + } else { + switch frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType: + conciergeCABundleData = v1Cluster.CertificateAuthorityData + case configv1alpha1.ImpersonationProxyFrontendType: + var err error + conciergeCABundleData, err = base64.StdEncoding.DecodeString(frontend.ImpersonationProxyInfo.CertificateAuthorityData) + if err != nil { + return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) } } + log.Info("discovered Concierge certificate authority bundle", "roots", countCACerts(conciergeCABundleData)) } switch auth := authenticator.(type) { @@ -342,13 +355,13 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // If the --oidc-issuer flag was not set explicitly, default it to the spec.issuer field of the JWTAuthenticator. if flags.oidc.issuer == "" { - log.Info("detected OIDC issuer", "issuer", auth.Spec.Issuer) + log.Info("discovered OIDC issuer", "issuer", auth.Spec.Issuer) flags.oidc.issuer = auth.Spec.Issuer } // If the --oidc-request-audience flag was not set explicitly, default it to the spec.audience field of the JWTAuthenticator. if flags.oidc.requestAudience == "" { - log.Info("detected OIDC audience", "audience", auth.Spec.Audience) + log.Info("discovered OIDC audience", "audience", auth.Spec.Audience) flags.oidc.requestAudience = auth.Spec.Audience } @@ -359,29 +372,11 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe if err != nil { return fmt.Errorf("tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator %s has invalid spec.tls.certificateAuthorityData: %w", auth.Name, err) } - log.Info("detected OIDC CA bundle", "length", len(decoded)) + log.Info("discovered OIDC CA bundle", "roots", countCACerts(decoded)) *oidcCABundle = string(decoded) } } - if flags.concierge.endpoint == "" { - log.Info("detected concierge endpoint", "endpoint", v1Cluster.Server) - flags.concierge.endpoint = v1Cluster.Server - } - - if conciergeCABundleData == nil { - if flags.concierge.caBundlePath == "" { - log.Info("detected concierge CA bundle", "length", len(v1Cluster.CertificateAuthorityData)) - conciergeCABundleData = v1Cluster.CertificateAuthorityData - } else { - caBundleString, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) - if err != nil { - return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) - } - conciergeCABundleData = []byte(caBundleString) - } - } - // Append the flags to configure the Concierge credential exchange at runtime. execConfig.Args = append(execConfig.Args, "--enable-concierge", @@ -393,14 +388,53 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe "--concierge-mode="+flags.concierge.mode.String(), ) - // If we're in impersonation proxy mode, the main server endpoint for the kubeconfig also needs to point to the proxy - if flags.concierge.mode == modeImpersonationProxy { - log.Info("switching kubeconfig cluster to point at impersonation proxy endpoint", "endpoint", flags.concierge.endpoint) - v1Cluster.CertificateAuthorityData = conciergeCABundleData - v1Cluster.Server = flags.concierge.endpoint + // Point kubectl at the concierge endpoint. + v1Cluster.Server = flags.concierge.endpoint + v1Cluster.CertificateAuthorityData = conciergeCABundleData + return nil +} + +func getConciergeFrontend(credentialIssuer *configv1alpha1.CredentialIssuer, mode conciergeMode) (*configv1alpha1.CredentialIssuerFrontend, error) { + for _, strategy := range credentialIssuer.Status.Strategies { + // Skip unhealthy strategies. + if strategy.Status != configv1alpha1.SuccessStrategyStatus { + continue + } + + // Backfill the .status.strategies[].frontend field from .status.kubeConfigInfo for backwards compatibility. + if strategy.Type == configv1alpha1.KubeClusterSigningCertificateStrategyType && strategy.Frontend == nil && credentialIssuer.Status.KubeConfigInfo != nil { + strategy = *strategy.DeepCopy() + strategy.Frontend = &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: credentialIssuer.Status.KubeConfigInfo.Server, + CertificateAuthorityData: credentialIssuer.Status.KubeConfigInfo.CertificateAuthorityData, + }, + } + } + + // If the strategy frontend is still nil, skip. + if strategy.Frontend == nil { + continue + } + + // Skip any unknown frontend types. + switch strategy.Frontend.Type { + case configv1alpha1.TokenCredentialRequestAPIFrontendType, configv1alpha1.ImpersonationProxyFrontendType: + default: + continue + } + // Skip strategies that don't match --concierge-mode. + if !mode.MatchesFrontend(strategy.Frontend) { + continue + } + return strategy.Frontend, nil } - return nil + if mode == modeUnknown { + return nil, fmt.Errorf("could not autodiscover --concierge-mode") + } + return nil, fmt.Errorf("could not find successful Concierge strategy matching --concierge-mode=%s", mode.String()) } func loadCABundlePaths(paths []string) (string, error) { @@ -614,3 +648,9 @@ func validateKubeconfig(ctx context.Context, flags getKubeconfigParams, kubeconf } } } + +func countCACerts(pemData []byte) int { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(pemData) + return len(pool.Subjects()) +} diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index c9dd5b77..34c78ce3 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -36,8 +36,10 @@ func TestGetKubeconfig(t *testing.T) { testOIDCCABundlePath := filepath.Join(tmpdir, "testca.pem") require.NoError(t, ioutil.WriteFile(testOIDCCABundlePath, testOIDCCA.Bundle(), 0600)) + testConciergeCA, err := certauthority.New(pkix.Name{CommonName: "Test Concierge CA"}, 1*time.Hour) + require.NoError(t, err) testConciergeCABundlePath := filepath.Join(tmpdir, "testconciergeca.pem") - require.NoError(t, ioutil.WriteFile(testConciergeCABundlePath, []byte("test-concierge-ca"), 0600)) + require.NoError(t, ioutil.WriteFile(testConciergeCABundlePath, testConciergeCA.Bundle(), 0600)) tests := []struct { name string @@ -324,7 +326,7 @@ func TestGetKubeconfig(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: could not autodiscover --concierge-mode and none was provided + Error: could not autodiscover --concierge-mode `), }, { @@ -375,6 +377,8 @@ func TestGetKubeconfig(t *testing.T) { }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in impersonation proxy mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-endpoint"`, }, wantError: true, wantStderr: here.Doc(` @@ -410,10 +414,10 @@ func TestGetKubeconfig(t *testing.T) { }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, - `"level"=0 "msg"="detected Concierge in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, - `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, }, wantError: true, wantStderr: here.Doc(` @@ -433,11 +437,22 @@ func TestGetKubeconfig(t *testing.T) { Server: "https://concierge-endpoint", CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Message: "Successfully fetched key", + LastUpdateTime: metav1.Now(), + // Simulate a previous version of CredentialIssuer that's missing this Frontend field. + Frontend: nil, + }}, }, }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, Spec: conciergev1alpha1.JWTAuthenticatorSpec{ + Issuer: "https://test-issuer.example.com", + Audience: "some-test-audience", TLS: &conciergev1alpha1.TLSSpec{ CertificateAuthorityData: "invalid-base64", }, @@ -446,9 +461,12 @@ func TestGetKubeconfig(t *testing.T) { }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected OIDC issuer" "issuer"=""`, - `"level"=0 "msg"="detected OIDC audience" "audience"=""`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://test-issuer.example.com"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="some-test-audience"`, }, wantError: true, wantStderr: here.Doc(` @@ -469,10 +487,18 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://concierge-endpoint", - CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", - }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.ImpersonationProxyStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.ListeningStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, }, }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, @@ -496,19 +522,28 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://concierge-endpoint", - CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", - }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.ImpersonationProxyStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.ListeningStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, }, }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in impersonation proxy mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-proxy-endpoint.example.com"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=1`, `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, - `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, }, wantError: true, wantStderr: here.Doc(` @@ -536,19 +571,28 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://concierge-endpoint", - CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", - }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, }, }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, - `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, }, wantStdout: here.Doc(` apiVersion: v1 @@ -597,19 +641,28 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://concierge-endpoint", - CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", - }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, }, }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, `"level"=0 "msg"="discovered WebhookAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, - `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, }, wantStdout: here.Doc(` apiVersion: v1 @@ -657,10 +710,18 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://concierge-endpoint", - CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", - }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }}, }, }, &conciergev1alpha1.JWTAuthenticator{ @@ -676,12 +737,13 @@ func TestGetKubeconfig(t *testing.T) { }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge operating in TokenCredentialRequest API mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://fake-server-url-value"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected OIDC issuer" "issuer"="https://example.com/issuer"`, - `"level"=0 "msg"="detected OIDC audience" "audience"="test-audience"`, - `"level"=0 "msg"="detected OIDC CA bundle" "length"=587`, - `"level"=0 "msg"="detected concierge endpoint" "endpoint"="https://fake-server-url-value"`, - `"level"=0 "msg"="detected concierge CA bundle" "length"=37`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="discovered OIDC CA bundle" "roots"=1`, }, wantStdout: here.Docf(` apiVersion: v1 @@ -731,7 +793,8 @@ func TestGetKubeconfig(t *testing.T) { "--concierge-api-group-suffix", "tuna.io", "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", "test-authenticator", - "--concierge-endpoint", "https://concierge-endpoint.example.com", + "--concierge-mode", "TokenCredentialRequestAPI", + "--concierge-endpoint", "https://explicit-concierge-endpoint.example.com", "--concierge-ca-bundle", testConciergeCABundlePath, "--oidc-issuer", "https://example.com/issuer", "--oidc-skip-browser", @@ -746,22 +809,33 @@ func TestGetKubeconfig(t *testing.T) { &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, Status: configv1alpha1.CredentialIssuerStatus{ - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://concierge-endpoint", - CertificateAuthorityData: "ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ==", - }, + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://concierge-endpoint.example.com", + CertificateAuthorityData: "dGVzdC10Y3ItYXBpLWNh", + }, + }, + }}, }, }, &conciergev1alpha1.WebhookAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, }, }, + wantLogs: []string{ + `"level"=0 "msg"="loaded Concierge certificate authority bundle" "roots"=1`, + }, wantStdout: here.Docf(` apiVersion: v1 clusters: - cluster: - certificate-authority-data: ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== - server: https://fake-server-url-value + certificate-authority-data: %s + server: https://explicit-concierge-endpoint.example.com name: pinniped contexts: - context: @@ -783,8 +857,8 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-api-group-suffix=tuna.io - --concierge-authenticator-name=test-authenticator - --concierge-authenticator-type=webhook - - --concierge-endpoint=https://concierge-endpoint.example.com - - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= + - --concierge-endpoint=https://explicit-concierge-endpoint.example.com + - --concierge-ca-bundle-data=%s - --concierge-mode=TokenCredentialRequestAPI - --issuer=https://example.com/issuer - --client-id=pinniped-cli @@ -798,22 +872,58 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), + `, + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + ), wantAPIGroupSuffix: "tuna.io", }, { - name: "configure impersonation proxy with autodetected JWT authenticator", + name: "configure impersonation proxy with autodiscovered JWT authenticator", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", - "--concierge-ca-bundle", testConciergeCABundlePath, - "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", "--concierge-mode", "ImpersonationProxy", "--skip-validation", }, conciergeObjects: []runtime.Object{ &configv1alpha1.CredentialIssuer{ ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, - Status: configv1alpha1.CredentialIssuerStatus{}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + // This TokenCredentialRequestAPI strategy would normally be chosen, but + // --concierge-mode=ImpersonationProxy should force it to be skipped. + { + Type: "SomeType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://token-credential-request-api-endpoint.test", + CertificateAuthorityData: "dGVzdC10Y3ItYXBpLWNh", + }, + }, + }, + // The endpoint and CA from this impersonation proxy strategy should be autodiscovered. + { + Type: "SomeOtherType", + Status: configv1alpha1.SuccessStrategyStatus, + Reason: "SomeOtherReason", + Message: "Some other message", + LastUpdateTime: metav1.Now(), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.ImpersonationProxyFrontendType, + ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ + Endpoint: "https://impersonation-proxy-endpoint.test", + CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + }, + }, + }, + }, + }, }, &conciergev1alpha1.JWTAuthenticator{ ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, @@ -828,17 +938,18 @@ func TestGetKubeconfig(t *testing.T) { }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=1`, `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected OIDC issuer" "issuer"="https://example.com/issuer"`, - `"level"=0 "msg"="detected OIDC audience" "audience"="test-audience"`, - `"level"=0 "msg"="detected OIDC CA bundle" "length"=587`, - `"level"=0 "msg"="switching kubeconfig cluster to point at impersonation proxy endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="discovered OIDC CA bundle" "roots"=1`, }, wantStdout: here.Docf(` apiVersion: v1 clusters: - cluster: - certificate-authority-data: dGVzdC1jb25jaWVyZ2UtY2E= + certificate-authority-data: %s server: https://impersonation-proxy-endpoint.test name: pinniped contexts: @@ -862,7 +973,7 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-name=test-authenticator - --concierge-authenticator-type=jwt - --concierge-endpoint=https://impersonation-proxy-endpoint.test - - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= + - --concierge-ca-bundle-data=%s - --concierge-mode=ImpersonationProxy - --issuer=https://example.com/issuer - --client-id=pinniped-cli @@ -872,10 +983,14 @@ func TestGetKubeconfig(t *testing.T) { command: '.../path/to/pinniped' env: [] provideClusterInfo: true - `, base64.StdEncoding.EncodeToString(testOIDCCA.Bundle())), + `, + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), + base64.StdEncoding.EncodeToString(testOIDCCA.Bundle()), + ), }, { - name: "autodetect impersonation proxy with autodetected JWT authenticator", + name: "autodetect impersonation proxy with autodiscovered JWT authenticator", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", "--skip-validation", @@ -929,12 +1044,13 @@ func TestGetKubeconfig(t *testing.T) { }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, - `"level"=0 "msg"="detected Concierge in impersonation proxy mode" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered Concierge operating in impersonation proxy mode"`, + `"level"=0 "msg"="discovered Concierge endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered Concierge certificate authority bundle" "roots"=0`, `"level"=0 "msg"="discovered JWTAuthenticator" "name"="test-authenticator"`, - `"level"=0 "msg"="detected OIDC issuer" "issuer"="https://example.com/issuer"`, - `"level"=0 "msg"="detected OIDC audience" "audience"="test-audience"`, - `"level"=0 "msg"="detected OIDC CA bundle" "length"=587`, - `"level"=0 "msg"="switching kubeconfig cluster to point at impersonation proxy endpoint" "endpoint"="https://impersonation-proxy-endpoint.test"`, + `"level"=0 "msg"="discovered OIDC issuer" "issuer"="https://example.com/issuer"`, + `"level"=0 "msg"="discovered OIDC audience" "audience"="test-audience"`, + `"level"=0 "msg"="discovered OIDC CA bundle" "roots"=1`, }, wantStdout: here.Docf(` apiVersion: v1 From 8c0a073cb63ddd9cb565cab9fa596ba5e4ef7518 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 8 Mar 2021 13:31:16 -0600 Subject: [PATCH 108/203] Fix this constant name to match its value. Signed-off-by: Matt Moyer --- .../controller/impersonatorconfig/impersonator_config.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index efbb3a27..c1e491aa 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -40,7 +40,7 @@ import ( const ( impersonationProxyPort = "8444" defaultHTTPSPort = 443 - oneYear = 100 * 365 * 24 * time.Hour + oneHundredYears = 100 * 365 * 24 * time.Hour caCommonName = "Pinniped Impersonation Proxy CA" caCrtKey = "ca.crt" caKeyKey = "ca.key" @@ -579,7 +579,7 @@ func (c *impersonatorConfigController) ensureCASecretIsCreated(ctx context.Conte } func (c *impersonatorConfigController) createCASecret(ctx context.Context) (*certauthority.CA, error) { - impersonationCA, err := certauthority.New(pkix.Name{CommonName: caCommonName}, oneYear) + impersonationCA, err := certauthority.New(pkix.Name{CommonName: caCommonName}, oneHundredYears) if err != nil { return nil, fmt.Errorf("could not create impersonation CA: %w", err) } @@ -673,7 +673,7 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c ips = []net.IP{ip} } - impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, oneYear) + impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, oneHundredYears) if err != nil { return nil, fmt.Errorf("could not create impersonation cert: %w", err) } From a059d8dfce4e8c3a19bf56e239c634bc742eae8a Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 8 Mar 2021 14:31:13 -0600 Subject: [PATCH 109/203] Refactor "get kubeconfig" a bit more to clean things up. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/flag_types.go | 39 ++++++++++++ cmd/pinniped/cmd/flag_types_test.go | 31 +++++++++ cmd/pinniped/cmd/kubeconfig.go | 98 +++++++++++------------------ cmd/pinniped/cmd/kubeconfig_test.go | 61 +++++------------- 4 files changed, 122 insertions(+), 107 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index c0e3624d..7b10e3a5 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -4,10 +4,15 @@ package cmd import ( + "bytes" + "crypto/x509" "flag" "fmt" + "io/ioutil" "strings" + "github.com/spf13/pflag" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" ) @@ -29,6 +34,8 @@ func (c *conciergeMode) String() string { return "ImpersonationProxy" case modeTokenCredentialRequestAPI: return "TokenCredentialRequestAPI" + case modeUnknown: + fallthrough default: return "TokenCredentialRequestAPI" } @@ -61,7 +68,39 @@ func (c *conciergeMode) MatchesFrontend(frontend *configv1alpha1.CredentialIssue return frontend.Type == configv1alpha1.ImpersonationProxyFrontendType case modeTokenCredentialRequestAPI: return frontend.Type == configv1alpha1.TokenCredentialRequestAPIFrontendType + case modeUnknown: + fallthrough default: return true } } + +// caBundlePathsVar represents a list of CA bundle paths, which load from disk when the flag is populated. +type caBundleVar []byte + +var _ pflag.Value = new(caBundleVar) + +func (c *caBundleVar) String() string { + return string(*c) +} + +func (c *caBundleVar) Set(path string) error { + pem, err := ioutil.ReadFile(path) + if err != nil { + return fmt.Errorf("could not read CA bundle path: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return fmt.Errorf("failed to load any CA certificates from %q", path) + } + if len(*c) == 0 { + *c = pem + return nil + } + *c = bytes.Join([][]byte{*c, pem}, []byte("\n")) + return nil +} + +func (c *caBundleVar) Type() string { + return "path" +} diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go index 8c0faeda..38295066 100644 --- a/cmd/pinniped/cmd/flag_types_test.go +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -4,11 +4,19 @@ package cmd import ( + "bytes" + "crypto/x509/pkix" + "fmt" + "io/ioutil" + "path/filepath" "testing" + "time" "github.com/stretchr/testify/require" configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/testutil" ) func TestConciergeModeFlag(t *testing.T) { @@ -41,3 +49,26 @@ func TestConciergeModeFlag(t *testing.T) { require.Equal(t, modeImpersonationProxy, m) require.Equal(t, "ImpersonationProxy", m.String()) } + +func TestCABundleFlag(t *testing.T) { + testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + require.NoError(t, err) + tmpdir := testutil.TempDir(t) + emptyFilePath := filepath.Join(tmpdir, "empty") + require.NoError(t, ioutil.WriteFile(emptyFilePath, []byte{}, 0600)) + + testCAPath := filepath.Join(tmpdir, "testca.pem") + require.NoError(t, ioutil.WriteFile(testCAPath, testCA.Bundle(), 0600)) + + c := caBundleVar{} + require.Equal(t, "path", c.Type()) + require.Equal(t, "", c.String()) + require.EqualError(t, c.Set("./does/not/exist"), "could not read CA bundle path: open ./does/not/exist: no such file or directory") + require.EqualError(t, c.Set(emptyFilePath), fmt.Sprintf("failed to load any CA certificates from %q", emptyFilePath)) + + require.NoError(t, c.Set(testCAPath)) + require.Equal(t, 1, bytes.Count(c, []byte("BEGIN CERTIFICATE"))) + + require.NoError(t, c.Set(testCAPath)) + require.Equal(t, 2, bytes.Count(c, []byte("BEGIN CERTIFICATE"))) +} diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 968e15d7..7ce12900 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -4,14 +4,12 @@ package cmd import ( - "bytes" "context" "crypto/tls" "crypto/x509" "encoding/base64" "fmt" "io" - "io/ioutil" "log" "net/http" "os" @@ -76,7 +74,7 @@ type getKubeconfigOIDCParams struct { skipBrowser bool sessionCachePath string debugSessionCache bool - caBundlePaths []string + caBundle caBundleVar requestAudience string } @@ -86,7 +84,7 @@ type getKubeconfigConciergeParams struct { authenticatorName string authenticatorType string apiGroupSuffix string - caBundlePath string + caBundle caBundleVar endpoint string mode conciergeMode } @@ -126,7 +124,7 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") - f.StringVar(&flags.concierge.caBundlePath, "concierge-ca-bundle", "", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge") + f.Var(&flags.concierge.caBundle, "concierge-ca-bundle", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge") f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") f.Var(&flags.concierge.mode, "concierge-mode", "Concierge mode of operation") @@ -136,7 +134,7 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringSliceVar(&flags.oidc.scopes, "oidc-scopes", []string{oidc.ScopeOfflineAccess, oidc.ScopeOpenID, "pinniped:request-audience"}, "OpenID Connect scopes to request during login") f.BoolVar(&flags.oidc.skipBrowser, "oidc-skip-browser", false, "During OpenID Connect login, skip opening the browser (just print the URL)") f.StringVar(&flags.oidc.sessionCachePath, "oidc-session-cache", "", "Path to OpenID Connect session cache file") - f.StringSliceVar(&flags.oidc.caBundlePaths, "oidc-ca-bundle", nil, "Path to TLS certificate authority bundle (PEM format, optional, can be repeated)") + f.Var(&flags.oidc.caBundle, "oidc-ca-bundle", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated)") f.BoolVar(&flags.oidc.debugSessionCache, "oidc-debug-session-cache", false, "Print debug logs related to the OpenID Connect session cache") f.StringVar(&flags.oidc.requestAudience, "oidc-request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange") f.StringVar(&flags.kubeconfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig file") @@ -187,11 +185,6 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f } execConfig.ProvideClusterInfo = true - oidcCABundle, err := loadCABundlePaths(flags.oidc.caBundlePaths) - if err != nil { - return fmt.Errorf("could not read --oidc-ca-bundle: %w", err) - } - clientConfig := newClientConfig(flags.kubeconfigPath, flags.kubeconfigContextOverride) currentKubeConfig, err := clientConfig.RawConfig() if err != nil { @@ -221,10 +214,26 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f if err != nil { return err } - - if err := configureConcierge(credentialIssuer, authenticator, &flags, cluster, &oidcCABundle, &execConfig, deps.log); err != nil { + if err := discoverConciergeParams(credentialIssuer, &flags, cluster, deps.log); err != nil { return err } + if err := discoverAuthenticatorParams(authenticator, &flags, deps.log); err != nil { + return err + } + // Append the flags to configure the Concierge credential exchange at runtime. + execConfig.Args = append(execConfig.Args, + "--enable-concierge", + "--concierge-api-group-suffix="+flags.concierge.apiGroupSuffix, + "--concierge-authenticator-name="+flags.concierge.authenticatorName, + "--concierge-authenticator-type="+flags.concierge.authenticatorType, + "--concierge-endpoint="+flags.concierge.endpoint, + "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(flags.concierge.caBundle), + "--concierge-mode="+flags.concierge.mode.String(), + ) + + // Point kubectl at the concierge endpoint. + cluster.Server = flags.concierge.endpoint + cluster.CertificateAuthorityData = flags.concierge.caBundle } // If one of the --static-* flags was passed, output a config that runs `pinniped login static`. @@ -263,8 +272,8 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f if flags.oidc.listenPort != 0 { execConfig.Args = append(execConfig.Args, "--listen-port="+strconv.Itoa(int(flags.oidc.listenPort))) } - if oidcCABundle != "" { - execConfig.Args = append(execConfig.Args, "--ca-bundle-data="+base64.StdEncoding.EncodeToString([]byte(oidcCABundle))) + if len(flags.oidc.caBundle) != 0 { + execConfig.Args = append(execConfig.Args, "--ca-bundle-data="+base64.StdEncoding.EncodeToString(flags.oidc.caBundle)) } if flags.oidc.sessionCachePath != "" { execConfig.Args = append(execConfig.Args, "--session-cache="+flags.oidc.sessionCachePath) @@ -282,7 +291,7 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f return writeConfigAsYAML(out, kubeconfig) } -func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig, log logr.Logger) error { +func discoverConciergeParams(credentialIssuer *configv1alpha1.CredentialIssuer, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, log logr.Logger) error { // Autodiscover the --concierge-mode. frontend, err := getConciergeFrontend(credentialIssuer, flags.concierge.mode) if err != nil { @@ -312,29 +321,24 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe log.Info("discovered Concierge endpoint", "endpoint", flags.concierge.endpoint) } - // Load specified --concierge-ca-bundle or autodiscover a value. - var conciergeCABundleData []byte - if flags.concierge.caBundlePath != "" { - caBundleString, err := loadCABundlePaths([]string{flags.concierge.caBundlePath}) - if err != nil { - return fmt.Errorf("could not read --concierge-ca-bundle: %w", err) - } - conciergeCABundleData = []byte(caBundleString) - log.Info("loaded Concierge certificate authority bundle", "roots", countCACerts(conciergeCABundleData)) - } else { + // Auto-set --concierge-ca-bundle if it wasn't explicitly set.. + if len(flags.concierge.caBundle) == 0 { switch frontend.Type { case configv1alpha1.TokenCredentialRequestAPIFrontendType: - conciergeCABundleData = v1Cluster.CertificateAuthorityData + flags.concierge.caBundle = v1Cluster.CertificateAuthorityData case configv1alpha1.ImpersonationProxyFrontendType: - var err error - conciergeCABundleData, err = base64.StdEncoding.DecodeString(frontend.ImpersonationProxyInfo.CertificateAuthorityData) + data, err := base64.StdEncoding.DecodeString(frontend.ImpersonationProxyInfo.CertificateAuthorityData) if err != nil { return fmt.Errorf("autodiscovered Concierge CA bundle is invalid: %w", err) } + flags.concierge.caBundle = data } - log.Info("discovered Concierge certificate authority bundle", "roots", countCACerts(conciergeCABundleData)) + log.Info("discovered Concierge certificate authority bundle", "roots", countCACerts(flags.concierge.caBundle)) } + return nil +} +func discoverAuthenticatorParams(authenticator metav1.Object, flags *getKubeconfigParams, log logr.Logger) error { switch auth := authenticator.(type) { case *conciergev1alpha1.WebhookAuthenticator: // If the --concierge-authenticator-type/--concierge-authenticator-name flags were not set explicitly, set @@ -367,30 +371,15 @@ func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authe // If the --oidc-ca-bundle flags was not set explicitly, default it to the // spec.tls.certificateAuthorityData field of the JWTAuthenticator. - if *oidcCABundle == "" && auth.Spec.TLS != nil && auth.Spec.TLS.CertificateAuthorityData != "" { + if len(flags.oidc.caBundle) == 0 && auth.Spec.TLS != nil && auth.Spec.TLS.CertificateAuthorityData != "" { decoded, err := base64.StdEncoding.DecodeString(auth.Spec.TLS.CertificateAuthorityData) if err != nil { return fmt.Errorf("tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator %s has invalid spec.tls.certificateAuthorityData: %w", auth.Name, err) } log.Info("discovered OIDC CA bundle", "roots", countCACerts(decoded)) - *oidcCABundle = string(decoded) + flags.oidc.caBundle = decoded } } - - // Append the flags to configure the Concierge credential exchange at runtime. - execConfig.Args = append(execConfig.Args, - "--enable-concierge", - "--concierge-api-group-suffix="+flags.concierge.apiGroupSuffix, - "--concierge-authenticator-name="+flags.concierge.authenticatorName, - "--concierge-authenticator-type="+flags.concierge.authenticatorType, - "--concierge-endpoint="+flags.concierge.endpoint, - "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(conciergeCABundleData), - "--concierge-mode="+flags.concierge.mode.String(), - ) - - // Point kubectl at the concierge endpoint. - v1Cluster.Server = flags.concierge.endpoint - v1Cluster.CertificateAuthorityData = conciergeCABundleData return nil } @@ -437,21 +426,6 @@ func getConciergeFrontend(credentialIssuer *configv1alpha1.CredentialIssuer, mod return nil, fmt.Errorf("could not find successful Concierge strategy matching --concierge-mode=%s", mode.String()) } -func loadCABundlePaths(paths []string) (string, error) { - if len(paths) == 0 { - return "", nil - } - blobs := make([][]byte, 0, len(paths)) - for _, p := range paths { - pem, err := ioutil.ReadFile(p) - if err != nil { - return "", err - } - blobs = append(blobs, pem) - } - return string(bytes.Join(blobs, []byte("\n"))), nil -} - func newExecKubeconfig(cluster *clientcmdapi.Cluster, execConfig *clientcmdapi.ExecConfig) clientcmdapi.Config { const name = "pinniped" return clientcmdapi.Config{ diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 34c78ce3..3e84ca44 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -69,7 +69,7 @@ func TestGetKubeconfig(t *testing.T) { --concierge-api-group-suffix string Concierge API group suffix (default "pinniped.dev") --concierge-authenticator-name string Concierge authenticator name (default: autodiscover) --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) - --concierge-ca-bundle string Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge + --concierge-ca-bundle path Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge --concierge-credential-issuer string Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover) --concierge-endpoint string API base for the Concierge endpoint --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) @@ -77,7 +77,7 @@ func TestGetKubeconfig(t *testing.T) { --kubeconfig string Path to kubeconfig file --kubeconfig-context string Kubeconfig context name (default: current active context) --no-concierge Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly - --oidc-ca-bundle strings Path to TLS certificate authority bundle (PEM format, optional, can be repeated) + --oidc-ca-bundle path Path to TLS certificate authority bundle (PEM format, optional, can be repeated) --oidc-client-id string OpenID Connect client ID (default: autodiscover) (default "pinniped-cli") --oidc-issuer string OpenID Connect issuer URL (default: autodiscover) --oidc-listen-port uint16 TCP port for localhost listener (authorization code flow only) @@ -102,13 +102,24 @@ func TestGetKubeconfig(t *testing.T) { `), }, { - name: "invalid CA bundle paths", + name: "invalid OIDC CA bundle path", args: []string{ "--oidc-ca-bundle", "./does/not/exist", }, wantError: true, wantStderr: here.Doc(` - Error: could not read --oidc-ca-bundle: open ./does/not/exist: no such file or directory + Error: invalid argument "./does/not/exist" for "--oidc-ca-bundle" flag: could not read CA bundle path: open ./does/not/exist: no such file or directory + `), + }, + { + name: "invalid Concierge CA bundle", + args: []string{ + "--kubeconfig", "./testdata/kubeconfig.yaml", + "--concierge-ca-bundle", "./does/not/exist", + }, + wantError: true, + wantStderr: here.Doc(` + Error: invalid argument "./does/not/exist" for "--concierge-ca-bundle" flag: could not read CA bundle path: open ./does/not/exist: no such file or directory `), }, { @@ -473,44 +484,6 @@ func TestGetKubeconfig(t *testing.T) { Error: tried to autodiscover --oidc-ca-bundle, but JWTAuthenticator test-authenticator has invalid spec.tls.certificateAuthorityData: illegal base64 data at input byte 7 `), }, - { - name: "invalid concierge ca bundle", - args: []string{ - "--kubeconfig", "./testdata/kubeconfig.yaml", - "--concierge-ca-bundle", "./does/not/exist", - "--concierge-endpoint", "https://impersonation-proxy-endpoint.test", - "--concierge-authenticator-name", "test-authenticator", - "--concierge-authenticator-type", "webhook", - "--concierge-mode", "ImpersonationProxy", - }, - conciergeObjects: []runtime.Object{ - &configv1alpha1.CredentialIssuer{ - ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{{ - Type: configv1alpha1.ImpersonationProxyStrategyType, - Status: configv1alpha1.SuccessStrategyStatus, - Reason: configv1alpha1.ListeningStrategyReason, - Frontend: &configv1alpha1.CredentialIssuerFrontend{ - Type: configv1alpha1.ImpersonationProxyFrontendType, - ImpersonationProxyInfo: &configv1alpha1.ImpersonationProxyInfo{ - Endpoint: "https://impersonation-proxy-endpoint.example.com", - CertificateAuthorityData: base64.StdEncoding.EncodeToString(testConciergeCA.Bundle()), - }, - }, - }}, - }, - }, - &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, - }, - wantLogs: []string{ - `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, - }, - wantError: true, - wantStderr: here.Doc(` - Error: could not read --concierge-ca-bundle: open ./does/not/exist: no such file or directory - `), - }, { name: "invalid static token flags", args: []string{ @@ -827,9 +800,7 @@ func TestGetKubeconfig(t *testing.T) { ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}, }, }, - wantLogs: []string{ - `"level"=0 "msg"="loaded Concierge certificate authority bundle" "roots"=1`, - }, + wantLogs: nil, wantStdout: here.Docf(` apiVersion: v1 clusters: From 6efbd81f75dcb56e7d6d45889254ef0cc9f94ddf Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 8 Mar 2021 14:33:38 -0600 Subject: [PATCH 110/203] Rename this flag types for consistency. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/flag_types.go | 44 +++++++++---------- cmd/pinniped/cmd/flag_types_test.go | 66 ++++++++++++++--------------- cmd/pinniped/cmd/kubeconfig.go | 8 ++-- cmd/pinniped/cmd/login_oidc.go | 2 +- cmd/pinniped/cmd/login_static.go | 2 +- 5 files changed, 61 insertions(+), 61 deletions(-) diff --git a/cmd/pinniped/cmd/flag_types.go b/cmd/pinniped/cmd/flag_types.go index 7b10e3a5..2bc55e06 100644 --- a/cmd/pinniped/cmd/flag_types.go +++ b/cmd/pinniped/cmd/flag_types.go @@ -16,20 +16,20 @@ import ( configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" ) -// conciergeMode represents the method by which we should connect to the Concierge on a cluster during login. +// conciergeModeFlag represents the method by which we should connect to the Concierge on a cluster during login. // this is meant to be a valid flag.Value implementation. -type conciergeMode int +type conciergeModeFlag int -var _ flag.Value = new(conciergeMode) +var _ flag.Value = new(conciergeModeFlag) const ( - modeUnknown conciergeMode = iota + modeUnknown conciergeModeFlag = iota modeTokenCredentialRequestAPI modeImpersonationProxy ) -func (c *conciergeMode) String() string { - switch *c { +func (f *conciergeModeFlag) String() string { + switch *f { case modeImpersonationProxy: return "ImpersonationProxy" case modeTokenCredentialRequestAPI: @@ -41,29 +41,29 @@ func (c *conciergeMode) String() string { } } -func (c *conciergeMode) Set(s string) error { +func (f *conciergeModeFlag) Set(s string) error { if strings.EqualFold(s, "") { - *c = modeUnknown + *f = modeUnknown return nil } if strings.EqualFold(s, "TokenCredentialRequestAPI") { - *c = modeTokenCredentialRequestAPI + *f = modeTokenCredentialRequestAPI return nil } if strings.EqualFold(s, "ImpersonationProxy") { - *c = modeImpersonationProxy + *f = modeImpersonationProxy return nil } return fmt.Errorf("invalid mode %q, valid modes are TokenCredentialRequestAPI and ImpersonationProxy", s) } -func (c *conciergeMode) Type() string { +func (f *conciergeModeFlag) Type() string { return "mode" } // MatchesFrontend returns true iff the flag matches the type of the provided frontend. -func (c *conciergeMode) MatchesFrontend(frontend *configv1alpha1.CredentialIssuerFrontend) bool { - switch *c { +func (f *conciergeModeFlag) MatchesFrontend(frontend *configv1alpha1.CredentialIssuerFrontend) bool { + switch *f { case modeImpersonationProxy: return frontend.Type == configv1alpha1.ImpersonationProxyFrontendType case modeTokenCredentialRequestAPI: @@ -76,15 +76,15 @@ func (c *conciergeMode) MatchesFrontend(frontend *configv1alpha1.CredentialIssue } // caBundlePathsVar represents a list of CA bundle paths, which load from disk when the flag is populated. -type caBundleVar []byte +type caBundleFlag []byte -var _ pflag.Value = new(caBundleVar) +var _ pflag.Value = new(caBundleFlag) -func (c *caBundleVar) String() string { - return string(*c) +func (f *caBundleFlag) String() string { + return string(*f) } -func (c *caBundleVar) Set(path string) error { +func (f *caBundleFlag) Set(path string) error { pem, err := ioutil.ReadFile(path) if err != nil { return fmt.Errorf("could not read CA bundle path: %w", err) @@ -93,14 +93,14 @@ func (c *caBundleVar) Set(path string) error { if !pool.AppendCertsFromPEM(pem) { return fmt.Errorf("failed to load any CA certificates from %q", path) } - if len(*c) == 0 { - *c = pem + if len(*f) == 0 { + *f = pem return nil } - *c = bytes.Join([][]byte{*c, pem}, []byte("\n")) + *f = bytes.Join([][]byte{*f, pem}, []byte("\n")) return nil } -func (c *caBundleVar) Type() string { +func (f *caBundleFlag) Type() string { return "path" } diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go index 38295066..6d967969 100644 --- a/cmd/pinniped/cmd/flag_types_test.go +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -20,34 +20,34 @@ import ( ) func TestConciergeModeFlag(t *testing.T) { - var m conciergeMode - require.Equal(t, "mode", m.Type()) - require.Equal(t, modeUnknown, m) - require.NoError(t, m.Set("")) - require.Equal(t, modeUnknown, m) - require.EqualError(t, m.Set("foo"), `invalid mode "foo", valid modes are TokenCredentialRequestAPI and ImpersonationProxy`) - require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) - require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) + var f conciergeModeFlag + require.Equal(t, "mode", f.Type()) + require.Equal(t, modeUnknown, f) + require.NoError(t, f.Set("")) + require.Equal(t, modeUnknown, f) + require.EqualError(t, f.Set("foo"), `invalid mode "foo", valid modes are TokenCredentialRequestAPI and ImpersonationProxy`) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) - require.NoError(t, m.Set("TokenCredentialRequestAPI")) - require.Equal(t, modeTokenCredentialRequestAPI, m) - require.Equal(t, "TokenCredentialRequestAPI", m.String()) - require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) - require.False(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) + require.NoError(t, f.Set("TokenCredentialRequestAPI")) + require.Equal(t, modeTokenCredentialRequestAPI, f) + require.Equal(t, "TokenCredentialRequestAPI", f.String()) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.False(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) - require.NoError(t, m.Set("tokencredentialrequestapi")) - require.Equal(t, modeTokenCredentialRequestAPI, m) - require.Equal(t, "TokenCredentialRequestAPI", m.String()) + require.NoError(t, f.Set("tokencredentialrequestapi")) + require.Equal(t, modeTokenCredentialRequestAPI, f) + require.Equal(t, "TokenCredentialRequestAPI", f.String()) - require.NoError(t, m.Set("ImpersonationProxy")) - require.Equal(t, modeImpersonationProxy, m) - require.Equal(t, "ImpersonationProxy", m.String()) - require.False(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) - require.True(t, m.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) + require.NoError(t, f.Set("ImpersonationProxy")) + require.Equal(t, modeImpersonationProxy, f) + require.Equal(t, "ImpersonationProxy", f.String()) + require.False(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.TokenCredentialRequestAPIFrontendType})) + require.True(t, f.MatchesFrontend(&configv1alpha1.CredentialIssuerFrontend{Type: configv1alpha1.ImpersonationProxyFrontendType})) - require.NoError(t, m.Set("impersonationproxy")) - require.Equal(t, modeImpersonationProxy, m) - require.Equal(t, "ImpersonationProxy", m.String()) + require.NoError(t, f.Set("impersonationproxy")) + require.Equal(t, modeImpersonationProxy, f) + require.Equal(t, "ImpersonationProxy", f.String()) } func TestCABundleFlag(t *testing.T) { @@ -60,15 +60,15 @@ func TestCABundleFlag(t *testing.T) { testCAPath := filepath.Join(tmpdir, "testca.pem") require.NoError(t, ioutil.WriteFile(testCAPath, testCA.Bundle(), 0600)) - c := caBundleVar{} - require.Equal(t, "path", c.Type()) - require.Equal(t, "", c.String()) - require.EqualError(t, c.Set("./does/not/exist"), "could not read CA bundle path: open ./does/not/exist: no such file or directory") - require.EqualError(t, c.Set(emptyFilePath), fmt.Sprintf("failed to load any CA certificates from %q", emptyFilePath)) + f := caBundleFlag{} + require.Equal(t, "path", f.Type()) + require.Equal(t, "", f.String()) + require.EqualError(t, f.Set("./does/not/exist"), "could not read CA bundle path: open ./does/not/exist: no such file or directory") + require.EqualError(t, f.Set(emptyFilePath), fmt.Sprintf("failed to load any CA certificates from %q", emptyFilePath)) - require.NoError(t, c.Set(testCAPath)) - require.Equal(t, 1, bytes.Count(c, []byte("BEGIN CERTIFICATE"))) + require.NoError(t, f.Set(testCAPath)) + require.Equal(t, 1, bytes.Count(f, []byte("BEGIN CERTIFICATE"))) - require.NoError(t, c.Set(testCAPath)) - require.Equal(t, 2, bytes.Count(c, []byte("BEGIN CERTIFICATE"))) + require.NoError(t, f.Set(testCAPath)) + require.Equal(t, 2, bytes.Count(f, []byte("BEGIN CERTIFICATE"))) } diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 7ce12900..1017eb5b 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -74,7 +74,7 @@ type getKubeconfigOIDCParams struct { skipBrowser bool sessionCachePath string debugSessionCache bool - caBundle caBundleVar + caBundle caBundleFlag requestAudience string } @@ -84,9 +84,9 @@ type getKubeconfigConciergeParams struct { authenticatorName string authenticatorType string apiGroupSuffix string - caBundle caBundleVar + caBundle caBundleFlag endpoint string - mode conciergeMode + mode conciergeModeFlag } type getKubeconfigParams struct { @@ -383,7 +383,7 @@ func discoverAuthenticatorParams(authenticator metav1.Object, flags *getKubeconf return nil } -func getConciergeFrontend(credentialIssuer *configv1alpha1.CredentialIssuer, mode conciergeMode) (*configv1alpha1.CredentialIssuerFrontend, error) { +func getConciergeFrontend(credentialIssuer *configv1alpha1.CredentialIssuer, mode conciergeModeFlag) (*configv1alpha1.CredentialIssuerFrontend, error) { for _, strategy := range credentialIssuer.Status.Strategies { // Skip unhealthy strategies. if strategy.Status != configv1alpha1.SuccessStrategyStatus { diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index e1db5689..7dd29943 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -71,7 +71,7 @@ type oidcLoginFlags struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string - conciergeMode conciergeMode + conciergeMode conciergeModeFlag } func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 9141c4e6..c9942551 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -47,7 +47,7 @@ type staticLoginParams struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string - conciergeMode conciergeMode + conciergeMode conciergeModeFlag } func staticLoginCommand(deps staticLoginDeps) *cobra.Command { From 8fd6a71312dd673b057b33c3ff4b5dea07756e59 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 8 Mar 2021 14:36:29 -0600 Subject: [PATCH 111/203] Use simpler prefix matching for impersonation headers. Signed-off-by: Matt Moyer --- .../concierge/impersonator/impersonator.go | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 7c88ed67..bbbcda39 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -10,7 +10,6 @@ import ( "net/http" "net/http/httputil" "net/url" - "regexp" "strings" "time" @@ -28,8 +27,6 @@ import ( "go.pinniped.dev/internal/kubeclient" ) -var impersonateHeaderRegex = regexp.MustCompile("Impersonate-.*") - type proxy struct { cache *authncache.Cache jsonDecoder runtime.Decoder @@ -150,11 +147,10 @@ func (e *httpError) Error() string { return e.message } func ensureNoImpersonationHeaders(r *http.Request) error { for key := range r.Header { - if impersonateHeaderRegex.MatchString(key) { + if isImpersonationHeader(key) { return fmt.Errorf("%q header already exists", key) } } - return nil } @@ -163,7 +159,9 @@ type roundTripperFunc func(*http.Request) (*http.Response, error) func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header { - newHeaders := http.Header{} + // Copy over all headers except the Authorization header from the original request to the new request. + newHeaders := requestHeaders.Clone() + newHeaders.Del("Authorization") // Leverage client-go's impersonation RoundTripper to set impersonation headers for us in the new // request. The client-go RoundTripper not only sets all of the impersonation headers for us, but @@ -176,12 +174,8 @@ func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header Extra: userInfo.GetExtra(), } impersonateHeaderSpy := roundTripperFunc(func(r *http.Request) (*http.Response, error) { - newHeaders.Set(transport.ImpersonateUserHeader, r.Header.Get(transport.ImpersonateUserHeader)) - for _, groupHeaderValue := range r.Header.Values(transport.ImpersonateGroupHeader) { - newHeaders.Add(transport.ImpersonateGroupHeader, groupHeaderValue) - } for headerKey, headerValues := range r.Header { - if strings.HasPrefix(headerKey, transport.ImpersonateUserExtraHeaderPrefix) { + if isImpersonationHeader(headerKey) { for _, headerValue := range headerValues { newHeaders.Add(headerKey, headerValue) } @@ -192,19 +186,13 @@ func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header fakeReq, _ := http.NewRequestWithContext(context.Background(), "", "", nil) //nolint:bodyclose // We return a nil http.Response above, so there is nothing to close. _, _ = transport.NewImpersonatingRoundTripper(impersonateConfig, impersonateHeaderSpy).RoundTrip(fakeReq) - - // Copy over all headers except the Authorization header from the original request to the new request. - for key := range requestHeaders { - if key != "Authorization" { - for _, val := range requestHeaders.Values(key) { - newHeaders.Add(key, val) - } - } - } - return newHeaders } +func isImpersonationHeader(header string) bool { + return strings.HasPrefix(http.CanonicalHeaderKey(header), "Impersonate") +} + func extractToken(token string, jsonDecoder runtime.Decoder) (*login.TokenCredentialRequest, error) { tokenCredentialRequestJSON, err := base64.StdEncoding.DecodeString(token) if err != nil { From a58b460bcbb6a912e59212c4db7c1c905026514a Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 8 Mar 2021 15:03:34 -0600 Subject: [PATCH 112/203] Switch TestImpersonationProxy to get clients from library.NewKubeclient instead of directly from kubernetes.NewForConfig. Signed-off-by: Matt Moyer --- .../concierge_impersonation_proxy_test.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 1bdd9316..e18528bd 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -73,7 +73,7 @@ func TestImpersonationProxy(t *testing.T) { return &config } - impersonationProxyViaSquidClient := func(caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { + impersonationProxyViaSquidClient := func(caData []byte, doubleImpersonateUser string) kubernetes.Interface { t.Helper() kubeconfig := impersonationProxyRestConfig("https://"+proxyServiceEndpoint, caData, doubleImpersonateUser) kubeconfig.Proxy = func(req *http.Request) (*url.URL, error) { @@ -82,17 +82,13 @@ func TestImpersonationProxy(t *testing.T) { t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) return proxyURL, nil } - impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) - require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") - return impersonationProxyClient + return library.NewKubeclient(t, kubeconfig).Kubernetes } - impersonationProxyViaLoadBalancerClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) *kubernetes.Clientset { + impersonationProxyViaLoadBalancerClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { t.Helper() kubeconfig := impersonationProxyRestConfig(proxyURL, caData, doubleImpersonateUser) - impersonationProxyClient, err := kubernetes.NewForConfig(kubeconfig) - require.NoError(t, err, "unexpected failure from kubernetes.NewForConfig()") - return impersonationProxyClient + return library.NewKubeclient(t, kubeconfig).Kubernetes } oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName(env), metav1.GetOptions{}) @@ -162,7 +158,7 @@ func TestImpersonationProxy(t *testing.T) { // Create an impersonation proxy client with that CA data to use for the rest of this test. // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. - var impersonationProxyClient *kubernetes.Clientset + var impersonationProxyClient kubernetes.Interface if env.HasCapability(library.HasExternalLoadBalancerProvider) { impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyURL, impersonationProxyCACertPEM, "") } else { @@ -338,7 +334,7 @@ func TestImpersonationProxy(t *testing.T) { // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. - var doubleImpersonationClient *kubernetes.Clientset + var doubleImpersonationClient kubernetes.Interface if env.HasCapability(library.HasExternalLoadBalancerProvider) { doubleImpersonationClient = impersonationProxyViaLoadBalancerClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate") } else { From 29d5e4322074cb465627c79984f0b5227498ea28 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 9 Mar 2021 12:12:52 -0600 Subject: [PATCH 113/203] Fix minor typo in e2e_test.go. Signed-off-by: Matt Moyer --- test/integration/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 910eb573..dfa4e2a0 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -155,7 +155,7 @@ func TestE2EFullIntegration(t *testing.T) { "--oidc-ca-bundle", testCABundlePath, "--oidc-session-cache", sessionCachePath, ) - t.Logf("stderr output from 'pinnipedget kubeconfig':\n%s\n\n", stderr) + t.Logf("stderr output from 'pinniped get kubeconfig':\n%s\n\n", stderr) t.Logf("test kubeconfig:\n%s\n\n", kubeconfigYAML) restConfig := library.NewRestConfigFromKubeconfig(t, kubeconfigYAML) From d6a0dfa4976d5b064e4c2dd8df635aa0421cbbd6 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 9 Mar 2021 12:39:44 -0600 Subject: [PATCH 114/203] Add some debug logging when "pinniped get kubeconfig" fails to find a successful strategy. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 8 ++++++++ cmd/pinniped/cmd/kubeconfig_test.go | 15 +++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 1017eb5b..243bc30f 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -295,6 +295,14 @@ func discoverConciergeParams(credentialIssuer *configv1alpha1.CredentialIssuer, // Autodiscover the --concierge-mode. frontend, err := getConciergeFrontend(credentialIssuer, flags.concierge.mode) if err != nil { + for _, strategy := range credentialIssuer.Status.Strategies { + log.Info("found CredentialIssuer strategy", + "type", strategy.Type, + "status", strategy.Status, + "reason", strategy.Reason, + "message", strategy.Message, + ) + } return err } diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 3e84ca44..0c828038 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -324,16 +324,27 @@ func TestGetKubeconfig(t *testing.T) { `), }, { - name: "autodetect webhook authenticator, bad credential issuer with no status", + name: "autodetect webhook authenticator, bad credential issuer with only failing strategy", args: []string{ "--kubeconfig", "./testdata/kubeconfig.yaml", }, conciergeObjects: []runtime.Object{ - &configv1alpha1.CredentialIssuer{ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}}, + &configv1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: "test-credential-issuer"}, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{{ + Type: "SomeType", + Status: configv1alpha1.ErrorStrategyStatus, + Reason: "SomeReason", + Message: "Some message", + }}, + }, + }, &conciergev1alpha1.WebhookAuthenticator{ObjectMeta: metav1.ObjectMeta{Name: "test-authenticator"}}, }, wantLogs: []string{ `"level"=0 "msg"="discovered CredentialIssuer" "name"="test-credential-issuer"`, + `"level"=0 "msg"="found CredentialIssuer strategy" "message"="Some message" "reason"="SomeReason" "status"="Error" "type"="SomeType"`, }, wantError: true, wantStderr: here.Doc(` From 883b90923d74694c734e123b38fc80efc08df749 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 9 Mar 2021 11:32:27 -0800 Subject: [PATCH 115/203] Add integration test for kubectl port-forward with impersonation --- .../concierge_impersonation_proxy_test.go | 58 ++++++++++++++++--- 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index e18528bd..7409be90 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -403,10 +403,8 @@ func TestImpersonationProxy(t *testing.T) { kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) - // Func to run kubeconfig commands. - kubectl := func(args ...string) (string, string, error) { - timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() + // func to create kubectl commands with a kubeconfig + kubectlCommand := func(timeout context.Context, args ...string) (*exec.Cmd, *bytes.Buffer, *bytes.Buffer) { allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) //nolint:gosec // we are not performing malicious argument injection against ourselves @@ -417,6 +415,15 @@ func TestImpersonationProxy(t *testing.T) { kubectlCmd.Env = envVarsWithProxy t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " ")) + return kubectlCmd, &stdout, &stderr + } + // Func to run kubeconfig commands. + runKubectl := func(args ...string) (string, string, error) { + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + + kubectlCmd, stdout, stderr := kubectlCommand(timeout, args...) + err := kubectlCmd.Run() t.Logf("kubectl stdout output: %s", stdout.String()) t.Logf("kubectl stderr output: %s", stderr.String()) @@ -424,17 +431,52 @@ func TestImpersonationProxy(t *testing.T) { } // Get pods in concierge namespace and pick one. - // We don't actually care which pod, just want to see that we can "exec echo" in one of them. + // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) require.NoError(t, err) require.Greater(t, len(pods.Items), 0) - podName := pods.Items[0].Name + var podName string + for _, pod := range pods.Items { + if !strings.Contains(pod.Name, "kube-cert-agent") { + podName = pod.Name + } + } + if podName == "" { + t.Error("could not find a concierge pod") + } // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" - stdout, _, err := kubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "echo", echoString) + stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "echo", echoString) require.NoError(t, err) - require.Equal(t, stdout, echoString+"\n") + require.Equal(t, echoString+"\n", stdout) + + // run the kubectl port-forward command + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + portForwardCmd, _, _ := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, podName, "443:8443") + portForwardCmd.Env = envVarsWithProxy + + // start, but don't wait for the command to finish + err = portForwardCmd.Start() + require.NoError(t, err) + + // then run curl something against it + time.Sleep(time.Second) + timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1") + var curlStdOut, curlStdErr bytes.Buffer + curlCmd.Stdout = &curlStdOut + curlCmd.Stderr = &curlStdErr + err = curlCmd.Run() + if err != nil { + t.Log("curl error: " + err.Error()) + t.Log("curlStdErr: " + curlStdErr.String()) + t.Log("stdout: " + curlStdOut.String()) + } + // we expect this to 403, but all we care is that it gets through + require.Contains(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") }) // Update configuration to force the proxy to disabled mode From 0abe10e6b281156151ffeec2c88c6dcdfcc042cf Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 9 Mar 2021 14:48:16 -0600 Subject: [PATCH 116/203] Add new behavior to "pinniped get kubeconfig" to wait for pending strategies to become non-pending. This behavior can be disabled with "--concierge-skip-wait". Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 38 +++++++++++++++++++++++++++++ cmd/pinniped/cmd/kubeconfig_test.go | 1 + 2 files changed, 39 insertions(+) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 243bc30f..1f59a970 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -87,6 +87,7 @@ type getKubeconfigConciergeParams struct { caBundle caBundleFlag endpoint string mode conciergeModeFlag + skipWait bool } type getKubeconfigParams struct { @@ -123,6 +124,7 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command { f.StringVar(&flags.concierge.authenticatorType, "concierge-authenticator-type", "", "Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover)") f.StringVar(&flags.concierge.authenticatorName, "concierge-authenticator-name", "", "Concierge authenticator name (default: autodiscover)") f.StringVar(&flags.concierge.apiGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") + f.BoolVar(&flags.concierge.skipWait, "concierge-skip-wait", false, "Skip waiting for any pending Concierge strategies to become ready (default: false)") f.Var(&flags.concierge.caBundle, "concierge-ca-bundle", "Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge") f.StringVar(&flags.concierge.endpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") @@ -205,6 +207,33 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f return err } + if !flags.concierge.skipWait { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + deadline, _ := ctx.Deadline() + attempts := 1 + + for { + if !hasPendingStrategy(credentialIssuer) { + break + } + deps.log.Info("waiting for CredentialIssuer pending strategies to finish", + "attempts", attempts, + "remaining", time.Until(deadline).Round(time.Second).String(), + ) + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + credentialIssuer, err = lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) + if err != nil { + return err + } + } + } + } + authenticator, err := lookupAuthenticator( clientset, flags.concierge.authenticatorType, @@ -636,3 +665,12 @@ func countCACerts(pemData []byte) int { pool.AppendCertsFromPEM(pemData) return len(pool.Subjects()) } + +func hasPendingStrategy(credentialIssuer *configv1alpha1.CredentialIssuer) bool { + for _, strategy := range credentialIssuer.Status.Strategies { + if strategy.Reason == configv1alpha1.PendingStrategyReason { + return true + } + } + return false +} diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 0c828038..53b83830 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -73,6 +73,7 @@ func TestGetKubeconfig(t *testing.T) { --concierge-credential-issuer string Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover) --concierge-endpoint string API base for the Concierge endpoint --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) + --concierge-skip-wait Skip waiting for any pending Concierge strategies to become ready (default: false) -h, --help help for kubeconfig --kubeconfig string Path to kubeconfig file --kubeconfig-context string Kubeconfig context name (default: current active context) From 0cb1538b39502d965a6061dfd79b91f3d667d2ee Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 9 Mar 2021 15:16:46 -0600 Subject: [PATCH 117/203] Fix linter warnings, including a bit of refactoring. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 64 +++++++++++-------- .../concierge_impersonation_proxy_test.go | 1 - 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 1f59a970..1e342226 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -202,38 +202,11 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f } if !flags.concierge.disabled { - credentialIssuer, err := lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) + credentialIssuer, err := waitForCredentialIssuer(ctx, clientset, flags, deps) if err != nil { return err } - if !flags.concierge.skipWait { - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - - deadline, _ := ctx.Deadline() - attempts := 1 - - for { - if !hasPendingStrategy(credentialIssuer) { - break - } - deps.log.Info("waiting for CredentialIssuer pending strategies to finish", - "attempts", attempts, - "remaining", time.Until(deadline).Round(time.Second).String(), - ) - select { - case <-ctx.Done(): - return ctx.Err() - case <-ticker.C: - credentialIssuer, err = lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) - if err != nil { - return err - } - } - } - } - authenticator, err := lookupAuthenticator( clientset, flags.concierge.authenticatorType, @@ -320,6 +293,41 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f return writeConfigAsYAML(out, kubeconfig) } +func waitForCredentialIssuer(ctx context.Context, clientset conciergeclientset.Interface, flags getKubeconfigParams, deps kubeconfigDeps) (*configv1alpha1.CredentialIssuer, error) { + credentialIssuer, err := lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) + if err != nil { + return nil, err + } + + if !flags.concierge.skipWait { + ticker := time.NewTicker(2 * time.Second) + defer ticker.Stop() + + deadline, _ := ctx.Deadline() + attempts := 1 + + for { + if !hasPendingStrategy(credentialIssuer) { + break + } + deps.log.Info("waiting for CredentialIssuer pending strategies to finish", + "attempts", attempts, + "remaining", time.Until(deadline).Round(time.Second).String(), + ) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ticker.C: + credentialIssuer, err = lookupCredentialIssuer(clientset, flags.concierge.credentialIssuer, deps.log) + if err != nil { + return nil, err + } + } + } + } + return credentialIssuer, nil +} + func discoverConciergeParams(credentialIssuer *configv1alpha1.CredentialIssuer, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, log logr.Logger) error { // Autodiscover the --concierge-mode. frontend, err := getConciergeFrontend(credentialIssuer, flags.concierge.mode) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 7409be90..d9c2742d 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -405,7 +405,6 @@ func TestImpersonationProxy(t *testing.T) { // func to create kubectl commands with a kubeconfig kubectlCommand := func(timeout context.Context, args ...string) (*exec.Cmd, *bytes.Buffer, *bytes.Buffer) { - allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) //nolint:gosec // we are not performing malicious argument injection against ourselves kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) From 005133fbfb33f591ab36ad6c9ff036c5e5d82cd1 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 9 Mar 2021 16:56:53 -0600 Subject: [PATCH 118/203] Add more debug logging when waiting for pending strategies. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 1e342226..4d6f1da7 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -310,6 +310,7 @@ func waitForCredentialIssuer(ctx context.Context, clientset conciergeclientset.I if !hasPendingStrategy(credentialIssuer) { break } + logStrategies(credentialIssuer, deps.log) deps.log.Info("waiting for CredentialIssuer pending strategies to finish", "attempts", attempts, "remaining", time.Until(deadline).Round(time.Second).String(), @@ -332,14 +333,7 @@ func discoverConciergeParams(credentialIssuer *configv1alpha1.CredentialIssuer, // Autodiscover the --concierge-mode. frontend, err := getConciergeFrontend(credentialIssuer, flags.concierge.mode) if err != nil { - for _, strategy := range credentialIssuer.Status.Strategies { - log.Info("found CredentialIssuer strategy", - "type", strategy.Type, - "status", strategy.Status, - "reason", strategy.Reason, - "message", strategy.Message, - ) - } + logStrategies(credentialIssuer, log) return err } @@ -383,6 +377,17 @@ func discoverConciergeParams(credentialIssuer *configv1alpha1.CredentialIssuer, return nil } +func logStrategies(credentialIssuer *configv1alpha1.CredentialIssuer, log logr.Logger) { + for _, strategy := range credentialIssuer.Status.Strategies { + log.Info("found CredentialIssuer strategy", + "type", strategy.Type, + "status", strategy.Status, + "reason", strategy.Reason, + "message", strategy.Message, + ) + } +} + func discoverAuthenticatorParams(authenticator metav1.Object, flags *getKubeconfigParams, log logr.Logger) error { switch auth := authenticator.(type) { case *conciergev1alpha1.WebhookAuthenticator: From c853707889ef5b9e0c69d4fe003b1b88fa94e83a Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 9 Mar 2021 16:58:44 -0800 Subject: [PATCH 119/203] Added integration test for using websockets via the impersonation proxy Tested that this test passed when using the kube api server directly, so it's just the impersonation proxy that must be improved. --- go.mod | 1 + go.sum | 2 + .../concierge_impersonation_proxy_test.go | 83 ++++++++++++++++--- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index a1c0e7e2..0d15fb2a 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect diff --git a/go.sum b/go.sum index 041d4e99..33664938 100644 --- a/go.sum +++ b/go.sum @@ -1178,6 +1178,8 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181003184128-c57b0facaced/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index d9c2742d..06f0a55d 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -6,7 +6,10 @@ package integration import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/base64" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -18,6 +21,8 @@ import ( "testing" "time" + "golang.org/x/net/websocket" + "github.com/stretchr/testify/require" v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" @@ -26,6 +31,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -181,19 +187,20 @@ func TestImpersonationProxy(t *testing.T) { ) } + // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. + namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + // Schedule the namespace for cleanup. + t.Cleanup(func() { + t.Logf("cleaning up test namespace %s", namespace.Name) + err = adminClient.CoreV1().Namespaces().Delete(context.Background(), namespace.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + }) + // Try more Kube API verbs through the impersonation proxy. t.Run("watching all the basic verbs", func(t *testing.T) { - // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. - namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, - }, metav1.CreateOptions{}) - require.NoError(t, err) - // Schedule the namespace for cleanup. - t.Cleanup(func() { - t.Logf("cleaning up test namespace %s", namespace.Name) - err = adminClient.CoreV1().Namespaces().Delete(context.Background(), namespace.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) // Create an RBAC rule to allow this user to read/write everything. library.CreateTestClusterRoleBinding(t, @@ -478,6 +485,54 @@ func TestImpersonationProxy(t *testing.T) { require.Contains(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") }) + t.Run("websocket client", func(t *testing.T) { + dest, _ := url.Parse(impersonationProxyURL) + dest.Scheme = "wss" + dest.Path = "/api/v1/namespaces/" + namespace.Name + "/configmaps" + dest.RawQuery = "watch=1&resourceVersion=0" + origin, _ := url.Parse("http://localhost") + + rootCAs := x509.NewCertPool() + rootCAs.AppendCertsFromPEM(impersonationProxyCACertPEM) + tlsConfig := &tls.Config{ + RootCAs: rootCAs, + } + + websocketConfig := websocket.Config{ + Location: dest, + Origin: origin, + TlsConfig: tlsConfig, + Version: 13, + Header: http.Header(make(map[string][]string)), + } + ws, err := websocket.DialConfig(&websocketConfig) + if err != nil { + t.Fatalf("failed to dial websocket: %v", err) + } + + // perform a create through the admin client + _, err = adminClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1"}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + + // see if the websocket client received an event for the create + var got watchJSON + err = websocket.JSON.Receive(ws, &got) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + if got.Type != watch.Added { + t.Errorf("Unexpected type: %v", got.Type) + } + + var createConfigMap corev1.ConfigMap + err = json.Unmarshal(got.Object, &createConfigMap) + require.NoError(t, err) + require.Equal(t, "configmap-1", createConfigMap.Name) + }) + // Update configuration to force the proxy to disabled mode configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { @@ -649,3 +704,9 @@ func impersonationProxyLoadBalancerName(env *library.TestEnv) string { func credentialIssuerName(env *library.TestEnv) string { return env.ConciergeAppName + "-config" } + +// watchJSON defines the expected JSON wire equivalent of watch.Event +type watchJSON struct { + Type watch.EventType `json:"type,omitempty"` + Object json.RawMessage `json:"object,omitempty"` +} From 0b300cbe4263293b6d1e7b207baef22f02919c58 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 10 Mar 2021 10:30:06 -0800 Subject: [PATCH 120/203] Use TokenCredentialRequest instead of base64 token with impersonator To make an impersonation request, first make a TokenCredentialRequest to get a certificate. That cert will either be issued by the Kube API server's CA or by a new CA specific to the impersonator. Either way, you can then make a request to the impersonator and present that client cert for auth and the impersonator will accept it and make the impesonation call on your behalf. The impersonator http handler now borrows some Kube library code to handle request processing. This will allow us to more closely mimic the behavior of a real API server, e.g. the client cert auth will work exactly like the real API server. Signed-off-by: Monis Khan --- deploy/concierge/deployment.yaml | 1 + go.mod | 2 +- internal/certauthority/certauthority.go | 11 +- .../dynamiccertauthority.go | 9 +- internal/concierge/apiserver/apiserver.go | 3 +- .../concierge/impersonator/impersonator.go | 380 ++++++----- .../impersonator/impersonator_test.go | 417 +++++------- internal/concierge/server/server.go | 54 +- internal/config/concierge/config.go | 3 + internal/config/concierge/config_test.go | 35 +- internal/config/concierge/types.go | 1 + .../controller/apicerts/apiservice_updater.go | 4 +- internal/controller/apicerts/certs_expirer.go | 18 +- .../controller/apicerts/certs_expirer_test.go | 22 +- internal/controller/apicerts/certs_manager.go | 52 +- .../controller/apicerts/certs_manager_test.go | 46 +- .../controller/apicerts/certs_observer.go | 4 +- .../impersonatorconfig/impersonator_config.go | 255 ++++--- .../impersonator_config_test.go | 633 ++++++++++++++---- .../controllermanager/prepare_controllers.go | 54 +- internal/dynamiccert/provider.go | 29 +- internal/issuer/issuer.go | 42 ++ internal/registry/credentialrequest/rest.go | 9 +- .../registry/credentialrequest/rest_test.go | 9 +- .../impersonationtoken/impersonationtoken.go | 65 -- .../concierge_credentialrequest_test.go | 51 +- .../concierge_impersonation_proxy_test.go | 151 +++-- test/library/credential_request.go | 27 + 28 files changed, 1486 insertions(+), 901 deletions(-) create mode 100644 internal/issuer/issuer.go delete mode 100644 internal/testutil/impersonationtoken/impersonationtoken.go create mode 100644 test/library/credential_request.go diff --git a/deploy/concierge/deployment.yaml b/deploy/concierge/deployment.yaml index 41119243..223b7fa8 100644 --- a/deploy/concierge/deployment.yaml +++ b/deploy/concierge/deployment.yaml @@ -46,6 +46,7 @@ data: impersonationLoadBalancerService: (@= defaultResourceNameWithSuffix("impersonation-proxy-load-balancer") @) impersonationTLSCertificateSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-tls-serving-certificate") @) impersonationCACertificateSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-ca-certificate") @) + impersonationSignerSecret: (@= defaultResourceNameWithSuffix("impersonation-proxy-signer-ca-certificate") @) labels: (@= json.encode(labels()).rstrip() @) kubeCertAgent: namePrefix: (@= defaultResourceNameWithSuffix("kube-cert-agent-") @) diff --git a/go.mod b/go.mod index 0d15fb2a..de809464 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index 64fa2ecb..806d6498 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -187,11 +187,12 @@ func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time. // Sign a cert, getting back the DER-encoded certificate bytes. template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: subject, - NotBefore: notBefore, - NotAfter: notAfter, - KeyUsage: x509.KeyUsageDigitalSignature, + SerialNumber: serialNumber, + Subject: subject, + NotBefore: notBefore, + NotAfter: notAfter, + KeyUsage: x509.KeyUsageDigitalSignature, + // TODO split this function into two funcs that handle client and serving certs differently ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, BasicConstraintsValid: true, IsCA: false, diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go index a2c97898..c15e4cc0 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 // Package dynamiccertauthority implements a x509 certificate authority capable of issuing @@ -9,18 +9,19 @@ import ( "crypto/x509/pkix" "time" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "go.pinniped.dev/internal/certauthority" - "go.pinniped.dev/internal/dynamiccert" ) // CA is a type capable of issuing certificates. type CA struct { - provider dynamiccert.Provider + provider dynamiccertificates.CertKeyContentProvider } // New creates a new CA, ready to issue certs whenever the provided provider has a keypair to // provide. -func New(provider dynamiccert.Provider) *CA { +func New(provider dynamiccertificates.CertKeyContentProvider) *CA { return &CA{ provider: provider, } diff --git a/internal/concierge/apiserver/apiserver.go b/internal/concierge/apiserver/apiserver.go index e5fc8da9..f64e0104 100644 --- a/internal/concierge/apiserver/apiserver.go +++ b/internal/concierge/apiserver/apiserver.go @@ -15,6 +15,7 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/client-go/pkg/version" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/plog" "go.pinniped.dev/internal/registry/credentialrequest" "go.pinniped.dev/internal/registry/whoamirequest" @@ -27,7 +28,7 @@ type Config struct { type ExtraConfig struct { Authenticator credentialrequest.TokenCredentialRequestAuthenticator - Issuer credentialrequest.CertIssuer + Issuer issuer.CertIssuer StartControllersPostStartHook func(ctx context.Context) Scheme *runtime.Scheme NegotiatedSerializer runtime.NegotiatedSerializer diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index bbbcda39..f943db80 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -4,58 +4,202 @@ package impersonator import ( - "context" - "encoding/base64" "fmt" + "net" "net/http" "net/http/httputil" "net/url" "strings" "time" - "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/errors" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authentication/request/bearertoken" - "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/endpoints/request" + genericapiserver "k8s.io/apiserver/pkg/server" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/rest" "k8s.io/client-go/transport" - "go.pinniped.dev/generated/latest/apis/concierge/login" "go.pinniped.dev/internal/constable" - "go.pinniped.dev/internal/controller/authenticator/authncache" + "go.pinniped.dev/internal/httputil/securityheader" "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/plog" ) -type proxy struct { - cache *authncache.Cache - jsonDecoder runtime.Decoder - proxy *httputil.ReverseProxy - log logr.Logger +// FactoryFunc is a function which can create an impersonator server. +// It returns a function which will start the impersonator server. +// That start function takes a stopCh which can be used to stop the server. +// Once a server has been stopped, don't start it again using the start function. +// Instead, call the factory function again to get a new start function. +type FactoryFunc func( + port int, + dynamicCertProvider dynamiccertificates.CertKeyContentProvider, + impersonationProxySignerCA dynamiccertificates.CAContentProvider, +) (func(stopCh <-chan struct{}) error, error) + +func New( + port int, + dynamicCertProvider dynamiccertificates.CertKeyContentProvider, // TODO: we need to check those optional interfaces and see what we need to implement + impersonationProxySignerCA dynamiccertificates.CAContentProvider, // TODO: we need to check those optional interfaces and see what we need to implement +) (func(stopCh <-chan struct{}) error, error) { + return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, nil, nil) } -func New(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr.Logger) (http.Handler, error) { - return newInternal(cache, jsonDecoder, log, func() (*rest.Config, error) { - client, err := kubeclient.New() +func newInternal( //nolint:funlen // yeah, it's kind of long. + port int, + dynamicCertProvider dynamiccertificates.CertKeyContentProvider, + impersonationProxySignerCA dynamiccertificates.CAContentProvider, + clientOpts []kubeclient.Option, // for unit testing, should always be nil in production + recOpts func(*genericoptions.RecommendedOptions), // for unit testing, should always be nil in production +) (func(stopCh <-chan struct{}) error, error) { + var listener net.Listener + + constructServer := func() (func(stopCh <-chan struct{}) error, error) { + // bare minimum server side scheme to allow for status messages to be encoded + scheme := runtime.NewScheme() + metav1.AddToGroupVersion(scheme, metav1.Unversioned) + codecs := serializer.NewCodecFactory(scheme) + + // this is unused for now but it is a safe value that we could use in the future + defaultEtcdPathPrefix := "/pinniped-impersonation-proxy-registry" + + recommendedOptions := genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + codecs.LegacyCodec(), + ) + recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet + recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider // serving certs (end user facing) + recommendedOptions.SecureServing.BindPort = port + + // wire up the impersonation proxy signer CA as a valid authenticator for client cert auth + // TODO fix comments + kubeClient, err := kubeclient.New(clientOpts...) if err != nil { return nil, err } - return client.JSONConfig, nil - }) -} + kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClient.Kubernetes) + if err != nil { + return nil, err + } + recommendedOptions.Authentication.ClientCert.ClientCA = "---irrelevant-but-needs-to-be-non-empty---" // drop when we pick up https://github.com/kubernetes/kubernetes/pull/100055 + recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider(impersonationProxySignerCA, kubeClientCA) -func newInternal(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr.Logger, getConfig func() (*rest.Config, error)) (*proxy, error) { - kubeconfig, err := getConfig() - if err != nil { - return nil, fmt.Errorf("could not get in-cluster config: %w", err) + if recOpts != nil { + recOpts(recommendedOptions) + } + + serverConfig := genericapiserver.NewRecommendedConfig(codecs) + + // Note that ApplyTo is going to create a network listener and bind to the requested port. + // It puts this listener into serverConfig.SecureServing.Listener. + err = recommendedOptions.ApplyTo(serverConfig) + if serverConfig.SecureServing != nil { + // Set the pointer from the outer function to allow the outer function to close the listener in case + // this function returns an error for any reason anywhere below here. + listener = serverConfig.SecureServing.Listener + } + if err != nil { + return nil, err + } + + // loopback authentication to this server does not really make sense since we just proxy everything to KAS + // thus we replace loopback connection config with one that does direct connections to KAS + // loopback config is mainly used by post start hooks, so this is mostly future proofing + serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClient.ProtoConfig) // assume proto is safe (hooks can override) + // remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken + // see sanity checks at the end of this function + serverConfig.LoopbackClientConfig.BearerToken = "" + + // assume proto config is safe because transport level configs do not use rest.ContentConfig + // thus if we are interacting with actual APIs, they should be using pre-built clients + impersonationProxy, err := newImpersonationReverseProxy(rest.CopyConfig(kubeClient.ProtoConfig)) + if err != nil { + return nil, err + } + + defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc + serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler { + // we ignore the passed in handler because we never have any REST APIs to delegate to + handler := defaultBuildHandlerChainFunc(impersonationProxy, c) + handler = securityheader.Wrap(handler) + return handler + } + + // TODO integration test this authorizer logic with system:masters + double impersonation + // overwrite the delegating authorizer with one that only cares about impersonation + // empty string is disallowed because request info has had bugs in the past where it would leave it empty + disallowedVerbs := sets.NewString("", "impersonate") + noImpersonationAuthorizer := &comparableAuthorizer{ + AuthorizerFunc: func(a authorizer.Attributes) (authorizer.Decision, string, error) { + // supporting impersonation is not hard, it would just require a bunch of testing + // and configuring the audit layer (to preserve the caller) which we can do later + // we would also want to delete the incoming impersonation headers + // instead of overwriting the delegating authorizer, we would + // actually use it to make the impersonation authorization checks + if disallowedVerbs.Has(a.GetVerb()) { + return authorizer.DecisionDeny, "impersonation is not allowed or invalid verb", nil + } + + return authorizer.DecisionAllow, "deferring authorization to kube API server", nil + }, + } + // TODO write a big comment explaining wth this is doing + serverConfig.Authorization.Authorizer = noImpersonationAuthorizer + + impersonationProxyServer, err := serverConfig.Complete().New("impersonation-proxy", genericapiserver.NewEmptyDelegate()) + if err != nil { + return nil, err + } + + preparedRun := impersonationProxyServer.PrepareRun() + + // wait until the very end to do sanity checks + + if preparedRun.Authorizer != noImpersonationAuthorizer { + return nil, constable.Error("invalid mutation of impersonation authorizer detected") + } + + // assert that we have a functioning token file to use and no bearer token + if len(preparedRun.LoopbackClientConfig.BearerToken) != 0 || len(preparedRun.LoopbackClientConfig.BearerTokenFile) == 0 { + return nil, constable.Error("invalid impersonator loopback rest config has wrong bearer token semantics") + } + + // TODO make sure this is closed on error + _ = preparedRun.SecureServingInfo.Listener + + return preparedRun.Run, nil } - serverURL, err := url.Parse(kubeconfig.Host) + result, err := constructServer() + // If there was any error during construction, then we would like to close the listener to free up the port. + if err != nil { + errs := []error{err} + if listener != nil { + errs = append(errs, listener.Close()) + } + return nil, errors.NewAggregate(errs) + } + return result, nil +} + +// No-op wrapping around AuthorizerFunc to allow for comparisons. +type comparableAuthorizer struct { + authorizer.AuthorizerFunc +} + +func newImpersonationReverseProxy(restConfig *rest.Config) (http.Handler, error) { + serverURL, err := url.Parse(restConfig.Host) if err != nil { return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err) } - kubeTransportConfig, err := kubeconfig.TransportConfig() + kubeTransportConfig, err := restConfig.TransportConfig() if err != nil { return nil, fmt.Errorf("could not get in-cluster transport config: %w", err) } @@ -66,148 +210,72 @@ func newInternal(cache *authncache.Cache, jsonDecoder runtime.Decoder, log logr. return nil, fmt.Errorf("could not get in-cluster transport: %w", err) } - reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) - reverseProxy.Transport = kubeRoundTripper - reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line - - return &proxy{ - cache: cache, - jsonDecoder: jsonDecoder, - proxy: reverseProxy, - log: log, - }, nil -} - -func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { - log := p.log.WithValues( - "url", r.URL.String(), - "method", r.Method, - ) - - if err := ensureNoImpersonationHeaders(r); err != nil { - log.Error(err, "impersonation header already exists") - http.Error(w, "impersonation header already exists", http.StatusBadRequest) - return - } - - // Never mutate request (see http.Handler docs). - newR := r.Clone(r.Context()) - - authentication, authenticated, err := bearertoken.New(authenticator.TokenFunc(func(ctx context.Context, token string) (*authenticator.Response, bool, error) { - tokenCredentialReq, err := extractToken(token, p.jsonDecoder) - if err != nil { - log.Error(err, "invalid token encoding") - return nil, false, &httpError{message: "invalid token encoding", code: http.StatusBadRequest} + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // TODO integration test using a bearer token + if len(r.Header.Values("Authorization")) != 0 { + plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "invalid authorization header", http.StatusInternalServerError) + return } - log = log.WithValues( - "authenticator", tokenCredentialReq.Spec.Authenticator, + if err := ensureNoImpersonationHeaders(r); err != nil { + plog.Error("noImpersonationAuthorizer logic did not prevent nested impersonation but it is always supposed to do so", + err, + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "invalid impersonation", http.StatusInternalServerError) + return + } + + userInfo, ok := request.UserFrom(r.Context()) + if !ok { + plog.Warning("aggregated API server logic did not set user info but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "invalid user", http.StatusInternalServerError) + return + } + + if len(userInfo.GetUID()) > 0 { + plog.Warning("rejecting request with UID since we cannot impersonate UIDs", + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "unexpected uid", http.StatusUnprocessableEntity) + return + } + + plog.Trace("proxying authenticated request", + "url", r.URL.String(), + "method", r.Method, + "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) - userInfo, err := p.cache.AuthenticateTokenCredentialRequest(newR.Context(), tokenCredentialReq) - if err != nil { - log.Error(err, "received invalid token") - return nil, false, &httpError{message: "invalid token", code: http.StatusUnauthorized} + reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) + impersonateConfig := transport.ImpersonationConfig{ + UserName: userInfo.GetName(), + Groups: userInfo.GetGroups(), + Extra: userInfo.GetExtra(), } - if userInfo == nil { - log.Info("received token that did not authenticate") - return nil, false, &httpError{message: "not authenticated", code: http.StatusUnauthorized} - } - log = log.WithValues("userID", userInfo.GetUID()) - - return &authenticator.Response{User: userInfo}, true, nil - })).AuthenticateRequest(newR) - if err != nil { - httpErr, ok := err.(*httpError) - if !ok { - log.Error(err, "unrecognized error") - http.Error(w, "unrecognized error", http.StatusInternalServerError) - } - http.Error(w, httpErr.message, httpErr.code) - return - } - if !authenticated { - log.Error(constable.Error("token authenticator did not find token"), "invalid token encoding") - http.Error(w, "invalid token encoding", http.StatusBadRequest) - return - } - - newR.Header = getProxyHeaders(authentication.User, r.Header) - - log.Info("proxying authenticated request") - p.proxy.ServeHTTP(w, newR) + reverseProxy.Transport = transport.NewImpersonatingRoundTripper(impersonateConfig, kubeRoundTripper) + reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line + // transport.NewImpersonatingRoundTripper clones the request before setting headers + // so this call will not accidentally mutate the input request (see http.Handler docs) + reverseProxy.ServeHTTP(w, r) + }), nil } -type httpError struct { - message string - code int -} - -func (e *httpError) Error() string { return e.message } - func ensureNoImpersonationHeaders(r *http.Request) error { for key := range r.Header { - if isImpersonationHeader(key) { + if strings.HasPrefix(key, "Impersonate") { return fmt.Errorf("%q header already exists", key) } } + return nil } - -type roundTripperFunc func(*http.Request) (*http.Response, error) - -func (f roundTripperFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } - -func getProxyHeaders(userInfo user.Info, requestHeaders http.Header) http.Header { - // Copy over all headers except the Authorization header from the original request to the new request. - newHeaders := requestHeaders.Clone() - newHeaders.Del("Authorization") - - // Leverage client-go's impersonation RoundTripper to set impersonation headers for us in the new - // request. The client-go RoundTripper not only sets all of the impersonation headers for us, but - // it also does some helpful escaping of characters that can't go into an HTTP header. To do this, - // we make a fake call to the impersonation RoundTripper with a fake HTTP request and a delegate - // RoundTripper that captures the impersonation headers set on the request. - impersonateConfig := transport.ImpersonationConfig{ - UserName: userInfo.GetName(), - Groups: userInfo.GetGroups(), - Extra: userInfo.GetExtra(), - } - impersonateHeaderSpy := roundTripperFunc(func(r *http.Request) (*http.Response, error) { - for headerKey, headerValues := range r.Header { - if isImpersonationHeader(headerKey) { - for _, headerValue := range headerValues { - newHeaders.Add(headerKey, headerValue) - } - } - } - return nil, nil - }) - fakeReq, _ := http.NewRequestWithContext(context.Background(), "", "", nil) - //nolint:bodyclose // We return a nil http.Response above, so there is nothing to close. - _, _ = transport.NewImpersonatingRoundTripper(impersonateConfig, impersonateHeaderSpy).RoundTrip(fakeReq) - return newHeaders -} - -func isImpersonationHeader(header string) bool { - return strings.HasPrefix(http.CanonicalHeaderKey(header), "Impersonate") -} - -func extractToken(token string, jsonDecoder runtime.Decoder) (*login.TokenCredentialRequest, error) { - tokenCredentialRequestJSON, err := base64.StdEncoding.DecodeString(token) - if err != nil { - return nil, fmt.Errorf("invalid base64 in encoded bearer token: %w", err) - } - - obj, err := runtime.Decode(jsonDecoder, tokenCredentialRequestJSON) - if err != nil { - return nil, fmt.Errorf("invalid object encoded in bearer token: %w", err) - } - - tokenCredentialRequest, ok := obj.(*login.TokenCredentialRequest) - if !ok { - return nil, fmt.Errorf("invalid TokenCredentialRequest encoded in bearer token: got %T", obj) - } - - return tokenCredentialRequest, nil -} diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 7a7124b2..31e9aae9 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -5,40 +5,125 @@ package impersonator import ( "context" - "fmt" + "crypto/x509/pkix" + "net" "net/http" "net/http/httptest" "net/url" + "strconv" "testing" + "time" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" + "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/features" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + genericoptions "k8s.io/apiserver/pkg/server/options" + utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd/api" + featuregatetesting "k8s.io/component-base/featuregate/testing" - authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" - "go.pinniped.dev/generated/latest/apis/concierge/login" - conciergescheme "go.pinniped.dev/internal/concierge/scheme" - "go.pinniped.dev/internal/controller/authenticator/authncache" - "go.pinniped.dev/internal/mocks/mocktokenauthenticator" + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" - "go.pinniped.dev/internal/testutil/impersonationtoken" - "go.pinniped.dev/internal/testutil/testlogger" ) -func TestImpersonator(t *testing.T) { - const ( - defaultAPIGroup = "pinniped.dev" - customAPIGroup = "walrus.tld" +func TestNew(t *testing.T) { + const port = 8444 - testUser = "test-user" - ) + ca, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) + require.NoError(t, err) + cert, key, err := ca.IssuePEM(pkix.Name{CommonName: "example.com"}, []string{"example.com"}, time.Hour) + require.NoError(t, err) + certKeyContent, err := dynamiccertificates.NewStaticCertKeyContent("cert-key", cert, key) + require.NoError(t, err) + caContent, err := dynamiccertificates.NewStaticCAContent("ca", ca.Bundle()) + require.NoError(t, err) + + // Punch out just enough stuff to make New actually run without error. + recOpts := func(options *genericoptions.RecommendedOptions) { + options.Authentication.RemoteKubeConfigFileOptional = true + options.Authorization.RemoteKubeConfigFileOptional = true + options.CoreAPI = nil + options.Admission = nil + } + defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIPriorityAndFairness, false)() + + tests := []struct { + name string + clientOpts []kubeclient.Option + wantErr string + }{ + { + name: "happy path", + clientOpts: []kubeclient.Option{ + kubeclient.WithConfig(&rest.Config{ + BearerToken: "should-be-ignored", + BearerTokenFile: "required-to-be-set", + }), + }, + }, + { + name: "no bearer token file", + clientOpts: []kubeclient.Option{ + kubeclient.WithConfig(&rest.Config{ + BearerToken: "should-be-ignored", + }), + }, + wantErr: "invalid impersonator loopback rest config has wrong bearer token semantics", + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + // This is a serial test because the production code binds to the port. + runner, constructionErr := newInternal(port, certKeyContent, caContent, tt.clientOpts, recOpts) + + if len(tt.wantErr) != 0 { + require.EqualError(t, constructionErr, tt.wantErr) + require.Nil(t, runner) + } else { + require.NoError(t, constructionErr) + require.NotNil(t, runner) + + stopCh := make(chan struct{}) + errCh := make(chan error) + go func() { + stopErr := runner(stopCh) + errCh <- stopErr + }() + + select { + case unexpectedExit := <-errCh: + t.Errorf("unexpected exit, err=%v (even nil error is failure)", unexpectedExit) + case <-time.After(10 * time.Second): + } + + close(stopCh) + exitErr := <-errCh + require.NoError(t, exitErr) + } + + // assert listener is closed is both cases above by trying to make another one on the same port + ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) + defer func() { + if ln == nil { + return + } + require.NoError(t, ln.Close()) + }() + require.NoError(t, listenErr) + + // TODO: create some client certs and assert the authorizer works correctly with system:masters + // and nested impersonation - we could also try to test what headers are sent to KAS + }) + } +} + +func TestImpersonator(t *testing.T) { + const testUser = "test-user" testGroups := []string{"test-group-1", "test-group-2"} testExtra := map[string][]string{ @@ -47,167 +132,97 @@ func TestImpersonator(t *testing.T) { } validURL, _ := url.Parse("http://pinniped.dev/blah") - newRequest := func(h http.Header) *http.Request { - r, err := http.NewRequestWithContext(context.Background(), http.MethodGet, validURL.String(), nil) + newRequest := func(h http.Header, userInfo user.Info) *http.Request { + ctx := context.Background() + if userInfo != nil { + ctx = request.WithUser(ctx, userInfo) + } + r, err := http.NewRequestWithContext(ctx, http.MethodGet, validURL.String(), nil) require.NoError(t, err) r.Header = h return r } - goodAuthenticator := corev1.TypedLocalObjectReference{ - Name: "authenticator-one", - APIGroup: stringPtr(authenticationv1alpha1.GroupName), - } - badAuthenticator := corev1.TypedLocalObjectReference{ - Name: "", - APIGroup: stringPtr(authenticationv1alpha1.GroupName), - } - tests := []struct { name string - apiGroupOverride string - getKubeconfig func() (*rest.Config, error) + restConfig *rest.Config wantCreationErr string request *http.Request wantHTTPBody string wantHTTPStatus int - wantLogs []string wantKubeAPIServerRequestHeaders http.Header - wantKubeAPIServerStatusCode int - expectMockToken func(*testing.T, *mocktokenauthenticator.MockTokenMockRecorder) + kubeAPIServerStatusCode int }{ { - name: "fail to get in-cluster config", - getKubeconfig: func() (*rest.Config, error) { - return nil, fmt.Errorf("some kubernetes error") - }, - wantCreationErr: "could not get in-cluster config: some kubernetes error", - }, - { - name: "invalid kubeconfig host", - getKubeconfig: func() (*rest.Config, error) { - return &rest.Config{Host: ":"}, nil - }, + name: "invalid kubeconfig host", + restConfig: &rest.Config{Host: ":"}, wantCreationErr: "could not parse host URL from in-cluster config: parse \":\": missing protocol scheme", }, { name: "invalid transport config", - getKubeconfig: func() (*rest.Config, error) { - return &rest.Config{ - Host: "pinniped.dev/blah", - ExecProvider: &api.ExecConfig{}, - AuthProvider: &api.AuthProviderConfig{}, - }, nil + restConfig: &rest.Config{ + Host: "pinniped.dev/blah", + ExecProvider: &api.ExecConfig{}, + AuthProvider: &api.AuthProviderConfig{}, }, wantCreationErr: "could not get in-cluster transport config: execProvider and authProvider cannot be used in combination", }, { name: "fail to get transport from config", - getKubeconfig: func() (*rest.Config, error) { - return &rest.Config{ - Host: "pinniped.dev/blah", - BearerToken: "test-bearer-token", - Transport: http.DefaultTransport, - TLSClientConfig: rest.TLSClientConfig{Insecure: true}, - }, nil + restConfig: &rest.Config{ + Host: "pinniped.dev/blah", + BearerToken: "test-bearer-token", + Transport: http.DefaultTransport, + TLSClientConfig: rest.TLSClientConfig{Insecure: true}, }, wantCreationErr: "could not get in-cluster transport: using a custom transport with TLS certificate options or the insecure flag is not allowed", }, { name: "Impersonate-User header already in request", - request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}), - wantHTTPBody: "impersonation header already exists\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-User\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}, nil), + wantHTTPBody: "invalid impersonation\n", + wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-Group header already in request", - request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}), - wantHTTPBody: "impersonation header already exists\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Group\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}, nil), + wantHTTPBody: "invalid impersonation\n", + wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-Extra header already in request", - request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}), - wantHTTPBody: "impersonation header already exists\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Extra-something\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}, nil), + wantHTTPBody: "invalid impersonation\n", + wantHTTPStatus: http.StatusInternalServerError, }, { - name: "Impersonate-* header already in request", - request: newRequest(map[string][]string{ - "Impersonate-Something": {"some-newfangled-impersonate-header"}, - }), - wantHTTPBody: "impersonation header already exists\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"impersonation header already exists\" \"error\"=\"\\\"Impersonate-Something\\\" header already exists\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + name: "Impersonate-* header already in request", + request: newRequest(map[string][]string{"Impersonate-Something": {"some-newfangled-impersonate-header"}}, nil), + wantHTTPBody: "invalid impersonation\n", + wantHTTPStatus: http.StatusInternalServerError, }, { - name: "missing authorization header", - request: newRequest(map[string][]string{}), - wantHTTPBody: "invalid token encoding\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"token authenticator did not find token\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + name: "unexpected authorization header", + request: newRequest(map[string][]string{"Authorization": {"panda"}}, nil), + wantHTTPBody: "invalid authorization header\n", + wantHTTPStatus: http.StatusInternalServerError, }, { - name: "authorization header missing bearer prefix", - request: newRequest(map[string][]string{"Authorization": {impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), - wantHTTPBody: "invalid token encoding\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"token authenticator did not find token\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + name: "missing user", + request: newRequest(map[string][]string{}, nil), + wantHTTPBody: "invalid user\n", + wantHTTPStatus: http.StatusInternalServerError, }, { - name: "token is not base64 encoded", - request: newRequest(map[string][]string{"Authorization": {"Bearer !!!"}}), - wantHTTPBody: "invalid token encoding\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid base64 in encoded bearer token: illegal base64 data at input byte 0\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, - }, - { - name: "base64 encoded token is not valid json", - request: newRequest(map[string][]string{"Authorization": {"Bearer aGVsbG8gd29ybGQK"}}), // aGVsbG8gd29ybGQK is "hello world" base64 encoded - wantHTTPBody: "invalid token encoding\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid object encoded in bearer token: couldn't get version/kind; json parse error: invalid character 'h' looking for beginning of value\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, - }, - { - name: "base64 encoded token is encoded with default api group but we are expecting custom api group", - apiGroupOverride: customAPIGroup, - request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), - wantHTTPBody: "invalid token encoding\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.pinniped.dev/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, - }, - { - name: "base64 encoded token is encoded with custom api group but we are expecting default api group", - request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}}), - wantHTTPBody: "invalid token encoding\n", - wantHTTPStatus: http.StatusBadRequest, - wantLogs: []string{"\"msg\"=\"invalid token encoding\" \"error\"=\"invalid object encoded in bearer token: no kind \\\"TokenCredentialRequest\\\" is registered for version \\\"login.concierge.walrus.tld/v1alpha1\\\" in scheme \\\"pkg/runtime/scheme.go:100\\\"\" \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, - }, - { - name: "token could not be authenticated", - request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "", &badAuthenticator, defaultAPIGroup)}}), - wantHTTPBody: "invalid token\n", - wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"msg\"=\"received invalid token\" \"error\"=\"no such authenticator\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, - }, - { - name: "token authenticates as nil", - request: newRequest(map[string][]string{"Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}}), - expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { - recorder.AuthenticateToken(gomock.Any(), "test-token").Return(nil, false, nil) - }, - wantHTTPBody: "not authenticated\n", - wantHTTPStatus: http.StatusUnauthorized, - wantLogs: []string{"\"level\"=0 \"msg\"=\"received token that did not authenticate\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\""}, + name: "unexpected UID", + request: newRequest(map[string][]string{}, &user.DefaultInfo{UID: "007"}), + wantHTTPBody: "unexpected uid\n", + wantHTTPStatus: http.StatusUnprocessableEntity, }, // happy path { - name: "token validates", + name: "authenticated user", request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, "User-Agent": {"test-user-agent"}, "Accept": {"some-accepted-format"}, "Accept-Encoding": {"some-accepted-encoding"}, @@ -216,17 +231,11 @@ func TestImpersonator(t *testing.T) { "Content-Type": {"some-type"}, "Content-Length": {"some-length"}, "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, }), - expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { - userInfo := user.DefaultInfo{ - Name: testUser, - Groups: testGroups, - UID: "test-uid", - Extra: testExtra, - } - response := &authenticator.Response{User: &userInfo} - recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) - }, wantKubeAPIServerRequestHeaders: map[string][]string{ "Authorization": {"Bearer some-service-account-token"}, "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, @@ -243,25 +252,17 @@ func TestImpersonator(t *testing.T) { }, wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, - wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, }, { - name: "token validates and the kube API request returns an error", + name: "user is authenticated but the kube API request returns an error", request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, defaultAPIGroup)}, - "User-Agent": {"test-user-agent"}, + "User-Agent": {"test-user-agent"}, + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, }), - expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { - userInfo := user.DefaultInfo{ - Name: testUser, - Groups: testGroups, - UID: "test-uid", - Extra: testExtra, - } - response := &authenticator.Response{User: &userInfo} - recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) - }, - wantKubeAPIServerStatusCode: http.StatusNotFound, + kubeAPIServerStatusCode: http.StatusNotFound, wantKubeAPIServerRequestHeaders: map[string][]string{ "Accept-Encoding": {"gzip"}, // because the rest client used in this test does not disable compression "Authorization": {"Bearer some-service-account-token"}, @@ -272,56 +273,14 @@ func TestImpersonator(t *testing.T) { "User-Agent": {"test-user-agent"}, }, wantHTTPStatus: http.StatusNotFound, - wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, - }, - { - name: "token validates with custom api group", - apiGroupOverride: customAPIGroup, - request: newRequest(map[string][]string{ - "Authorization": {"Bearer " + impersonationtoken.Make(t, "test-token", &goodAuthenticator, customAPIGroup)}, - "Other-Header": {"test-header-value-1"}, - "User-Agent": {"test-user-agent"}, - }), - expectMockToken: func(t *testing.T, recorder *mocktokenauthenticator.MockTokenMockRecorder) { - userInfo := user.DefaultInfo{ - Name: testUser, - Groups: testGroups, - UID: "test-uid", - Extra: testExtra, - } - response := &authenticator.Response{User: &userInfo} - recorder.AuthenticateToken(gomock.Any(), "test-token").Return(response, true, nil) - }, - wantKubeAPIServerRequestHeaders: map[string][]string{ - "Accept-Encoding": {"gzip"}, // because the rest client used in this test does not disable compression - "Authorization": {"Bearer some-service-account-token"}, - "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, - "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, - "Impersonate-Group": {"test-group-1", "test-group-2"}, - "Impersonate-User": {"test-user"}, - "User-Agent": {"test-user-agent"}, - "Other-Header": {"test-header-value-1"}, - }, - wantHTTPBody: "successful proxied response", - wantHTTPStatus: http.StatusOK, - wantLogs: []string{"\"level\"=0 \"msg\"=\"proxying authenticated request\" \"authenticator\"={\"apiGroup\":\"authentication.concierge.pinniped.dev\",\"kind\":\"\",\"name\":\"authenticator-one\"} \"method\"=\"GET\" \"url\"=\"http://pinniped.dev/blah\" \"userID\"=\"test-uid\""}, }, } for _, tt := range tests { tt := tt - testLog := testlogger.New(t) t.Run(tt.name, func(t *testing.T) { - defer func() { - if t.Failed() { - for i, line := range testLog.Lines() { - t.Logf("testLog line %d: %q", i+1, line) - } - } - }() - - if tt.wantKubeAPIServerStatusCode == 0 { - tt.wantKubeAPIServerStatusCode = http.StatusOK + if tt.kubeAPIServerStatusCode == 0 { + tt.kubeAPIServerStatusCode = http.StatusOK } serverWasCalled := false @@ -329,8 +288,8 @@ func TestImpersonator(t *testing.T) { testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { serverWasCalled = true serverSawHeaders = r.Header - if tt.wantKubeAPIServerStatusCode != http.StatusOK { - w.WriteHeader(tt.wantKubeAPIServerStatusCode) + if tt.kubeAPIServerStatusCode != http.StatusOK { + w.WriteHeader(tt.kubeAPIServerStatusCode) } else { _, _ = w.Write([]byte("successful proxied response")) } @@ -340,30 +299,11 @@ func TestImpersonator(t *testing.T) { BearerToken: "some-service-account-token", TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testServerCA)}, } - if tt.getKubeconfig == nil { - tt.getKubeconfig = func() (*rest.Config, error) { - return &testServerKubeconfig, nil - } + if tt.restConfig == nil { + tt.restConfig = &testServerKubeconfig } - // stole this from cache_test, hopefully it is sufficient - cacheWithMockAuthenticator := authncache.New() - ctrl := gomock.NewController(t) - defer ctrl.Finish() - key := authncache.Key{Name: "authenticator-one", APIGroup: *goodAuthenticator.APIGroup} - mockToken := mocktokenauthenticator.NewMockToken(ctrl) - cacheWithMockAuthenticator.Store(key, mockToken) - - if tt.expectMockToken != nil { - tt.expectMockToken(t, mockToken.EXPECT()) - } - - apiGroup := defaultAPIGroup - if tt.apiGroupOverride != "" { - apiGroup = tt.apiGroupOverride - } - - proxy, err := newInternal(cacheWithMockAuthenticator, makeDecoder(t, apiGroup), testLog, tt.getKubeconfig) + proxy, err := newImpersonationReverseProxy(tt.restConfig) if tt.wantCreationErr != "" { require.EqualError(t, err, tt.wantCreationErr) return @@ -380,11 +320,8 @@ func TestImpersonator(t *testing.T) { if tt.wantHTTPBody != "" { require.Equal(t, tt.wantHTTPBody, w.Body.String()) } - if tt.wantLogs != nil { - require.Equal(t, tt.wantLogs, testLog.Lines()) - } - if tt.wantHTTPStatus == http.StatusOK || tt.wantKubeAPIServerStatusCode != http.StatusOK { + if tt.wantHTTPStatus == http.StatusOK || tt.kubeAPIServerStatusCode != http.StatusOK { require.True(t, serverWasCalled, "Should have proxied the request to the Kube API server, but didn't") require.Equal(t, tt.wantKubeAPIServerRequestHeaders, serverSawHeaders) } else { @@ -393,19 +330,3 @@ func TestImpersonator(t *testing.T) { }) } } - -func stringPtr(s string) *string { return &s } - -func makeDecoder(t *testing.T, apiGroupSuffix string) runtime.Decoder { - t.Helper() - - scheme, loginGV, _ := conciergescheme.New(apiGroupSuffix) - codecs := serializer.NewCodecFactory(scheme) - respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) - require.True(t, ok, "couldn't find serializer info for media type") - - return codecs.DecoderToVersion(respInfo.Serializer, schema.GroupVersion{ - Group: loginGV.Group, - Version: login.SchemeGroupVersion.Version, - }) -} diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index cc623393..d485ee19 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -17,7 +17,6 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" "go.pinniped.dev/internal/certauthority/dynamiccertauthority" "go.pinniped.dev/internal/concierge/apiserver" conciergescheme "go.pinniped.dev/internal/concierge/scheme" @@ -27,6 +26,7 @@ import ( "go.pinniped.dev/internal/downward" "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/plog" "go.pinniped.dev/internal/registry/credentialrequest" ) @@ -116,10 +116,14 @@ func (a *App) runServer(ctx context.Context) error { // keep incoming requests fast. dynamicServingCertProvider := dynamiccert.New() - // This cert provider will be used to provide a signing key to the + // This cert provider will be used to provide the Kube signing key to the // cert issuer used to issue certs to Pinniped clients wishing to login. dynamicSigningCertProvider := dynamiccert.New() + // This cert provider will be used to provide the impersonation proxy signing key to the + // cert issuer used to issue certs to Pinniped clients wishing to login. + impersonationProxySigningCertProvider := dynamiccert.New() + // Get the "real" name of the login concierge API group (i.e., the API group name with the // injected suffix). scheme, loginGV, identityGV := conciergescheme.New(*cfg.APIGroupSuffix) @@ -128,29 +132,34 @@ func (a *App) runServer(ctx context.Context) error { // post start hook of the aggregated API server. startControllersFunc, err := controllermanager.PrepareControllers( &controllermanager.Config{ - ServerInstallationInfo: podInfo, - APIGroupSuffix: *cfg.APIGroupSuffix, - NamesConfig: &cfg.NamesConfig, - Labels: cfg.Labels, - KubeCertAgentConfig: &cfg.KubeCertAgentConfig, - DiscoveryURLOverride: cfg.DiscoveryInfo.URL, - DynamicServingCertProvider: dynamicServingCertProvider, - DynamicSigningCertProvider: dynamicSigningCertProvider, - ServingCertDuration: time.Duration(*cfg.APIConfig.ServingCertificateConfig.DurationSeconds) * time.Second, - ServingCertRenewBefore: time.Duration(*cfg.APIConfig.ServingCertificateConfig.RenewBeforeSeconds) * time.Second, - AuthenticatorCache: authenticators, - LoginJSONDecoder: getLoginJSONDecoder(loginGV.Group, scheme), + ServerInstallationInfo: podInfo, + APIGroupSuffix: *cfg.APIGroupSuffix, + NamesConfig: &cfg.NamesConfig, + Labels: cfg.Labels, + KubeCertAgentConfig: &cfg.KubeCertAgentConfig, + DiscoveryURLOverride: cfg.DiscoveryInfo.URL, + DynamicServingCertProvider: dynamicServingCertProvider, + DynamicSigningCertProvider: dynamicSigningCertProvider, + ImpersonationSigningCertProvider: impersonationProxySigningCertProvider, + ServingCertDuration: time.Duration(*cfg.APIConfig.ServingCertificateConfig.DurationSeconds) * time.Second, + ServingCertRenewBefore: time.Duration(*cfg.APIConfig.ServingCertificateConfig.RenewBeforeSeconds) * time.Second, + AuthenticatorCache: authenticators, }, ) if err != nil { return fmt.Errorf("could not prepare controllers: %w", err) } + certIssuer := issuer.CertIssuers{ + dynamiccertauthority.New(dynamicSigningCertProvider), // attempt to use the real Kube CA if possible + dynamiccertauthority.New(impersonationProxySigningCertProvider), // fallback to our internal CA if we need to + } + // Get the aggregated API server config. aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig( dynamicServingCertProvider, authenticators, - dynamiccertauthority.New(dynamicSigningCertProvider), + certIssuer, startControllersFunc, *cfg.APIGroupSuffix, scheme, @@ -175,7 +184,7 @@ func (a *App) runServer(ctx context.Context) error { func getAggregatedAPIServerConfig( dynamicCertProvider dynamiccert.Provider, authenticator credentialrequest.TokenCredentialRequestAuthenticator, - issuer credentialrequest.CertIssuer, + issuer issuer.CertIssuer, startControllersPostStartHook func(context.Context), apiGroupSuffix string, scheme *runtime.Scheme, @@ -222,16 +231,3 @@ func getAggregatedAPIServerConfig( } return apiServerConfig, nil } - -func getLoginJSONDecoder(loginConciergeAPIGroup string, loginConciergeScheme *runtime.Scheme) runtime.Decoder { - scheme := loginConciergeScheme - codecs := serializer.NewCodecFactory(scheme) - respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) - if !ok { - panic(fmt.Errorf("unknown content type: %s ", runtime.ContentTypeJSON)) // static input, programmer error - } - return codecs.DecoderToVersion(respInfo.Serializer, schema.GroupVersion{ - Group: loginConciergeAPIGroup, - Version: loginapi.SchemeGroupVersion.Version, - }) -} diff --git a/internal/config/concierge/config.go b/internal/config/concierge/config.go index b6720cea..b4d4c9a5 100644 --- a/internal/config/concierge/config.go +++ b/internal/config/concierge/config.go @@ -119,6 +119,9 @@ func validateNames(names *NamesConfigSpec) error { if names.ImpersonationCACertificateSecret == "" { missingNames = append(missingNames, "impersonationCACertificateSecret") } + if names.ImpersonationSignerSecret == "" { + missingNames = append(missingNames, "impersonationSignerSecret") + } if len(missingNames) > 0 { return constable.Error("missing required names: " + strings.Join(missingNames, ", ")) } diff --git a/internal/config/concierge/config_test.go b/internal/config/concierge/config_test.go index 5b8177ff..0034d06b 100644 --- a/internal/config/concierge/config_test.go +++ b/internal/config/concierge/config_test.go @@ -41,6 +41,8 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value + impersonationSignerSecret: impersonationSignerSecret-value labels: myLabelKey1: myLabelValue1 myLabelKey2: myLabelValue2 @@ -69,6 +71,7 @@ func TestFromPath(t *testing.T) { ImpersonationLoadBalancerService: "impersonationLoadBalancerService-value", ImpersonationTLSCertificateSecret: "impersonationTLSCertificateSecret-value", ImpersonationCACertificateSecret: "impersonationCACertificateSecret-value", + ImpersonationSignerSecret: "impersonationSignerSecret-value", }, Labels: map[string]string{ "myLabelKey1": "myLabelValue1", @@ -94,6 +97,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantConfig: &Config{ DiscoveryInfo: DiscoveryInfoSpec{ @@ -114,6 +118,7 @@ func TestFromPath(t *testing.T) { ImpersonationLoadBalancerService: "impersonationLoadBalancerService-value", ImpersonationTLSCertificateSecret: "impersonationTLSCertificateSecret-value", ImpersonationCACertificateSecret: "impersonationCACertificateSecret-value", + ImpersonationSignerSecret: "impersonationSignerSecret-value", }, Labels: map[string]string{}, KubeCertAgentConfig: KubeCertAgentSpec{ @@ -127,7 +132,8 @@ func TestFromPath(t *testing.T) { yaml: here.Doc(``), wantError: "validate names: missing required names: servingCertificateSecret, credentialIssuer, " + "apiService, impersonationConfigMap, impersonationLoadBalancerService, " + - "impersonationTLSCertificateSecret, impersonationCACertificateSecret", + "impersonationTLSCertificateSecret, impersonationCACertificateSecret, " + + "impersonationSignerSecret", }, { name: "Missing apiService name", @@ -140,6 +146,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: apiService", }, @@ -154,6 +161,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: credentialIssuer", }, @@ -168,6 +176,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: servingCertificateSecret", }, @@ -182,6 +191,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: impersonationConfigMap", }, @@ -196,6 +206,7 @@ func TestFromPath(t *testing.T) { impersonationConfigMap: impersonationConfigMap-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: impersonationLoadBalancerService", }, @@ -210,6 +221,7 @@ func TestFromPath(t *testing.T) { impersonationConfigMap: impersonationConfigMap-value impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: impersonationTLSCertificateSecret", }, @@ -224,9 +236,25 @@ func TestFromPath(t *testing.T) { impersonationConfigMap: impersonationConfigMap-value impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: impersonationCACertificateSecret", }, + { + name: "Missing impersonationSignerSecret name", + yaml: here.Doc(` + --- + names: + servingCertificateSecret: pinniped-concierge-api-tls-serving-certificate + credentialIssuer: pinniped-config + apiService: pinniped-api + impersonationConfigMap: impersonationConfigMap-value + impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value + impersonationCACertificateSecret: impersonationCACertificateSecret-value + `), + wantError: "validate names: missing required names: impersonationSignerSecret", + }, { name: "Missing several required names", yaml: here.Doc(` @@ -236,6 +264,7 @@ func TestFromPath(t *testing.T) { credentialIssuer: pinniped-config apiService: pinniped-api impersonationLoadBalancerService: impersonationLoadBalancerService-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate names: missing required names: impersonationConfigMap, " + "impersonationTLSCertificateSecret, impersonationCACertificateSecret", @@ -256,6 +285,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate api: durationSeconds cannot be smaller than renewBeforeSeconds", }, @@ -275,6 +305,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate api: renewBefore must be positive", }, @@ -294,6 +325,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate api: renewBefore must be positive", }, @@ -314,6 +346,7 @@ func TestFromPath(t *testing.T) { impersonationLoadBalancerService: impersonationLoadBalancerService-value impersonationTLSCertificateSecret: impersonationTLSCertificateSecret-value impersonationCACertificateSecret: impersonationCACertificateSecret-value + impersonationSignerSecret: impersonationSignerSecret-value `), wantError: "validate apiGroupSuffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", }, diff --git a/internal/config/concierge/types.go b/internal/config/concierge/types.go index 2a151274..cfec621e 100644 --- a/internal/config/concierge/types.go +++ b/internal/config/concierge/types.go @@ -40,6 +40,7 @@ type NamesConfigSpec struct { ImpersonationLoadBalancerService string `json:"impersonationLoadBalancerService"` ImpersonationTLSCertificateSecret string `json:"impersonationTLSCertificateSecret"` ImpersonationCACertificateSecret string `json:"impersonationCACertificateSecret"` + ImpersonationSignerSecret string `json:"impersonationSignerSecret"` } // ServingCertificateConfigSpec contains the configuration knobs for the API's diff --git a/internal/controller/apicerts/apiservice_updater.go b/internal/controller/apicerts/apiservice_updater.go index 4e28e519..3cfe4fcc 100644 --- a/internal/controller/apicerts/apiservice_updater.go +++ b/internal/controller/apicerts/apiservice_updater.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -64,7 +64,7 @@ func (c *apiServiceUpdaterController) Sync(ctx controllerlib.Context) error { } // Update the APIService to give it the new CA bundle. - if err := UpdateAPIService(ctx.Context, c.aggregatorClient, c.apiServiceName, c.namespace, certSecret.Data[caCertificateSecretKey]); err != nil { + if err := UpdateAPIService(ctx.Context, c.aggregatorClient, c.apiServiceName, c.namespace, certSecret.Data[CACertificateSecretKey]); err != nil { return fmt.Errorf("could not update the API service: %w", err) } diff --git a/internal/controller/apicerts/certs_expirer.go b/internal/controller/apicerts/certs_expirer.go index 336f619f..2b643c3e 100644 --- a/internal/controller/apicerts/certs_expirer.go +++ b/internal/controller/apicerts/certs_expirer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -30,6 +30,8 @@ type certsExpirerController struct { // renewBefore is the amount of time after the cert's issuance where // this controller will start to try to rotate it. renewBefore time.Duration + + secretKey string } // NewCertsExpirerController returns a controllerlib.Controller that will delete a @@ -42,6 +44,7 @@ func NewCertsExpirerController( secretInformer corev1informers.SecretInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, renewBefore time.Duration, + secretKey string, ) controllerlib.Controller { return controllerlib.New( controllerlib.Config{ @@ -52,6 +55,7 @@ func NewCertsExpirerController( k8sClient: k8sClient, secretInformer: secretInformer, renewBefore: renewBefore, + secretKey: secretKey, }, }, withInformer( @@ -74,13 +78,9 @@ func (c *certsExpirerController) Sync(ctx controllerlib.Context) error { return nil } - notBefore, notAfter, err := getCertBounds(secret) + notBefore, notAfter, err := c.getCertBounds(secret) if err != nil { - // If we can't read the cert, then really all we can do is log something, - // since if we returned an error then the controller lib would just call us - // again and again, which would probably yield the same results. - klog.Warningf("certsExpirerController Sync found that the secret is malformed: %s", err.Error()) - return nil + return fmt.Errorf("failed to get cert bounds for secret %q with key %q: %w", secret.Name, c.secretKey, err) } certAge := time.Since(notBefore) @@ -105,8 +105,8 @@ func (c *certsExpirerController) Sync(ctx controllerlib.Context) error { // certificate in the provided secret, or an error. Not that it expects the // provided secret to contain the well-known data keys from this package (see // certs_manager.go). -func getCertBounds(secret *corev1.Secret) (time.Time, time.Time, error) { - certPEM := secret.Data[tlsCertificateChainSecretKey] +func (c *certsExpirerController) getCertBounds(secret *corev1.Secret) (time.Time, time.Time, error) { + certPEM := secret.Data[c.secretKey] if certPEM == nil { return time.Time{}, time.Time{}, constable.Error("failed to find certificate") } diff --git a/internal/controller/apicerts/certs_expirer_test.go b/internal/controller/apicerts/certs_expirer_test.go index f8f16595..7e82819a 100644 --- a/internal/controller/apicerts/certs_expirer_test.go +++ b/internal/controller/apicerts/certs_expirer_test.go @@ -98,7 +98,8 @@ func TestExpirerControllerFilters(t *testing.T) { nil, // k8sClient, not needed secretsInformer, withInformer.WithInformer, - 0, // renewBefore, not needed + 0, // renewBefore, not needed + "", // not needed ) unrelated := corev1.Secret{} @@ -115,6 +116,7 @@ func TestExpirerControllerSync(t *testing.T) { t.Parallel() const certsSecretResourceName = "some-resource-name" + const fakeTestKey = "some-awesome-key" tests := []struct { name string @@ -132,6 +134,7 @@ func TestExpirerControllerSync(t *testing.T) { name: "secret missing key", fillSecretData: func(t *testing.T, m map[string][]byte) {}, wantDelete: false, + wantError: `failed to get cert bounds for secret "some-resource-name" with key "some-awesome-key": failed to find certificate`, }, { name: "lifetime below threshold", @@ -143,8 +146,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, wantDelete: false, }, @@ -158,8 +160,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, wantDelete: true, }, @@ -173,8 +174,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, wantDelete: true, }, @@ -188,8 +188,7 @@ func TestExpirerControllerSync(t *testing.T) { ) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"] = certPEM + m[fakeTestKey] = certPEM }, configKubeAPIClient: func(c *kubernetesfake.Clientset) { c.PrependReactor("delete", "secrets", func(_ kubetesting.Action) (bool, runtime.Object, error) { @@ -204,11 +203,11 @@ func TestExpirerControllerSync(t *testing.T) { privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) - // See certs_manager.go for this constant. - m["tlsCertificateChain"], err = x509.MarshalPKCS8PrivateKey(privateKey) + m[fakeTestKey], err = x509.MarshalPKCS8PrivateKey(privateKey) require.NoError(t, err) }, wantDelete: false, + wantError: `failed to get cert bounds for secret "some-resource-name" with key "some-awesome-key": failed to decode certificate PEM`, }, } for _, test := range tests { @@ -253,6 +252,7 @@ func TestExpirerControllerSync(t *testing.T) { kubeInformers.Core().V1().Secrets(), controllerlib.WithInformer, test.renewBefore, + fakeTestKey, ) // Must start informers before calling TestRunSynchronously(). diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index 22f0f6df..f0c11d0d 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -21,9 +21,10 @@ import ( ) const ( - caCertificateSecretKey = "caCertificate" - tlsPrivateKeySecretKey = "tlsPrivateKey" - tlsCertificateChainSecretKey = "tlsCertificateChain" + CACertificateSecretKey = "caCertificate" + CACertificatePrivateKeySecretKey = "caCertificatePrivateKey" + tlsPrivateKeySecretKey = "tlsPrivateKey" + TLSCertificateChainSecretKey = "tlsCertificateChain" ) type certsManagerController struct { @@ -98,23 +99,11 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { return fmt.Errorf("could not initialize CA: %w", err) } - // Using the CA from above, create a TLS server cert. - serviceEndpoint := c.serviceNameForGeneratedCertCommonName + "." + c.namespace + ".svc" - tlsCert, err := ca.Issue( - pkix.Name{CommonName: serviceEndpoint}, - []string{serviceEndpoint}, - nil, - c.certDuration, - ) + caPrivateKeyPEM, err := ca.PrivateKeyToPEM() if err != nil { - return fmt.Errorf("could not issue serving certificate: %w", err) + return fmt.Errorf("could not get CA private key: %w", err) } - // Write the CA's public key bundle and the serving certs to a secret. - tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(tlsCert) - if err != nil { - return fmt.Errorf("could not PEM encode serving certificate: %w", err) - } secret := corev1.Secret{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ @@ -123,11 +112,34 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { Labels: c.certsSecretLabels, }, StringData: map[string]string{ - caCertificateSecretKey: string(ca.Bundle()), - tlsPrivateKeySecretKey: string(tlsPrivateKeyPEM), - tlsCertificateChainSecretKey: string(tlsCertChainPEM), + CACertificateSecretKey: string(ca.Bundle()), + CACertificatePrivateKeySecretKey: string(caPrivateKeyPEM), }, } + + // Using the CA from above, create a TLS server cert if we have service name. + if len(c.serviceNameForGeneratedCertCommonName) != 0 { + serviceEndpoint := c.serviceNameForGeneratedCertCommonName + "." + c.namespace + ".svc" + tlsCert, err := ca.Issue( + pkix.Name{CommonName: serviceEndpoint}, + []string{serviceEndpoint}, + nil, + c.certDuration, + ) + if err != nil { + return fmt.Errorf("could not issue serving certificate: %w", err) + } + + // Write the CA's public key bundle and the serving certs to a secret. + tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(tlsCert) + if err != nil { + return fmt.Errorf("could not PEM encode serving certificate: %w", err) + } + + secret.StringData[tlsPrivateKeySecretKey] = string(tlsPrivateKeyPEM) + secret.StringData[TLSCertificateChainSecretKey] = string(tlsCertChainPEM) + } + _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("could not create secret: %w", err) diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index f70f8500..b32b9228 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -49,7 +49,7 @@ func TestManagerControllerOptions(t *testing.T) { observableWithInitialEventOption.WithInitialEvent, 0, "Pinniped CA", - "pinniped-api", + "ignored", ) secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) }) @@ -118,6 +118,7 @@ func TestManagerControllerSync(t *testing.T) { const installedInNamespace = "some-namespace" const certsSecretResourceName = "some-resource-name" const certDuration = 12345678 * time.Second + const defaultServiceName = "pinniped-api" var r *require.Assertions @@ -131,7 +132,7 @@ func TestManagerControllerSync(t *testing.T) { // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. - var startInformersAndController = func() { + var startInformersAndController = func(serviceName string) { // Set this at the last second to allow for injection of server override. subject = NewCertsManagerController( installedInNamespace, @@ -146,7 +147,7 @@ func TestManagerControllerSync(t *testing.T) { controllerlib.WithInitialEvent, certDuration, "Pinniped CA", - "pinniped-api", + serviceName, ) // Set this at the last second to support calling subject.Name(). @@ -191,7 +192,7 @@ func TestManagerControllerSync(t *testing.T) { }) it("creates the serving cert Secret", func() { - startInformersAndController() + startInformersAndController(defaultServiceName) err := controllerlib.TestSync(t, subject, *syncContext) r.NoError(err) @@ -208,14 +209,17 @@ func TestManagerControllerSync(t *testing.T) { "myLabelKey2": "myLabelValue2", }, actualSecret.Labels) actualCACert := actualSecret.StringData["caCertificate"] + actualCAPrivateKey := actualSecret.StringData["caCertificatePrivateKey"] actualPrivateKey := actualSecret.StringData["tlsPrivateKey"] actualCertChain := actualSecret.StringData["tlsCertificateChain"] r.NotEmpty(actualCACert) + r.NotEmpty(actualCAPrivateKey) r.NotEmpty(actualPrivateKey) r.NotEmpty(actualCertChain) + r.Len(actualSecret.StringData, 4) - // Validate the created CA's lifetime. validCACert := testutil.ValidateCertificate(t, actualCACert, actualCACert) + validCACert.RequireMatchesPrivateKey(actualCAPrivateKey) validCACert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) // Validate the created cert using the CA, and also validate the cert's hostname @@ -225,6 +229,34 @@ func TestManagerControllerSync(t *testing.T) { validCert.RequireMatchesPrivateKey(actualPrivateKey) }) + it("creates the CA but not service when the service name is empty", func() { + startInformersAndController("") + err := controllerlib.TestSync(t, subject, *syncContext) + r.NoError(err) + + // Check all the relevant fields from the create Secret action + r.Len(kubeAPIClient.Actions(), 1) + actualAction := kubeAPIClient.Actions()[0].(coretesting.CreateActionImpl) + r.Equal(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, actualAction.GetResource()) + r.Equal(installedInNamespace, actualAction.GetNamespace()) + actualSecret := actualAction.GetObject().(*corev1.Secret) + r.Equal(certsSecretResourceName, actualSecret.Name) + r.Equal(installedInNamespace, actualSecret.Namespace) + r.Equal(map[string]string{ + "myLabelKey1": "myLabelValue1", + "myLabelKey2": "myLabelValue2", + }, actualSecret.Labels) + actualCACert := actualSecret.StringData["caCertificate"] + actualCAPrivateKey := actualSecret.StringData["caCertificatePrivateKey"] + r.NotEmpty(actualCACert) + r.NotEmpty(actualCAPrivateKey) + r.Len(actualSecret.StringData, 2) + + validCACert := testutil.ValidateCertificate(t, actualCACert, actualCACert) + validCACert.RequireMatchesPrivateKey(actualCAPrivateKey) + validCACert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) + }) + when("creating the Secret fails", func() { it.Before(func() { kubeAPIClient.PrependReactor( @@ -237,7 +269,7 @@ func TestManagerControllerSync(t *testing.T) { }) it("returns the create error", func() { - startInformersAndController() + startInformersAndController(defaultServiceName) err := controllerlib.TestSync(t, subject, *syncContext) r.EqualError(err, "could not create secret: create failed") }) @@ -257,7 +289,7 @@ func TestManagerControllerSync(t *testing.T) { }) it("does not need to make any API calls with its API client", func() { - startInformersAndController() + startInformersAndController(defaultServiceName) err := controllerlib.TestSync(t, subject, *syncContext) r.NoError(err) r.Empty(kubeAPIClient.Actions()) diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go index 7f05d243..c6380741 100644 --- a/internal/controller/apicerts/certs_observer.go +++ b/internal/controller/apicerts/certs_observer.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package apicerts @@ -62,7 +62,7 @@ func (c *certsObserverController) Sync(_ controllerlib.Context) error { } // Mutate the in-memory cert provider to update with the latest cert values. - c.dynamicCertProvider.Set(certSecret.Data[tlsCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]) + c.dynamicCertProvider.Set(certSecret.Data[TLSCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]) klog.Info("certsObserverController Sync updated certs in the dynamic cert provider") return nil } diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index c1e491aa..8d741a4f 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -10,19 +10,18 @@ import ( "crypto/x509/pkix" "encoding/base64" "encoding/pem" - "errors" "fmt" "net" - "net/http" "strings" - "sync" "time" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" @@ -31,14 +30,17 @@ import ( "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/clusterhost" "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/constable" pinnipedcontroller "go.pinniped.dev/internal/controller" + "go.pinniped.dev/internal/controller/apicerts" "go.pinniped.dev/internal/controller/issuerconfig" "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/plog" ) const ( - impersonationProxyPort = "8444" + impersonationProxyPort = 8444 defaultHTTPSPort = 443 oneHundredYears = 100 * 365 * 24 * time.Hour caCommonName = "Pinniped Impersonation Proxy CA" @@ -54,6 +56,7 @@ type impersonatorConfigController struct { generatedLoadBalancerServiceName string tlsSecretName string caSecretName string + impersonationSignerSecretName string k8sClient kubernetes.Interface pinnipedAPIClient pinnipedclientset.Interface @@ -62,19 +65,17 @@ type impersonatorConfigController struct { servicesInformer corev1informers.ServiceInformer secretsInformer corev1informers.SecretInformer - labels map[string]string - clock clock.Clock - startTLSListenerFunc StartTLSListenerFunc - httpHandlerFactory func() (http.Handler, error) + labels map[string]string + clock clock.Clock + impersonationSigningCertProvider dynamiccert.Provider + impersonatorFunc impersonator.FactoryFunc - server *http.Server - hasControlPlaneNodes *bool - tlsCert *tls.Certificate // always read/write using tlsCertMutex - tlsCertMutex sync.RWMutex + hasControlPlaneNodes *bool + serverStopCh chan struct{} + errorCh chan error + tlsServingCertDynamicCertProvider dynamiccert.Provider } -type StartTLSListenerFunc func(network, listenAddress string, config *tls.Config) (net.Listener, error) - func NewImpersonatorConfigController( namespace string, configMapResourceName string, @@ -91,28 +92,32 @@ func NewImpersonatorConfigController( caSecretName string, labels map[string]string, clock clock.Clock, - startTLSListenerFunc StartTLSListenerFunc, - httpHandlerFactory func() (http.Handler, error), + impersonatorFunc impersonator.FactoryFunc, + impersonationSignerSecretName string, + impersonationSigningCertProvider dynamiccert.Provider, ) controllerlib.Controller { + secretNames := sets.NewString(tlsSecretName, caSecretName, impersonationSignerSecretName) return controllerlib.New( controllerlib.Config{ Name: "impersonator-config-controller", Syncer: &impersonatorConfigController{ - namespace: namespace, - configMapResourceName: configMapResourceName, - credentialIssuerResourceName: credentialIssuerResourceName, - generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, - tlsSecretName: tlsSecretName, - caSecretName: caSecretName, - k8sClient: k8sClient, - pinnipedAPIClient: pinnipedAPIClient, - configMapsInformer: configMapsInformer, - servicesInformer: servicesInformer, - secretsInformer: secretsInformer, - labels: labels, - clock: clock, - startTLSListenerFunc: startTLSListenerFunc, - httpHandlerFactory: httpHandlerFactory, + namespace: namespace, + configMapResourceName: configMapResourceName, + credentialIssuerResourceName: credentialIssuerResourceName, + generatedLoadBalancerServiceName: generatedLoadBalancerServiceName, + tlsSecretName: tlsSecretName, + caSecretName: caSecretName, + impersonationSignerSecretName: impersonationSignerSecretName, + k8sClient: k8sClient, + pinnipedAPIClient: pinnipedAPIClient, + configMapsInformer: configMapsInformer, + servicesInformer: servicesInformer, + secretsInformer: secretsInformer, + labels: labels, + clock: clock, + impersonationSigningCertProvider: impersonationSigningCertProvider, + impersonatorFunc: impersonatorFunc, + tlsServingCertDynamicCertProvider: dynamiccert.New(), }, }, withInformer( @@ -128,7 +133,7 @@ func NewImpersonatorConfigController( withInformer( secretsInformer, pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { - return (obj.GetName() == tlsSecretName || obj.GetName() == caSecretName) && obj.GetNamespace() == namespace + return obj.GetNamespace() == namespace && secretNames.Has(obj.GetName()) }, nil), controllerlib.InformerOption{}, ), @@ -138,13 +143,14 @@ func NewImpersonatorConfigController( Namespace: namespace, Name: configMapResourceName, }), + // TODO fix these controller options to make this a singleton queue ) } func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error { plog.Debug("Starting impersonatorConfigController Sync") - strategy, err := c.doSync(syncCtx.Context) + strategy, err := c.doSync(syncCtx) if err != nil { strategy = &v1alpha1.CredentialIssuerStrategy{ @@ -154,6 +160,8 @@ func (c *impersonatorConfigController) Sync(syncCtx controllerlib.Context) error Message: err.Error(), LastUpdateTime: metav1.NewTime(c.clock.Now()), } + // The impersonator is not ready, so clear the signer CA from the dynamic provider. + c.clearSignerCA() } updateStrategyErr := c.updateStrategy(syncCtx.Context, strategy) @@ -186,7 +194,9 @@ type certNameInfo struct { clientEndpoint string } -func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.CredentialIssuerStrategy, error) { +func (c *impersonatorConfigController) doSync(syncCtx controllerlib.Context) (*v1alpha1.CredentialIssuerStrategy, error) { + ctx := syncCtx.Context + config, err := c.loadImpersonationProxyConfiguration() if err != nil { return nil, err @@ -206,12 +216,12 @@ func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.Cr } if c.shouldHaveImpersonator(config) { - if err = c.ensureImpersonatorIsStarted(); err != nil { + if err = c.ensureImpersonatorIsStarted(syncCtx); err != nil { return nil, err } } else { - if err = c.ensureImpersonatorIsStopped(); err != nil { - return nil, err + if err = c.ensureImpersonatorIsStopped(true); err != nil { + return nil, err // TODO write unit test that errors during stopping the server are returned by sync } } @@ -227,6 +237,8 @@ func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.Cr nameInfo, err := c.findDesiredTLSCertificateName(config) if err != nil { + // Unexpected error while determining the name that should go into the certs, so clear any existing certs. + c.tlsServingCertDynamicCertProvider.Set(nil, nil) return nil, err } @@ -242,7 +254,13 @@ func (c *impersonatorConfigController) doSync(ctx context.Context) (*v1alpha1.Cr return nil, err } - return c.doSyncResult(nameInfo, config, impersonationCA), nil + credentialIssuerStrategyResult := c.doSyncResult(nameInfo, config, impersonationCA) + + if err = c.loadSignerCA(credentialIssuerStrategyResult.Status); err != nil { + return nil, err + } + + return credentialIssuerStrategyResult, nil } func (c *impersonatorConfigController) loadImpersonationProxyConfiguration() (*impersonator.Config, error) { @@ -325,50 +343,73 @@ func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, erro return true, secret, nil } -func (c *impersonatorConfigController) ensureImpersonatorIsStarted() error { - if c.server != nil { - return nil - } - - handler, err := c.httpHandlerFactory() - if err != nil { - return err - } - - listener, err := c.startTLSListenerFunc("tcp", ":"+impersonationProxyPort, &tls.Config{ - MinVersion: tls.VersionTLS12, // Allow v1.2 because clients like the default `curl` on MacOS don't support 1.3 yet. - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - return c.getTLSCert(), nil - }, - }) - if err != nil { - return err - } - - c.server = &http.Server{Handler: handler} - - go func() { - plog.Info("Starting impersonation proxy", "port", impersonationProxyPort) - err = c.server.Serve(listener) - if errors.Is(err, http.ErrServerClosed) { - plog.Info("The impersonation proxy server has shut down") - } else { - plog.Error("Unexpected shutdown of the impersonation proxy server", err) +func (c *impersonatorConfigController) ensureImpersonatorIsStarted(syncCtx controllerlib.Context) error { + if c.serverStopCh != nil { + // The server was already started, but it could have died in the background, so make a non-blocking + // check to see if it has sent any errors on the errorCh. + select { + case runningErr := <-c.errorCh: + if runningErr == nil { + // The server sent a nil error, meaning that it shutdown without reporting any particular + // error for some reason. We would still like to report this as an error for logging purposes. + runningErr = constable.Error("unexpected shutdown of proxy server") + } + // The server has stopped, so finish shutting it down. + // If that fails too, return both errors for logging purposes. + // By returning an error, the sync function will be called again + // and we'll have a change to restart the server. + close(c.errorCh) // We don't want ensureImpersonatorIsStopped to block on reading this channel. + stoppingErr := c.ensureImpersonatorIsStopped(false) + return errors.NewAggregate([]error{runningErr, stoppingErr}) + default: + // Seems like it is still running, so nothing to do. + return nil } + } + + plog.Info("Starting impersonation proxy", "port", impersonationProxyPort) + startImpersonatorFunc, err := c.impersonatorFunc( + impersonationProxyPort, + c.tlsServingCertDynamicCertProvider, + dynamiccert.NewCAProvider(c.impersonationSigningCertProvider), + ) + if err != nil { + return err + } + + c.serverStopCh = make(chan struct{}) + c.errorCh = make(chan error) + + // startImpersonatorFunc will block until the server shuts down (or fails to start), so run it in the background. + go func() { + startOrStopErr := startImpersonatorFunc(c.serverStopCh) + // The server has stopped, so enqueue ourselves for another sync, so we can + // try to start the server again as quickly as possible. + syncCtx.Queue.AddRateLimited(syncCtx.Key) // TODO this a race because the main controller go routine could run and complete before we send on the err chan + // Forward any errors returned by startImpersonatorFunc on the errorCh. + c.errorCh <- startOrStopErr }() + return nil } -func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { - if c.server != nil { - plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) - err := c.server.Close() - c.server = nil - if err != nil { - return err - } +func (c *impersonatorConfigController) ensureImpersonatorIsStopped(shouldCloseErrChan bool) error { + if c.serverStopCh == nil { + return nil } - return nil + + plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) + close(c.serverStopCh) + stopErr := <-c.errorCh + + if shouldCloseErrChan { + close(c.errorCh) + } + + c.serverStopCh = nil + c.errorCh = nil + + return stopErr } func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.Context) error { @@ -385,7 +426,7 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C Type: v1.ServiceTypeLoadBalancer, Ports: []v1.ServicePort{ { - TargetPort: intstr.Parse(impersonationProxyPort), + TargetPort: intstr.FromInt(impersonationProxyPort), Port: defaultHTTPSPort, Protocol: v1.ProtocolTCP, }, @@ -614,19 +655,19 @@ func (c *impersonatorConfigController) createCASecret(ctx context.Context) (*cer func (c *impersonatorConfigController) findDesiredTLSCertificateName(config *impersonator.Config) (*certNameInfo, error) { if config.HasEndpoint() { - return c.findTLSCertificateNameFromEndpointConfig(config) + return c.findTLSCertificateNameFromEndpointConfig(config), nil } return c.findTLSCertificateNameFromLoadBalancer() } -func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) (*certNameInfo, error) { +func (c *impersonatorConfigController) findTLSCertificateNameFromEndpointConfig(config *impersonator.Config) *certNameInfo { endpointMaybeWithPort := config.Endpoint endpointWithoutPort := strings.Split(endpointMaybeWithPort, ":")[0] parsedAsIP := net.ParseIP(endpointWithoutPort) if parsedAsIP != nil { - return &certNameInfo{ready: true, selectedIP: parsedAsIP, clientEndpoint: endpointMaybeWithPort}, nil + return &certNameInfo{ready: true, selectedIP: parsedAsIP, clientEndpoint: endpointMaybeWithPort} } - return &certNameInfo{ready: true, selectedHostname: endpointWithoutPort, clientEndpoint: endpointMaybeWithPort}, nil + return &certNameInfo{ready: true, selectedHostname: endpointWithoutPort, clientEndpoint: endpointMaybeWithPort} } func (c *impersonatorConfigController) findTLSCertificateNameFromLoadBalancer() (*certNameInfo, error) { @@ -707,16 +748,16 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secret) error { certPEM := tlsSecret.Data[v1.TLSCertKey] keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + _, err := tls.X509KeyPair(certPEM, keyPEM) if err != nil { - c.setTLSCert(nil) + c.tlsServingCertDynamicCertProvider.Set(nil, nil) return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) } plog.Info("Loading TLS certificates for impersonation proxy", "certPEM", string(certPEM), "secret", c.tlsSecretName, "namespace", c.namespace) - c.setTLSCert(&tlsCert) + c.tlsServingCertDynamicCertProvider.Set(certPEM, keyPEM) return nil } @@ -736,11 +777,43 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return err } - c.setTLSCert(nil) + c.tlsServingCertDynamicCertProvider.Set(nil, nil) return nil } +func (c *impersonatorConfigController) loadSignerCA(status v1alpha1.StrategyStatus) error { + // Clear it when the impersonator is not completely ready. + if status != v1alpha1.SuccessStrategyStatus { + c.clearSignerCA() + return nil + } + + signingCertSecret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.impersonationSignerSecretName) + if err != nil { + return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err) + } + + certPEM := signingCertSecret.Data[apicerts.CACertificateSecretKey] + keyPEM := signingCertSecret.Data[apicerts.CACertificatePrivateKeySecretKey] + _, err = tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err) + } + + plog.Info("Loading credential signing certificate for impersonation proxy", + "certPEM", string(certPEM), + "fromSecret", c.impersonationSignerSecretName, + "namespace", c.namespace) + c.impersonationSigningCertProvider.Set(certPEM, keyPEM) + return nil +} + +func (c *impersonatorConfigController) clearSignerCA() { + plog.Info("Clearing credential signing certificate for impersonation proxy") + c.impersonationSigningCertProvider.Set(nil, nil) +} + func (c *impersonatorConfigController) doSyncResult(nameInfo *certNameInfo, config *impersonator.Config, ca *certauthority.CA) *v1alpha1.CredentialIssuerStrategy { switch { case c.disabledExplicitly(config): @@ -784,15 +857,3 @@ func (c *impersonatorConfigController) doSyncResult(nameInfo *certNameInfo, conf } } } - -func (c *impersonatorConfigController) setTLSCert(cert *tls.Certificate) { - c.tlsCertMutex.Lock() - defer c.tlsCertMutex.Unlock() - c.tlsCert = cert -} - -func (c *impersonatorConfigController) getTLSCert() *tls.Certificate { - c.tlsCertMutex.RLock() - defer c.tlsCertMutex.RUnlock() - return c.tlsCert -} diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 65ea68ff..66cab3e7 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -17,7 +17,7 @@ import ( "net/http" "reflect" "regexp" - "strings" + "sync" "testing" "time" @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apiserver/pkg/server/dynamiccertificates" kubeinformers "k8s.io/client-go/informers" kubernetesfake "k8s.io/client-go/kubernetes/fake" coretesting "k8s.io/client-go/testing" @@ -37,33 +38,13 @@ import ( "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/controller/apicerts" "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" ) -type tlsListenerWrapper struct { - listener net.Listener - closeError error -} - -func (t *tlsListenerWrapper) Accept() (net.Conn, error) { - return t.listener.Accept() -} - -func (t *tlsListenerWrapper) Close() error { - if t.closeError != nil { - // Really close the connection and then "pretend" that there was an error during close. - _ = t.listener.Close() - return t.closeError - } - return t.listener.Close() -} - -func (t *tlsListenerWrapper) Addr() net.Addr { - return t.listener.Addr() -} - func TestImpersonatorConfigControllerOptions(t *testing.T) { spec.Run(t, "options", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" @@ -71,6 +52,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { const generatedLoadBalancerServiceName = "some-service-resource-name" const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential const caSecretName = "some-ca-secret-name" + const caSignerName = "some-ca-signer-name" var r *require.Assertions var observableWithInformerOption *testutil.ObservableWithInformerOption @@ -105,6 +87,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { nil, nil, nil, + caSignerName, nil, ) configMapsInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapsInformer) @@ -210,12 +193,13 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { when("watching Secret objects", func() { var subject controllerlib.Filter - var target1, target2, wrongNamespace1, wrongNamespace2, wrongName, unrelated *corev1.Secret + var target1, target2, target3, wrongNamespace1, wrongNamespace2, wrongName, unrelated *corev1.Secret it.Before(func() { subject = secretsInformerFilter target1 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: installedInNamespace}} target2 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: installedInNamespace}} + target3 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSignerName, Namespace: installedInNamespace}} wrongNamespace1 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: tlsSecretName, Namespace: "wrong-namespace"}} wrongNamespace2 = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: caSecretName, Namespace: "wrong-namespace"}} wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} @@ -232,6 +216,10 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { r.True(subject.Update(target2, unrelated)) r.True(subject.Update(unrelated, target2)) r.True(subject.Delete(target2)) + r.True(subject.Add(target3)) + r.True(subject.Update(target3, unrelated)) + r.True(subject.Update(unrelated, target3)) + r.True(subject.Delete(target3)) }) }) @@ -285,6 +273,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { const loadBalancerServiceName = "some-service-resource-name" const tlsSecretName = "some-tls-secret-name" //nolint:gosec // this is not a credential const caSecretName = "some-ca-secret-name" + const caSignerName = "some-ca-signer-name" const localhostIP = "127.0.0.1" const httpsPort = ":443" const fakeServerResponseBody = "hello, world!" @@ -300,44 +289,139 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var cancelContext context.Context var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context - var startTLSListenerFuncWasCalled int - var startTLSListenerFuncError error - var startTLSListenerUponCloseError error - var httpHandlerFactoryFuncError error + var impersonatorFuncWasCalled int + var impersonatorFuncError error + var impersonatorFuncReturnedFuncError error var startedTLSListener net.Listener var frozenNow time.Time + var signingCertProvider dynamiccert.Provider + var signingCACertPEM, signingCAKeyPEM []byte + var signingCASecret *corev1.Secret + var testHTTPServer *http.Server + var testHTTPServerMutex sync.RWMutex + var testHTTPServerInterruptCh chan struct{} + var queue *testQueue + var validClientCert *tls.Certificate - var startTLSListenerFunc = func(network, listenAddress string, config *tls.Config) (net.Listener, error) { - startTLSListenerFuncWasCalled++ - r.Equal("tcp", network) - r.Equal(":8444", listenAddress) - r.Equal(uint16(tls.VersionTLS12), config.MinVersion) - if startTLSListenerFuncError != nil { - return nil, startTLSListenerFuncError + var impersonatorFunc = func( + port int, + dynamicCertProvider dynamiccertificates.CertKeyContentProvider, + impersonationProxySignerCAProvider dynamiccertificates.CAContentProvider, + ) (func(stopCh <-chan struct{}) error, error) { + impersonatorFuncWasCalled++ + r.Equal(8444, port) + r.NotNil(dynamicCertProvider) + r.NotNil(impersonationProxySignerCAProvider) + + if impersonatorFuncError != nil { + return nil, impersonatorFuncError } - var err error - startedTLSListener, err = tls.Listen(network, localhostIP+":0", config) // automatically choose the port for unit tests - r.NoError(err) - return &tlsListenerWrapper{listener: startedTLSListener, closeError: startTLSListenerUponCloseError}, nil + + // Return a func that starts a fake server when called, and shuts down the fake server when stopCh is closed. + // This fake server is enough like the real impersonation proxy server for this unit test because it + // uses the supplied providers to serve TLS. The goal of this unit test is to make sure that the server + // was started/stopped/configured correctly, not to test the actual impersonation behavior. + return func(stopCh <-chan struct{}) error { + if impersonatorFuncReturnedFuncError != nil { + return impersonatorFuncReturnedFuncError + } + + var err error + // automatically choose the port for unit tests + startedTLSListener, err = tls.Listen("tcp", localhostIP+":0", &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + certPEM, keyPEM := dynamicCertProvider.CurrentCertKeyContent() + if certPEM != nil && keyPEM != nil { + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + r.NoError(err) + return &tlsCert, nil + } + return nil, nil // no cached TLS certs + }, + ClientAuth: tls.RequestClientCert, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + // Docs say that this will always be called in tls.RequestClientCert mode + // and that the second parameter will always be nil in that case. + // rawCerts will be raw ASN.1 certificates provided by the peer. + if rawCerts == nil || len(rawCerts) != 1 { + return fmt.Errorf("expected to get one client cert on incoming request to test server") + } + clientCert := rawCerts[0] + currentClientCertCA := impersonationProxySignerCAProvider.CurrentCABundleContent() + if currentClientCertCA == nil { + return fmt.Errorf("impersonationProxySignerCAProvider does not have a current CA certificate") + } + // Assert that the client's cert was signed by the CA cert that the controller put into + // the CAContentProvider that was passed in. + parsed, err := x509.ParseCertificate(clientCert) + require.NoError(t, err) + t.Log("PARSED CLIENT CERT") + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(currentClientCertCA)) + opts := x509.VerifyOptions{Roots: roots} + _, err = parsed.Verify(opts) + require.NoError(t, err) + return nil + }, + }) + r.NoError(err) + + testHTTPServerMutex.Lock() // this is to satisfy the race detector + testHTTPServer = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + _, err := fmt.Fprint(w, fakeServerResponseBody) + r.NoError(err) + })} + testHTTPServerMutex.Unlock() + + // Start serving requests in the background. + go func() { + err := testHTTPServer.Serve(startedTLSListener) + if !errors.Is(err, http.ErrServerClosed) { + t.Log("Got an unexpected error while starting the fake http server!") + r.NoError(err) // causes the test to crash, which is good enough because this should never happen + } + }() + + if testHTTPServerInterruptCh == nil { + // Wait in the foreground for the stopCh to be closed, and kill the server when that happens. + // This is similar to the behavior of the real impersonation server. + <-stopCh + } else { + // The test supplied an interrupt channel because it wants to test unexpected termination + // of the server, so wait for that channel to close instead of waiting for the one that + // was passed in from the production code. + <-testHTTPServerInterruptCh + } + + err = testHTTPServer.Close() + t.Log("Got an unexpected error while stopping the fake http server!") + r.NoError(err) // causes the test to crash, which is good enough because this should never happen + + return nil + }, nil } var testServerAddr = func() string { + require.Eventually(t, func() bool { + return startedTLSListener != nil + }, 20*time.Second, 50*time.Millisecond, "TLS listener never became not nil") + return startedTLSListener.Addr().String() } - var closeTLSListener = func() { - if startedTLSListener != nil { - err := startedTLSListener.Close() - // Ignore when the production code has already closed the server because there is nothing to - // clean up in that case. - if err != nil && !strings.Contains(err.Error(), "use of closed network connection") { - r.NoError(err) - } + var closeTestHTTPServer = func() { + // If a test left it running, then close it. + testHTTPServerMutex.RLock() // this is to satisfy the race detector + defer testHTTPServerMutex.RUnlock() + if testHTTPServer != nil { + err := testHTTPServer.Close() + r.NoError(err) } } var requireTLSServerIsRunning = func(caCrt []byte, addr string, dnsOverrides map[string]string) { - r.Greater(startTLSListenerFuncWasCalled, 0) + r.Greater(impersonatorFuncWasCalled, 0) realDialer := &net.Dialer{} overrideDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { @@ -361,8 +445,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { rootCAs := x509.NewCertPool() rootCAs.AppendCertsFromPEM(caCrt) tr = &http.Transport{ - TLSClientConfig: &tls.Config{RootCAs: rootCAs}, - DialContext: overrideDialContext, + TLSClientConfig: &tls.Config{ + // Server's TLS serving cert CA + RootCAs: rootCAs, + // Client cert which is supposed to work against the server's dynamic CAContentProvider + Certificates: []tls.Certificate{*validClientCert}, + }, + DialContext: overrideDialContext, } } client := &http.Client{Transport: tr} @@ -385,7 +474,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var requireTLSServerIsRunningWithoutCerts = func() { - r.Greater(startTLSListenerFuncWasCalled, 0) + r.Greater(impersonatorFuncWasCalled, 0) tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec } @@ -406,14 +495,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var requireTLSServerIsNoLongerRunning = func() { - r.Greater(startTLSListenerFuncWasCalled, 0) + r.Greater(impersonatorFuncWasCalled, 0) var err error expectedErrorRegex := "dial tcp .*: connect: connection refused" expectedErrorRegexCompiled, err := regexp.Compile(expectedErrorRegex) r.NoError(err) assert.Eventually(t, func() bool { _, err = tls.Dial( - startedTLSListener.Addr().Network(), + "tcp", testServerAddr(), &tls.Config{InsecureSkipVerify: true}, //nolint:gosec ) @@ -424,7 +513,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var requireTLSServerWasNeverStarted = func() { - r.Equal(0, startTLSListenerFuncWasCalled) + r.Equal(0, impersonatorFuncWasCalled) } // Defer starting the informers until the last possible moment so that the @@ -447,13 +536,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { caSecretName, labels, clock.NewFakeClock(frozenNow), - startTLSListenerFunc, - func() (http.Handler, error) { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, err := fmt.Fprint(w, fakeServerResponseBody) - r.NoError(err) - }), httpHandlerFactoryFuncError - }, + impersonatorFunc, + caSignerName, + signingCertProvider, ) // Set this at the last second to support calling subject.Name(). @@ -464,6 +549,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { Namespace: installedInNamespace, Name: configMapResourceName, }, + Queue: queue, } // Must start informers before calling TestRunSynchronously() @@ -536,6 +622,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return newSecretWithData(resourceName, newTLSCertSecretData(ca, []string{"foo", "bar"}, ip)) } + var newSigningKeySecret = func(resourceName string, certPEM, keyPEM []byte) *corev1.Secret { + return newSecretWithData(resourceName, map[string][]byte{ + apicerts.CACertificateSecretKey: certPEM, + apicerts.CACertificatePrivateKeySecretKey: keyPEM, + }) + } + var newLoadBalancerService = func(resourceName string, status corev1.ServiceStatus) *corev1.Service { return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -842,12 +935,26 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { validCert.RequireLifetime(time.Now().Add(-10*time.Second), time.Now().Add(100*time.Hour*24*365), 10*time.Second) } + var requireSigningCertProviderHasLoadedCerts = func(certPEM, keyPEM []byte) { + actualCert, actualKey := signingCertProvider.CurrentCertKeyContent() + // Cast to string for better failure messages. + r.Equal(string(certPEM), string(actualCert)) + r.Equal(string(keyPEM), string(actualKey)) + } + + var requireSigningCertProviderIsEmpty = func() { + actualCert, actualKey := signingCertProvider.CurrentCertKeyContent() + r.Nil(actualCert) + r.Nil(actualKey) + } + var runControllerSync = func() error { return controllerlib.TestSync(t, subject, *syncContext) } it.Before(func() { r = require.New(t) + queue = &testQueue{} cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactoryWithOptions(kubeInformerClient, 0, @@ -856,15 +963,26 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { kubeAPIClient = kubernetesfake.NewSimpleClientset() pinnipedAPIClient = pinnipedfake.NewSimpleClientset() frozenNow = time.Date(2021, time.March, 2, 7, 42, 0, 0, time.Local) + signingCertProvider = dynamiccert.New() + + ca := newCA() + signingCACertPEM = ca.Bundle() + var err error + signingCAKeyPEM, err = ca.PrivateKeyToPEM() + r.NoError(err) + signingCASecret = newSigningKeySecret(caSignerName, signingCACertPEM, signingCAKeyPEM) + validClientCert, err = ca.Issue(pkix.Name{}, nil, nil, time.Hour) + r.NoError(err) }) it.After(func() { cancelContextCancelFunc() - closeTLSListener() + closeTestHTTPServer() }) when("the ConfigMap does not yet exist in the installation namespace or it was deleted (defaults to auto mode)", func() { it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) addImpersonatorConfigMapToTracker("some-other-unrelated-configmap", "foo: bar", kubeInformerClient) }) @@ -880,6 +998,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) requireCredentialIssuer(newAutoDisabledStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -900,6 +1019,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[1]) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[2]) requireCredentialIssuer(newAutoDisabledStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -917,6 +1037,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -935,6 +1056,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -953,6 +1075,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -970,6 +1093,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) requireCredentialIssuer(newErrorStrategy("could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name")) + requireSigningCertProviderIsEmpty() }) }) @@ -990,6 +1114,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) @@ -999,6 +1124,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1019,6 +1145,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) @@ -1028,6 +1155,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1048,6 +1176,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, firstHostname, map[string]string{firstHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) @@ -1057,6 +1186,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 3) // nothing changed requireCredentialIssuer(newSuccessStrategy(firstHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1082,6 +1212,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1106,6 +1237,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newErrorStrategy("error on delete")) + requireSigningCertProviderIsEmpty() }) }) @@ -1123,7 +1255,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addSecretToTrackers(tlsSecret, kubeAPIClient, kubeInformerClient) }) - it("returns an error and keeps running the proxy with the old cert", func() { + it("returns an error and keeps the proxy running but now without certs", func() { startInformersAndController() r.NoError(runControllerSync()) r.Len(kubeAPIClient.Actions(), 1) @@ -1135,13 +1267,18 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { errString := "could not find valid IP addresses or hostnames from load balancer some-namespace/some-service-resource-name" r.EqualError(runControllerSync(), errString) r.Len(kubeAPIClient.Actions(), 1) // no new actions - requireTLSServerIsRunning(caCrt, testServerAddr(), nil) + requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() }) }) }) when("the ConfigMap is already in the installation namespace", func() { + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + }) + when("the configuration is auto mode with an endpoint", func() { it.Before(func() { configMapYAML := fmt.Sprintf("{mode: auto, endpoint: %s}", localhostIP) @@ -1160,6 +1297,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) r.Len(kubeAPIClient.Actions(), 1) requireCredentialIssuer(newAutoDisabledStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -1177,6 +1315,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) }) @@ -1194,6 +1333,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) r.Len(kubeAPIClient.Actions(), 1) requireCredentialIssuer(newManuallyDisabledStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -1213,13 +1353,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() }) - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") + it("returns an error when the impersonation TLS server fails to start", func() { + impersonatorFuncError = errors.New("impersonation server start error") startInformersAndController() - r.EqualError(runControllerSync(), "tls error") - requireCredentialIssuer(newErrorStrategy("tls error")) + r.EqualError(runControllerSync(), "impersonation server start error") + requireCredentialIssuer(newErrorStrategy("impersonation server start error")) + requireSigningCertProviderIsEmpty() }) }) @@ -1239,13 +1381,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCASecretWasCreated(kubeAPIClient.Actions()[1]) requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() }) - it("returns an error when the tls listener fails to start", func() { - startTLSListenerFuncError = errors.New("tls error") + it("returns an error when the impersonation TLS server fails to start", func() { + impersonatorFuncError = errors.New("impersonation server start error") startInformersAndController() - r.EqualError(runControllerSync(), "tls error") - requireCredentialIssuer(newErrorStrategy("tls error")) + r.EqualError(runControllerSync(), "impersonation server start error") + requireCredentialIssuer(newErrorStrategy("impersonation server start error")) + requireSigningCertProviderIsEmpty() }) }) @@ -1271,6 +1415,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireNodesListed(kubeAPIClient.Actions()[0]) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1292,6 +1437,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1313,6 +1459,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeIPWithPort. requireTLSServerIsRunning(ca, fakeIPWithPort, map[string]string{fakeIPWithPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeIPWithPort, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1334,6 +1481,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostnameWithPort. requireTLSServerIsRunning(ca, fakeHostnameWithPort, map[string]string{fakeHostnameWithPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostnameWithPort, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1357,6 +1505,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) @@ -1372,9 +1521,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. deleteSecretFromTracker(tlsSecretName, kubeInformerClient) + waitForObjectToBeDeletedFromInformer(tlsSecretName, kubeInformers.Core().V1().Secrets()) addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[4], kubeInformers.Core().V1().Secrets()) // Switch the endpoint config back to an IP. @@ -1387,6 +1538,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1408,8 +1560,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) - // Simulate the informer cache's background update from its watch for the CA Secret. + // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) // Delete the TLS Secret that was just created from the Kube API server. Note that we never @@ -1423,6 +1576,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1444,8 +1598,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) - // Simulate the informer cache's background update from its watch for the CA Secret. + // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // Delete the CA Secret that was just created from the Kube API server. Note that we never @@ -1461,6 +1616,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) @@ -1480,8 +1636,9 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) - // Simulate the informer cache's background update from its watch for the CA Secret. + // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) // Simulate someone updating the CA Secret out of band, e.g. when a human edits it with kubectl. @@ -1505,6 +1662,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeHostname. requireTLSServerIsRunning(caCrt, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeHostname, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) when("deleting the TLS cert due to mismatched CA results in an error", func() { @@ -1522,6 +1680,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasDeleted(kubeAPIClient.Actions()[3]) // tried to delete cert but failed requireCredentialIssuer(newErrorStrategy("error on tls secret delete")) + requireSigningCertProviderIsEmpty() }) }) }) @@ -1543,6 +1702,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) @@ -1556,6 +1716,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 4) requireLoadBalancerWasDeleted(kubeAPIClient.Actions()[3]) requireCredentialIssuer(newManuallyDisabledStrategy()) + requireSigningCertProviderIsEmpty() deleteServiceFromTracker(loadBalancerServiceName, kubeInformerClient) waitForObjectToBeDeletedFromInformer(loadBalancerServiceName, kubeInformers.Core().V1().Services()) @@ -1568,25 +1729,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 5) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[4]) requireCredentialIssuer(newPendingStrategy()) - }) - - when("there is an error while shutting down the server", func() { - it.Before(func() { - startTLSListenerUponCloseError = errors.New("fake server close error") - }) - - it("returns the error from the sync function", func() { - startInformersAndController() - r.NoError(runControllerSync()) - requireTLSServerIsRunningWithoutCerts() - - // Update the configmap. - updateImpersonatorConfigMapInInformerAndWait(configMapResourceName, "mode: disabled", kubeInformers.Core().V1().ConfigMaps()) - - r.EqualError(runControllerSync(), "fake server close error") - requireTLSServerIsNoLongerRunning() - requireCredentialIssuer(newErrorStrategy("fake server close error")) - }) + requireSigningCertProviderIsEmpty() }) }) @@ -1608,6 +1751,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) @@ -1622,6 +1766,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasDeleted(kubeAPIClient.Actions()[4]) // the Secret was deleted because it contained a cert with the wrong IP requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Services()) @@ -1633,6 +1778,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 5) // no new actions while it is waiting for the load balancer's ingress requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() // Update the ingress of the LB in the informer's client and run Sync again. fakeIP := "127.0.0.123" @@ -1643,6 +1789,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Check that the server is running and that TLS certs that are being served are are for fakeIP. requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + httpsPort: testServerAddr()}) requireCredentialIssuer(newSuccessStrategy(fakeIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[5], kubeInformers.Core().V1().Secrets()) @@ -1658,12 +1805,14 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[8], ca) // recreated because the endpoint was updated, reused the old CA requireTLSServerIsRunning(ca, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) }) }) when("sync is called more than once", func() { it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) }) @@ -1676,15 +1825,17 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time - requireTLSServerIsRunningWithoutCerts() // still running + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time + requireTLSServerIsRunningWithoutCerts() // still running requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() r.Len(kubeAPIClient.Actions(), 3) // no new API calls }) @@ -1697,6 +1848,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) @@ -1705,20 +1857,22 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP}}, kubeInformers.Core().V1().Services()) r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time requireTLSServerIsRunning(ca, testServerAddr(), nil) // running with certs now requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Secrets()) r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started again + r.Equal(1, impersonatorFuncWasCalled) // wasn't started again r.Len(kubeAPIClient.Actions(), 4) // no more actions requireTLSServerIsRunning(ca, testServerAddr(), nil) // still running requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) it("creates certs from the hostname listed on the load balancer", func() { @@ -1731,6 +1885,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { ca := requireCASecretWasCreated(kubeAPIClient.Actions()[2]) requireTLSServerIsRunningWithoutCerts() requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) @@ -1739,20 +1894,56 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { updateLoadBalancerServiceInInformerAndWait(loadBalancerServiceName, []corev1.LoadBalancerIngress{{IP: localhostIP, Hostname: hostname}}, kubeInformers.Core().V1().Services()) r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a second time r.Len(kubeAPIClient.Actions(), 4) requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) // uses the ca from last time requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // running with certs now requireCredentialIssuer(newSuccessStrategy(hostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) // Simulate the informer cache's background update from its watch. addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[3], kubeInformers.Core().V1().Secrets()) r.NoError(runControllerSync()) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Equal(1, impersonatorFuncWasCalled) // wasn't started a third time r.Len(kubeAPIClient.Actions(), 4) // no more actions requireTLSServerIsRunning(ca, hostname, map[string]string{hostname + httpsPort: testServerAddr()}) // still running requireCredentialIssuer(newSuccessStrategy(hostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + }) + }) + + when("there is already a CredentialIssuer", func() { + preExistingStrategy := v1alpha1.CredentialIssuerStrategy{ + Type: v1alpha1.KubeClusterSigningCertificateStrategyType, + Status: v1alpha1.SuccessStrategyStatus, + Reason: v1alpha1.FetchedKeyStrategyReason, + Message: "happy other unrelated strategy", + LastUpdateTime: metav1.NewTime(frozenNow), + Frontend: &v1alpha1.CredentialIssuerFrontend{ + Type: v1alpha1.TokenCredentialRequestAPIFrontendType, + }, + } + + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + r.NoError(pinnipedAPIClient.Tracker().Add(&v1alpha1.CredentialIssuer{ + ObjectMeta: metav1.ObjectMeta{Name: credentialIssuerResourceName}, + Status: v1alpha1.CredentialIssuerStatus{Strategies: []v1alpha1.CredentialIssuerStrategy{preExistingStrategy}}, + })) + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("merges into the existing strategy array on the CredentialIssuer", func() { + startInformersAndController() + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + credentialIssuer := getCredentialIssuer() + r.Equal([]v1alpha1.CredentialIssuerStrategy{preExistingStrategy, newPendingStrategy()}, credentialIssuer.Status.Strategies) }) }) @@ -1761,21 +1952,110 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.EqualError(runControllerSync(), "no nodes found") requireCredentialIssuer(newErrorStrategy("no nodes found")) + requireSigningCertProviderIsEmpty() requireTLSServerWasNeverStarted() }) }) - when("the http handler factory function returns an error", func() { + when("the impersonator start function returned by the impersonatorFunc returns an error immediately", func() { it.Before(func() { addNodeWithRoleToTracker("worker", kubeAPIClient) - httpHandlerFactoryFuncError = errors.New("some factory error") + impersonatorFuncReturnedFuncError = errors.New("some immediate impersonator startup error") }) - it("returns an error", func() { + it("causes an immediate resync, returns an error on that next sync, and then restarts the server in a following sync", func() { startInformersAndController() - r.EqualError(runControllerSync(), "some factory error") - requireCredentialIssuer(newErrorStrategy("some factory error")) - requireTLSServerWasNeverStarted() + // The failure happens in a background goroutine, so the first sync succeeds. + r.NoError(runControllerSync()) + // Eventually the server is not really running, because the startup failed. + r.Nil(startedTLSListener) + r.Equal(impersonatorFuncWasCalled, 1) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // The controller's first sync should have started a background routine which, when the server dies, + // requests to re-enqueue the original sync key to cause its sync method to get called again in the near future. + r.Eventually(func() bool { + queue.mutex.RLock() // this is to satisfy the race detector + defer queue.mutex.RUnlock() + return syncContext.Key == queue.key + }, 10*time.Second, 10*time.Millisecond) + + // The next sync should error because the server died in the background. This second + // sync should be able to detect the error and return it. + r.EqualError(runControllerSync(), "some immediate impersonator startup error") + requireCredentialIssuer(newErrorStrategy("some immediate impersonator startup error")) + requireSigningCertProviderIsEmpty() + + // Next time the controller starts the server, the server will start successfully. + impersonatorFuncReturnedFuncError = nil + + // One more sync and the controller should try to restart the server. + // Now everything should be working correctly. + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the impersonator server dies for no apparent reason after running for a while", func() { + it.Before(func() { + addNodeWithRoleToTracker("worker", kubeAPIClient) + }) + + it("causes an immediate resync, returns an error on that next sync, and then restarts the server in a following sync", func() { + // Prepare to be able to cause the server to die for no apparent reason. + testHTTPServerInterruptCh = make(chan struct{}) + + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + requireCASecretWasCreated(kubeAPIClient.Actions()[2]) + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() + requireTLSServerIsRunningWithoutCerts() + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Services()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Simulate that impersonation server dies for no apparent reason. + close(testHTTPServerInterruptCh) + + // The controller's first sync should have started a background routine which, when the server dies, + // requests to re-enqueue the original sync key to cause its sync method to get called again in the near future. + r.Eventually(func() bool { + queue.mutex.RLock() // this is to satisfy the race detector + defer queue.mutex.RUnlock() + return syncContext.Key == queue.key + }, 10*time.Second, 10*time.Millisecond) + + // The next sync should error because the server died in the background. This second + // sync should be able to detect the error and return it. + r.EqualError(runControllerSync(), "unexpected shutdown of proxy server") + requireCredentialIssuer(newErrorStrategy("unexpected shutdown of proxy server")) + requireSigningCertProviderIsEmpty() + + // Next time the controller starts the server, the server should behave as normal. + testHTTPServerInterruptCh = nil + + // One more sync and the controller should try to restart the server. + // Now everything should be working correctly. + r.NoError(runControllerSync()) + requireTLSServerIsRunningWithoutCerts() + requireCredentialIssuer(newPendingStrategy()) + requireSigningCertProviderIsEmpty() }) }) @@ -1789,6 +2069,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { errString := "invalid impersonator configuration: decode yaml: error unmarshaling JSON: while decoding JSON: json: cannot unmarshal string into Go value of type impersonator.Config" r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() requireTLSServerWasNeverStarted() }) }) @@ -1805,6 +2086,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.EqualError(runControllerSync(), "error on create") requireCredentialIssuer(newErrorStrategy("error on create")) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() }) }) @@ -1826,6 +2108,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.EqualError(runControllerSync(), "error on tls secret create") requireCredentialIssuer(newErrorStrategy("error on tls secret create")) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1851,6 +2134,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() r.EqualError(runControllerSync(), "error on ca secret create") requireCredentialIssuer(newErrorStrategy("error on ca secret create")) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1870,6 +2154,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { errString := "could not load CA: tls: failed to find any PEM data in certificate input" r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1891,6 +2176,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("does not start the impersonator, deletes the loadbalancer, returns an error", func() { r.EqualError(runControllerSync(), "error on delete") requireCredentialIssuer(newErrorStrategy("error on delete")) + requireSigningCertProviderIsEmpty() requireTLSServerWasNeverStarted() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1901,6 +2187,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("the PEM formatted data in the TLS Secret is not a valid cert", func() { it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", localhostIP) addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) @@ -1927,6 +2214,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[3], ca) requireTLSServerIsRunning(ca, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1941,6 +2229,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { errString := "PEM data represented an invalid cert, but got error while deleting it: error on delete" r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -1954,6 +2243,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a tls secret already exists but it is not valid", func() { var caCrt []byte it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) ca := newCA() @@ -1974,6 +2264,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) when("there is an error while the invalid cert is being deleted", func() { @@ -1988,6 +2279,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { errString := "found missing or not PEM-encoded data in TLS Secret, but got error while deleting it: error on delete" r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -2000,6 +2292,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("a tls secret already exists but the private key is not valid", func() { var caCrt []byte it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled", kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) ca := newCA() @@ -2022,6 +2315,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], caCrt) requireTLSServerIsRunning(caCrt, testServerAddr(), nil) requireCredentialIssuer(newSuccessStrategy(localhostIP, caCrt)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) }) when("there is an error while the invalid cert is being deleted", func() { @@ -2036,6 +2330,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { errString := "cert had an invalid private key, but got error while deleting it: error on delete" r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() requireTLSServerIsRunningWithoutCerts() r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) @@ -2047,6 +2342,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { when("there is an error while creating or updating the CredentialIssuer status", func() { it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) pinnipedAPIClient.PrependReactor("create", "credentialissuers", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { return true, nil, fmt.Errorf("error on create") @@ -2072,37 +2368,110 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("there is already a CredentialIssuer", func() { - preExistingStrategy := v1alpha1.CredentialIssuerStrategy{ - Type: v1alpha1.KubeClusterSigningCertificateStrategyType, - Status: v1alpha1.SuccessStrategyStatus, - Reason: v1alpha1.FetchedKeyStrategyReason, - Message: "happy other unrelated strategy", - LastUpdateTime: metav1.NewTime(frozenNow), - Frontend: &v1alpha1.CredentialIssuerFrontend{ - Type: v1alpha1.TokenCredentialRequestAPIFrontendType, - }, - } - + when("the impersonator is ready but there is a problem with the signing secret, which should be created by another controller", func() { + const fakeHostname = "foo.example.com" it.Before(func() { - r.NoError(pinnipedAPIClient.Tracker().Add(&v1alpha1.CredentialIssuer{ - ObjectMeta: metav1.ObjectMeta{Name: credentialIssuerResourceName}, - Status: v1alpha1.CredentialIssuerStatus{Strategies: []v1alpha1.CredentialIssuerStrategy{preExistingStrategy}}, - })) + configMapYAML := fmt.Sprintf("{mode: enabled, endpoint: %s}", fakeHostname) + addImpersonatorConfigMapToTracker(configMapResourceName, configMapYAML, kubeInformerClient) addNodeWithRoleToTracker("worker", kubeAPIClient) }) - it("merges into the existing strategy array on the CredentialIssuer", func() { - startInformersAndController() - r.NoError(runControllerSync()) - requireTLSServerIsRunningWithoutCerts() - r.Len(kubeAPIClient.Actions(), 3) - requireNodesListed(kubeAPIClient.Actions()[0]) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) - requireCASecretWasCreated(kubeAPIClient.Actions()[2]) - credentialIssuer := getCredentialIssuer() - r.Equal([]v1alpha1.CredentialIssuerStrategy{preExistingStrategy, newPendingStrategy()}, credentialIssuer.Status.Strategies) + when("it does not exist in the informers", func() { + it("returns the error", func() { + startInformersAndController() + errString := `could not load the impersonator's credential signing secret: secret "some-ca-signer-name" not found` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + + when("it does not have the expected fields", func() { + it.Before(func() { + addSecretToTrackers(newEmptySecret(caSignerName), kubeInformerClient) + }) + + it("returns the error", func() { + startInformersAndController() + errString := `could not load the impersonator's credential signing secret: tls: failed to find any PEM data in certificate input` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the cert is invalid", func() { + it.Before(func() { + signingCASecret.Data[apicerts.CACertificateSecretKey] = []byte("not a valid PEM formatted cert") + addSecretToTrackers(signingCASecret, kubeInformerClient) + }) + + it("returns the error", func() { + startInformersAndController() + errString := `could not load the impersonator's credential signing secret: tls: failed to find any PEM data in certificate input` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) + }) + + when("the cert goes from being valid to being invalid", func() { + const fakeHostname = "foo.example.com" + it.Before(func() { + addSecretToTrackers(signingCASecret, kubeInformerClient) + }) + + it("returns the error and clears the dynamic provider", func() { + startInformersAndController() + r.NoError(runControllerSync()) + r.Len(kubeAPIClient.Actions(), 3) + requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireCASecretWasCreated(kubeAPIClient.Actions()[1]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[2], ca) + // Check that the server is running and that TLS certs that are being served are are for fakeHostname. + requireTLSServerIsRunning(ca, fakeHostname, map[string]string{fakeHostname + httpsPort: testServerAddr()}) + requireCredentialIssuer(newSuccessStrategy(fakeHostname, ca)) + requireSigningCertProviderHasLoadedCerts(signingCACertPEM, signingCAKeyPEM) + + // Simulate the informer cache's background update from its watch. + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[1], kubeInformers.Core().V1().Secrets()) + addObjectFromCreateActionToInformerAndWait(kubeAPIClient.Actions()[2], kubeInformers.Core().V1().Secrets()) + + // Now update the signer CA to something invalid. + deleteSecretFromTracker(caSignerName, kubeInformerClient) + waitForObjectToBeDeletedFromInformer(caSignerName, kubeInformers.Core().V1().Secrets()) + updatedSigner := newEmptySecret(caSignerName) + addSecretToTrackers(updatedSigner, kubeInformerClient) + waitForObjectToAppearInInformer(updatedSigner, kubeInformers.Core().V1().Secrets()) + + errString := `could not load the impersonator's credential signing secret: tls: failed to find any PEM data in certificate input` + r.EqualError(runControllerSync(), errString) + requireCredentialIssuer(newErrorStrategy(errString)) + requireSigningCertProviderIsEmpty() + }) }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) } + +type testQueue struct { + key controllerlib.Key + mutex sync.RWMutex + + controllerlib.Queue +} + +func (q *testQueue) AddRateLimited(key controllerlib.Key) { + q.mutex.Lock() // this is to satisfy the race detector + defer q.mutex.Unlock() + + if q.key != (controllerlib.Key{}) { + panic("called more than once") + } + + if key == (controllerlib.Key{}) { + panic("unexpected empty key") + } + + q.key = key +} diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index d727a1bc..5af528e4 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -7,12 +7,9 @@ package controllermanager import ( "context" - "crypto/tls" "fmt" - "net/http" "time" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/clock" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" @@ -67,9 +64,11 @@ type Config struct { // DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert. DynamicServingCertProvider dynamiccert.Provider - // DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's + // DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's // TODO fix comment // signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login. DynamicSigningCertProvider dynamiccert.Provider + // TODO fix comment + ImpersonationSigningCertProvider dynamiccert.Provider // ServingCertDuration is the validity period, in seconds, of the API serving certificate. ServingCertDuration time.Duration @@ -81,10 +80,6 @@ type Config struct { // AuthenticatorCache is a cache of authenticators shared amongst various authenticated-related controllers. AuthenticatorCache *authncache.Cache - // LoginJSONDecoder can decode login.concierge.pinniped.dev types (e.g., TokenCredentialRequest) - // into their internal representation. - LoginJSONDecoder runtime.Decoder - // Labels are labels that should be added to any resources created by the controllers. Labels map[string]string } @@ -188,6 +183,7 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { informers.installationNamespaceK8s.Core().V1().Secrets(), controllerlib.WithInformer, c.ServingCertRenewBefore, + apicerts.TLSCertificateChainSecretKey, ), singletonWorker, ). @@ -295,18 +291,36 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { c.NamesConfig.ImpersonationCACertificateSecret, c.Labels, clock.RealClock{}, - tls.Listen, - func() (http.Handler, error) { - impersonationProxyHandler, err := impersonator.New( - c.AuthenticatorCache, - c.LoginJSONDecoder, - klogr.New().WithName("impersonation-proxy"), - ) - if err != nil { - return nil, fmt.Errorf("could not create impersonation proxy: %w", err) - } - return impersonationProxyHandler, nil - }, + impersonator.New, + c.NamesConfig.ImpersonationSignerSecret, + c.ImpersonationSigningCertProvider, + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsManagerController( + c.ServerInstallationInfo.Namespace, + c.NamesConfig.ImpersonationSignerSecret, + c.Labels, + client.Kubernetes, + informers.installationNamespaceK8s.Core().V1().Secrets(), + controllerlib.WithInformer, + controllerlib.WithInitialEvent, + 365*24*time.Hour, // 1 year hard coded value + "Pinniped Impersonation Proxy CA", + "", // optional, means do not give me a serving cert + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsExpirerController( + c.ServerInstallationInfo.Namespace, + c.NamesConfig.ImpersonationSignerSecret, + client.Kubernetes, + informers.installationNamespaceK8s.Core().V1().Secrets(), + controllerlib.WithInformer, + c.ServingCertRenewBefore, + apicerts.CACertificateSecretKey, ), singletonWorker, ) diff --git a/internal/dynamiccert/provider.go b/internal/dynamiccert/provider.go index a4ca6ad5..77687fb1 100644 --- a/internal/dynamiccert/provider.go +++ b/internal/dynamiccert/provider.go @@ -1,9 +1,10 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package dynamiccert import ( + "crypto/x509" "sync" "k8s.io/apiserver/pkg/server/dynamiccertificates" @@ -13,6 +14,8 @@ import ( // certificate and matching key. type Provider interface { dynamiccertificates.CertKeyContentProvider + // TODO dynamiccertificates.Notifier + // TODO dynamiccertificates.ControllerRunner ??? Set(certPEM, keyPEM []byte) } @@ -43,3 +46,27 @@ func (p *provider) CurrentCertKeyContent() (cert []byte, key []byte) { defer p.mutex.RUnlock() return p.certPEM, p.keyPEM } + +func NewCAProvider(delegate dynamiccertificates.CertKeyContentProvider) dynamiccertificates.CAContentProvider { + return &caContentProvider{delegate: delegate} +} + +type caContentProvider struct { + delegate dynamiccertificates.CertKeyContentProvider +} + +func (c *caContentProvider) Name() string { + return "DynamicCAProvider" +} + +func (c *caContentProvider) CurrentCABundleContent() []byte { + ca, _ := c.delegate.CurrentCertKeyContent() + return ca +} + +func (c *caContentProvider) VerifyOptions() (x509.VerifyOptions, bool) { + return x509.VerifyOptions{}, false // assume we are unioned via dynamiccertificates.NewUnionCAContentProvider +} + +// TODO look at both the serving side union struct and the ca side union struct for all optional interfaces +// and then implement everything that makes sense for us to implement diff --git a/internal/issuer/issuer.go b/internal/issuer/issuer.go new file mode 100644 index 00000000..94e2f235 --- /dev/null +++ b/internal/issuer/issuer.go @@ -0,0 +1,42 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package issuer + +import ( + "crypto/x509/pkix" + "time" + + "k8s.io/apimachinery/pkg/util/errors" + + "go.pinniped.dev/internal/constable" +) + +const defaultCertIssuerErr = constable.Error("failed to issue cert") + +type CertIssuer interface { + IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) (certPEM, keyPEM []byte, err error) +} + +var _ CertIssuer = CertIssuers{} + +type CertIssuers []CertIssuer + +func (c CertIssuers) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { + var errs []error + + for _, issuer := range c { + certPEM, keyPEM, err := issuer.IssuePEM(subject, dnsNames, ttl) + if err != nil { + errs = append(errs, err) + continue + } + return certPEM, keyPEM, nil + } + + if err := errors.NewAggregate(errs); err != nil { + return nil, nil, err + } + + return nil, nil, defaultCertIssuerErr +} diff --git a/internal/registry/credentialrequest/rest.go b/internal/registry/credentialrequest/rest.go index a8906676..a1846a40 100644 --- a/internal/registry/credentialrequest/rest.go +++ b/internal/registry/credentialrequest/rest.go @@ -22,20 +22,17 @@ import ( "k8s.io/utils/trace" loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" + "go.pinniped.dev/internal/issuer" ) // clientCertificateTTL is the TTL for short-lived client certificates returned by this API. const clientCertificateTTL = 5 * time.Minute -type CertIssuer interface { - IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) -} - type TokenCredentialRequestAuthenticator interface { AuthenticateTokenCredentialRequest(ctx context.Context, req *loginapi.TokenCredentialRequest) (user.Info, error) } -func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer CertIssuer, resource schema.GroupResource) *REST { +func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer issuer.CertIssuer, resource schema.GroupResource) *REST { return &REST{ authenticator: authenticator, issuer: issuer, @@ -45,7 +42,7 @@ func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer CertIssue type REST struct { authenticator TokenCredentialRequestAuthenticator - issuer CertIssuer + issuer issuer.CertIssuer tableConvertor rest.TableConvertor } diff --git a/internal/registry/credentialrequest/rest_test.go b/internal/registry/credentialrequest/rest_test.go index 8542b99e..6b71ec20 100644 --- a/internal/registry/credentialrequest/rest_test.go +++ b/internal/registry/credentialrequest/rest_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/klog/v2" loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/mocks/credentialrequestmocks" "go.pinniped.dev/internal/testutil" ) @@ -353,12 +354,12 @@ func requireSuccessfulResponseWithAuthenticationFailureMessage(t *testing.T, err }) } -func successfulIssuer(ctrl *gomock.Controller) CertIssuer { - issuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - issuer.EXPECT(). +func successfulIssuer(ctrl *gomock.Controller) issuer.CertIssuer { + certIssuer := credentialrequestmocks.NewMockCertIssuer(ctrl) + certIssuer.EXPECT(). IssuePEM(gomock.Any(), gomock.Any(), gomock.Any()). Return([]byte("test-cert"), []byte("test-key"), nil) - return issuer + return certIssuer } func stringPtr(s string) *string { diff --git a/internal/testutil/impersonationtoken/impersonationtoken.go b/internal/testutil/impersonationtoken/impersonationtoken.go deleted file mode 100644 index 64b65378..00000000 --- a/internal/testutil/impersonationtoken/impersonationtoken.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -// Package impersonationtoken contains a test utility to generate a token to be used against our -// impersonation proxy. -// -// It is its own package to fix import cycles involving concierge/scheme, testutil, and groupsuffix. -package impersonationtoken - -import ( - "encoding/base64" - "testing" - - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/serializer" - - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" - conciergescheme "go.pinniped.dev/internal/concierge/scheme" - "go.pinniped.dev/internal/groupsuffix" -) - -func Make( - t *testing.T, - token string, - authenticator *corev1.TypedLocalObjectReference, - apiGroupSuffix string, -) string { - t.Helper() - - // The impersonation test token should be a base64-encoded TokenCredentialRequest object. The API - // group of the TokenCredentialRequest object, and its Spec.Authenticator, should match whatever - // is installed on the cluster. This API group is usually replaced by the kubeclient middleware, - // but this object is not touched by the middleware since it is in a HTTP header. Therefore, we - // need to make a manual edit here. - scheme, loginGV, _ := conciergescheme.New(apiGroupSuffix) - tokenCredentialRequest := loginv1alpha1.TokenCredentialRequest{ - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginGV.Group + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: token, - Authenticator: *authenticator.DeepCopy(), - }, - } - - // It is assumed that the provided authenticator uses the default pinniped.dev API group, since - // this is usually replaced by the kubeclient middleware. Since we are not going through the - // kubeclient middleware, we need to make this replacement ourselves. - require.NotNil(t, tokenCredentialRequest.Spec.Authenticator.APIGroup, "expected authenticator to have non-nil API group") - authenticatorAPIGroup, ok := groupsuffix.Replace(*tokenCredentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - require.True(t, ok, "couldn't replace suffix of %q with %q", *tokenCredentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - tokenCredentialRequest.Spec.Authenticator.APIGroup = &authenticatorAPIGroup - - codecs := serializer.NewCodecFactory(scheme) - respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeJSON) - require.True(t, ok, "couldn't find serializer info for media type") - - reqJSON, err := runtime.Encode(respInfo.PrettySerializer, &tokenCredentialRequest) - require.NoError(t, err) - return base64.StdEncoding.EncodeToString(reqJSON) -} diff --git a/test/integration/concierge_credentialrequest_test.go b/test/integration/concierge_credentialrequest_test.go index de48bf85..196670d4 100644 --- a/test/integration/concierge_credentialrequest_test.go +++ b/test/integration/concierge_credentialrequest_test.go @@ -30,11 +30,13 @@ func TestUnsuccessfulCredentialRequest(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - response, err := makeRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, corev1.TypedLocalObjectReference{ - APIGroup: &auth1alpha1.SchemeGroupVersion.Group, - Kind: "WebhookAuthenticator", - Name: "some-webhook-that-does-not-exist", - })) + response, err := library.CreateTokenCredentialRequest(ctx, t, + validCredentialRequestSpecWithRealToken(t, corev1.TypedLocalObjectReference{ + APIGroup: &auth1alpha1.SchemeGroupVersion.Group, + Kind: "WebhookAuthenticator", + Name: "some-webhook-that-does-not-exist", + }), + ) require.NoError(t, err) require.Nil(t, response.Status.Credential) require.NotNil(t, response.Status.Message) @@ -88,10 +90,9 @@ func TestSuccessfulCredentialRequest(t *testing.T) { var response *loginv1alpha1.TokenCredentialRequest successfulResponse := func() bool { var err error - response, err = makeRequest(ctx, t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: token, - Authenticator: authenticator, - }) + response, err = library.CreateTokenCredentialRequest(ctx, t, + loginv1alpha1.TokenCredentialRequestSpec{Token: token, Authenticator: authenticator}, + ) require.NoError(t, err, "the request should never fail at the HTTP level") return response.Status.Credential != nil } @@ -141,10 +142,9 @@ func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthentic defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: "not a good token", - Authenticator: testWebhook, - }) + response, err := library.CreateTokenCredentialRequest(context.Background(), t, + loginv1alpha1.TokenCredentialRequestSpec{Token: "not a good token", Authenticator: testWebhook}, + ) require.NoError(t, err) @@ -164,10 +164,9 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T defer cancel() testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: "", - Authenticator: testWebhook, - }) + response, err := library.CreateTokenCredentialRequest(context.Background(), t, + loginv1alpha1.TokenCredentialRequestSpec{Token: "", Authenticator: testWebhook}, + ) require.Error(t, err) statusError, isStatus := err.(*errors.StatusError) @@ -193,7 +192,7 @@ func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheCl testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - response, err := makeRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, testWebhook)) + response, err := library.CreateTokenCredentialRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, testWebhook)) require.NoError(t, err) @@ -202,22 +201,6 @@ func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheCl require.Equal(t, stringPtr("authentication failed"), response.Status.Message) } -func makeRequest(ctx context.Context, t *testing.T, spec loginv1alpha1.TokenCredentialRequestSpec) (*loginv1alpha1.TokenCredentialRequest, error) { - t.Helper() - env := library.IntegrationEnv(t) - - client := library.NewAnonymousConciergeClientset(t) - - ctx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - - return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, &loginv1alpha1.TokenCredentialRequest{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{Namespace: env.ConciergeNamespace}, - Spec: spec, - }, metav1.CreateOptions{}) -} - func validCredentialRequestSpecWithRealToken(t *testing.T, authenticator corev1.TypedLocalObjectReference) loginv1alpha1.TokenCredentialRequestSpec { return loginv1alpha1.TokenCredentialRequestSpec{ Token: library.IntegrationEnv(t).TestUser.Token, diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 06f0a55d..a48e3203 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -10,6 +10,7 @@ import ( "crypto/x509" "encoding/base64" "encoding/json" + "encoding/pem" "fmt" "io/ioutil" "net/http" @@ -21,9 +22,8 @@ import ( "testing" "time" - "golang.org/x/net/websocket" - "github.com/stretchr/testify/require" + "golang.org/x/net/websocket" v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -38,10 +38,10 @@ import ( "sigs.k8s.io/yaml" "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/concierge/impersonator" "go.pinniped.dev/internal/testutil" - "go.pinniped.dev/internal/testutil/impersonationtoken" "go.pinniped.dev/test/library" ) @@ -49,7 +49,7 @@ import ( // - load balancers not supported, has squid proxy (e.g. kind) // - load balancers supported, has squid proxy (e.g. EKS) // - load balancers supported, no squid proxy (e.g. GKE) -func TestImpersonationProxy(t *testing.T) { +func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's complex. env := library.IntegrationEnv(t) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute) @@ -64,14 +64,67 @@ func TestImpersonationProxy(t *testing.T) { // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) + expectedProxyServiceEndpointURL := "https://" + proxyServiceEndpoint // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) - impersonationProxyRestConfig := func(host string, caData []byte, doubleImpersonateUser string) *rest.Config { + credentialAlmostExpired := func(credential *loginv1alpha1.TokenCredentialRequest) bool { + pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) + parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) + require.NoError(t, err) + timeRemaining := time.Until(parsedCredential.NotAfter) + if timeRemaining < 2*time.Minute { + t.Logf("The TokenCredentialRequest cred is almost expired and needs to be refreshed. Expires in %s.", timeRemaining) + return true + } + t.Logf("The TokenCredentialRequest cred is good for some more time (%s) so using it.", timeRemaining) + return false + } + + var tokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest + refreshCredential := func() *loginv1alpha1.ClusterCredential { + if tokenCredentialRequestResponse == nil || credentialAlmostExpired(tokenCredentialRequestResponse) { + var err error + // Make a TokenCredentialRequest. This can either return a cert signed by the Kube API server's CA (e.g. on kind) + // or a cert signed by the impersonator's signing CA (e.g. on GKE). Either should be accepted by the impersonation + // proxy server as a valid authentication. + // + // However, we issue short-lived certs, so this cert will only be valid for a few minutes. + // Cache it until it is almost expired and then refresh it whenever it is close to expired. + tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: authenticator, + }) + require.NoError(t, err) + + require.Empty(t, tokenCredentialRequestResponse.Status.Message) // no error message + require.NotEmpty(t, tokenCredentialRequestResponse.Status.Credential.ClientCertificateData) + require.NotEmpty(t, tokenCredentialRequestResponse.Status.Credential.ClientKeyData) + + // At the moment the credential request should not have returned a token. In the future, if we make it return + // tokens, we should revisit this test's rest config below. + require.Empty(t, tokenCredentialRequestResponse.Status.Credential.Token) + } + return tokenCredentialRequestResponse.Status.Credential + } + + impersonationProxyRestConfig := func(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, doubleImpersonateUser string) *rest.Config { config := rest.Config{ - Host: host, - TLSClientConfig: rest.TLSClientConfig{Insecure: caData == nil, CAData: caData}, - BearerToken: impersonationtoken.Make(t, env.TestUser.Token, &authenticator, env.APIGroupSuffix), + Host: host, + TLSClientConfig: rest.TLSClientConfig{ + Insecure: caData == nil, + CAData: caData, + CertData: []byte(credential.ClientCertificateData), + KeyData: []byte(credential.ClientKeyData), + }, + // kubectl would set both the client cert and the token, so we'll do that too. + // The Kube API server will ignore the token if the client cert successfully authenticates. + // Only if the client cert is not present or fails to authenticate will it use the token. + // Historically, it works that way because some web browsers will always send your + // corporate-assigned client cert even if it is not valid, and it doesn't want to treat + // that as a failure if you also sent a perfectly good token. + // We would like the impersonation proxy to imitate that behavior, so we test it here. + BearerToken: "this is not valid", } if doubleImpersonateUser != "" { config.Impersonate = rest.ImpersonationConfig{UserName: doubleImpersonateUser} @@ -79,9 +132,9 @@ func TestImpersonationProxy(t *testing.T) { return &config } - impersonationProxyViaSquidClient := func(caData []byte, doubleImpersonateUser string) kubernetes.Interface { + impersonationProxyViaSquidClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { t.Helper() - kubeconfig := impersonationProxyRestConfig("https://"+proxyServiceEndpoint, caData, doubleImpersonateUser) + kubeconfig := impersonationProxyRestConfig(refreshCredential(), proxyURL, caData, doubleImpersonateUser) kubeconfig.Proxy = func(req *http.Request) (*url.URL, error) { proxyURL, err := url.Parse(env.Proxy) require.NoError(t, err) @@ -93,10 +146,17 @@ func TestImpersonationProxy(t *testing.T) { impersonationProxyViaLoadBalancerClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { t.Helper() - kubeconfig := impersonationProxyRestConfig(proxyURL, caData, doubleImpersonateUser) + kubeconfig := impersonationProxyRestConfig(refreshCredential(), proxyURL, caData, doubleImpersonateUser) return library.NewKubeclient(t, kubeconfig).Kubernetes } + newImpersonationProxyClient := func(proxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) kubernetes.Interface { + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + return impersonationProxyViaLoadBalancerClient(proxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + } + return impersonationProxyViaSquidClient(proxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + } + oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName(env), metav1.GetOptions{}) if !k8serrors.IsNotFound(err) { require.NoError(t, err) // other errors aside from NotFound are unexpected @@ -142,7 +202,7 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. - _, err = impersonationProxyViaSquidClient(nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidClient(expectedProxyServiceEndpointURL, nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableViaSquidError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). @@ -161,29 +221,30 @@ func TestImpersonationProxy(t *testing.T) { // in the strategies array or it may be included in an error state. It can be in an error state for // awhile when it is waiting for the load balancer to be assigned an ip/hostname. impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminConciergeClient) - - // Create an impersonation proxy client with that CA data to use for the rest of this test. - // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. - var impersonationProxyClient kubernetes.Interface - if env.HasCapability(library.HasExternalLoadBalancerProvider) { - impersonationProxyClient = impersonationProxyViaLoadBalancerClient(impersonationProxyURL, impersonationProxyCACertPEM, "") - } else { + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { // In this case, we specified the endpoint in the configmap, so check that it was reported correctly in the CredentialIssuer. require.Equal(t, "https://"+proxyServiceEndpoint, impersonationProxyURL) - impersonationProxyClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "") + } + + // Because our credentials expire so quickly, we'll always use a new client, to give us a chance to refresh our + // credentials before they expire. Create a closure to capture the arguments to newImpersonationProxyClient + // so we don't have to keep repeating them. + // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. + impersonationProxyClient := func() kubernetes.Interface { + return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "") } // Test that the user can perform basic actions through the client with their username and group membership // influencing RBAC checks correctly. t.Run( "access as user", - library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyClient), + library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyClient()), ) for _, group := range env.TestUser.ExpectedGroups { group := group t.Run( "access as group "+group, - library.AccessAsGroupTest(ctx, group, impersonationProxyClient), + library.AccessAsGroupTest(ctx, group, impersonationProxyClient()), ) } @@ -201,7 +262,6 @@ func TestImpersonationProxy(t *testing.T) { // Try more Kube API verbs through the impersonation proxy. t.Run("watching all the basic verbs", func(t *testing.T) { - // Create an RBAC rule to allow this user to read/write everything. library.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, @@ -214,7 +274,7 @@ func TestImpersonationProxy(t *testing.T) { // Create and start informer to exercise the "watch" verb for us. informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( - impersonationProxyClient, + impersonationProxyClient(), 0, k8sinformers.WithNamespace(namespace.Name)) informer := informerFactory.Core().V1().ConfigMaps() @@ -234,17 +294,17 @@ func TestImpersonationProxy(t *testing.T) { } // Test "create" verb through the impersonation proxy. - _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) - _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) - _, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, metav1.CreateOptions{}, ) @@ -260,11 +320,11 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 50*time.Millisecond) // Test "get" verb through the impersonation proxy. - configMap3, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Get(ctx, "configmap-3", metav1.GetOptions{}) + configMap3, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Get(ctx, "configmap-3", metav1.GetOptions{}) require.NoError(t, err) // Test "list" verb through the impersonation proxy. - listResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + listResult, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) @@ -272,7 +332,7 @@ func TestImpersonationProxy(t *testing.T) { // Test "update" verb through the impersonation proxy. configMap3.Data = map[string]string{"foo": "bar"} - updateResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Update(ctx, configMap3, metav1.UpdateOptions{}) + updateResult, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Update(ctx, configMap3, metav1.UpdateOptions{}) require.NoError(t, err) require.Equal(t, "bar", updateResult.Data["foo"]) @@ -283,7 +343,7 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 50*time.Millisecond) // Test "patch" verb through the impersonation proxy. - patchResult, err := impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Patch(ctx, + patchResult, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Patch(ctx, "configmap-3", types.MergePatchType, []byte(`{"data":{"baz":"42"}}`), @@ -300,7 +360,7 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 50*time.Millisecond) // Test "delete" verb through the impersonation proxy. - err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) + err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMap shows up in the informer's cache. @@ -311,7 +371,7 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 50*time.Millisecond) // Test "deletecollection" verb through the impersonation proxy. - err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMaps shows up in the informer's cache. @@ -321,7 +381,7 @@ func TestImpersonationProxy(t *testing.T) { }, 10*time.Second, 50*time.Millisecond) // There should be no ConfigMaps left. - listResult, err = impersonationProxyClient.CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + listResult, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) @@ -341,24 +401,22 @@ func TestImpersonationProxy(t *testing.T) { // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. - var doubleImpersonationClient kubernetes.Interface - if env.HasCapability(library.HasExternalLoadBalancerProvider) { - doubleImpersonationClient = impersonationProxyViaLoadBalancerClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate") - } else { - doubleImpersonationClient = impersonationProxyViaSquidClient(impersonationProxyCACertPEM, "other-user-to-impersonate") - } + doubleImpersonationClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate") // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. // We could use any resource for this, but we happen to know that this one should exist. - _, err = impersonationProxyClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + _, err = impersonationProxyClient().CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.NoError(t, err) // Now we'll see what happens when we add an impersonation header to the request. This should generate a // request similar to the one above, except that it will have an impersonation header. _, err = doubleImpersonationClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) // Double impersonation is not supported yet, so we should get an error. - expectedErr := fmt.Sprintf("the server rejected our request for an unknown reason (get secrets %s)", impersonationProxyTLSSecretName(env)) - require.EqualError(t, err, expectedErr) + require.EqualError(t, err, fmt.Sprintf( + `users "other-user-to-impersonate" is forbidden: `+ + `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `impersonation is not allowed or invalid verb`, + env.TestUser.ExpectedUsername)) }) t.Run("kubectl as a client", func(t *testing.T) { @@ -495,7 +553,8 @@ func TestImpersonationProxy(t *testing.T) { rootCAs := x509.NewCertPool() rootCAs.AppendCertsFromPEM(impersonationProxyCACertPEM) tlsConfig := &tls.Config{ - RootCAs: rootCAs, + MinVersion: tls.VersionTLS12, + RootCAs: rootCAs, } websocketConfig := websocket.Config{ @@ -563,7 +622,7 @@ func TestImpersonationProxy(t *testing.T) { require.Eventually(t, func() bool { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. - _, err = impersonationProxyViaSquidClient(nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidClient(expectedProxyServiceEndpointURL, nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return err.Error() == serviceUnavailableViaSquidError }, 20*time.Second, 500*time.Millisecond) } @@ -705,7 +764,7 @@ func credentialIssuerName(env *library.TestEnv) string { return env.ConciergeAppName + "-config" } -// watchJSON defines the expected JSON wire equivalent of watch.Event +// watchJSON defines the expected JSON wire equivalent of watch.Event. type watchJSON struct { Type watch.EventType `json:"type,omitempty"` Object json.RawMessage `json:"object,omitempty"` diff --git a/test/library/credential_request.go b/test/library/credential_request.go new file mode 100644 index 00000000..44aeaff2 --- /dev/null +++ b/test/library/credential_request.go @@ -0,0 +1,27 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package library + +import ( + "context" + "testing" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" +) + +func CreateTokenCredentialRequest(ctx context.Context, t *testing.T, spec v1alpha1.TokenCredentialRequestSpec) (*v1alpha1.TokenCredentialRequest, error) { + t.Helper() + + client := NewAnonymousConciergeClientset(t) + + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, + &v1alpha1.TokenCredentialRequest{Spec: spec}, v1.CreateOptions{}, + ) +} From 6582c23edb6d16bb288d48c166af0e3567e5e8cc Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Wed, 10 Mar 2021 11:24:42 -0800 Subject: [PATCH 121/203] Fix a race detector error in a unit test Signed-off-by: Ryan Richard --- .../impersonatorconfig/impersonator_config.go | 2 +- .../impersonator_config_test.go | 117 ++++++++++-------- 2 files changed, 67 insertions(+), 52 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 8d741a4f..a06e44dc 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -357,7 +357,7 @@ func (c *impersonatorConfigController) ensureImpersonatorIsStarted(syncCtx contr // The server has stopped, so finish shutting it down. // If that fails too, return both errors for logging purposes. // By returning an error, the sync function will be called again - // and we'll have a change to restart the server. + // and we'll have a chance to restart the server. close(c.errorCh) // We don't want ensureImpersonatorIsStopped to block on reading this channel. stoppingErr := c.ensureImpersonatorIsStopped(false) return errors.NewAggregate([]error{runningErr, stoppingErr}) diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 66cab3e7..60d029fb 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -289,14 +289,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var cancelContext context.Context var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context - var impersonatorFuncWasCalled int - var impersonatorFuncError error - var impersonatorFuncReturnedFuncError error - var startedTLSListener net.Listener var frozenNow time.Time var signingCertProvider dynamiccert.Provider var signingCACertPEM, signingCAKeyPEM []byte var signingCASecret *corev1.Secret + var impersonatorFuncWasCalled int + var impersonatorFuncError error + var impersonatorFuncReturnedFuncError error + var startedTLSListener net.Listener + var startedTLSListenerMutex sync.RWMutex var testHTTPServer *http.Server var testHTTPServerMutex sync.RWMutex var testHTTPServerInterruptCh chan struct{} @@ -317,6 +318,48 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return nil, impersonatorFuncError } + startedTLSListenerMutex.Lock() // this is to satisfy the race detector + defer startedTLSListenerMutex.Unlock() + var err error + // Bind a listener to the port. Automatically choose the port for unit tests instead of using the real port. + startedTLSListener, err = tls.Listen("tcp", localhostIP+":0", &tls.Config{ + MinVersion: tls.VersionTLS12, + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + certPEM, keyPEM := dynamicCertProvider.CurrentCertKeyContent() + if certPEM != nil && keyPEM != nil { + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + r.NoError(err) + return &tlsCert, nil + } + return nil, nil // no cached TLS certs + }, + ClientAuth: tls.RequestClientCert, + VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { + // Docs say that this will always be called in tls.RequestClientCert mode + // and that the second parameter will always be nil in that case. + // rawCerts will be raw ASN.1 certificates provided by the peer. + if len(rawCerts) != 1 { + return fmt.Errorf("expected to get one client cert on incoming request to test server") + } + clientCert := rawCerts[0] + currentClientCertCA := impersonationProxySignerCAProvider.CurrentCABundleContent() + if currentClientCertCA == nil { + return fmt.Errorf("impersonationProxySignerCAProvider does not have a current CA certificate") + } + // Assert that the client's cert was signed by the CA cert that the controller put into + // the CAContentProvider that was passed in. + parsed, err := x509.ParseCertificate(clientCert) + require.NoError(t, err) + roots := x509.NewCertPool() + require.True(t, roots.AppendCertsFromPEM(currentClientCertCA)) + opts := x509.VerifyOptions{Roots: roots} + _, err = parsed.Verify(opts) + require.NoError(t, err) + return nil + }, + }) + r.NoError(err) + // Return a func that starts a fake server when called, and shuts down the fake server when stopCh is closed. // This fake server is enough like the real impersonation proxy server for this unit test because it // uses the supplied providers to serve TLS. The goal of this unit test is to make sure that the server @@ -326,47 +369,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return impersonatorFuncReturnedFuncError } - var err error - // automatically choose the port for unit tests - startedTLSListener, err = tls.Listen("tcp", localhostIP+":0", &tls.Config{ - MinVersion: tls.VersionTLS12, - GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { - certPEM, keyPEM := dynamicCertProvider.CurrentCertKeyContent() - if certPEM != nil && keyPEM != nil { - tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) - r.NoError(err) - return &tlsCert, nil - } - return nil, nil // no cached TLS certs - }, - ClientAuth: tls.RequestClientCert, - VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error { - // Docs say that this will always be called in tls.RequestClientCert mode - // and that the second parameter will always be nil in that case. - // rawCerts will be raw ASN.1 certificates provided by the peer. - if rawCerts == nil || len(rawCerts) != 1 { - return fmt.Errorf("expected to get one client cert on incoming request to test server") - } - clientCert := rawCerts[0] - currentClientCertCA := impersonationProxySignerCAProvider.CurrentCABundleContent() - if currentClientCertCA == nil { - return fmt.Errorf("impersonationProxySignerCAProvider does not have a current CA certificate") - } - // Assert that the client's cert was signed by the CA cert that the controller put into - // the CAContentProvider that was passed in. - parsed, err := x509.ParseCertificate(clientCert) - require.NoError(t, err) - t.Log("PARSED CLIENT CERT") - roots := x509.NewCertPool() - require.True(t, roots.AppendCertsFromPEM(currentClientCertCA)) - opts := x509.VerifyOptions{Roots: roots} - _, err = parsed.Verify(opts) - require.NoError(t, err) - return nil - }, - }) - r.NoError(err) - testHTTPServerMutex.Lock() // this is to satisfy the race detector testHTTPServer = &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { _, err := fmt.Fprint(w, fakeServerResponseBody) @@ -376,7 +378,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { // Start serving requests in the background. go func() { - err := testHTTPServer.Serve(startedTLSListener) + startedTLSListenerMutex.RLock() // this is to satisfy the race detector + listener := startedTLSListener + startedTLSListenerMutex.RUnlock() + err := testHTTPServer.Serve(listener) if !errors.Is(err, http.ErrServerClosed) { t.Log("Got an unexpected error while starting the fake http server!") r.NoError(err) // causes the test to crash, which is good enough because this should never happen @@ -394,7 +399,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { <-testHTTPServerInterruptCh } - err = testHTTPServer.Close() + err := testHTTPServer.Close() t.Log("Got an unexpected error while stopping the fake http server!") r.NoError(err) // causes the test to crash, which is good enough because this should never happen @@ -403,11 +408,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var testServerAddr = func() string { + var listener net.Listener require.Eventually(t, func() bool { - return startedTLSListener != nil + startedTLSListenerMutex.RLock() // this is to satisfy the race detector + listener = startedTLSListener + defer startedTLSListenerMutex.RUnlock() + return listener != nil }, 20*time.Second, 50*time.Millisecond, "TLS listener never became not nil") - return startedTLSListener.Addr().String() + return listener.Addr().String() } var closeTestHTTPServer = func() { @@ -1967,9 +1976,15 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { startInformersAndController() // The failure happens in a background goroutine, so the first sync succeeds. r.NoError(runControllerSync()) - // Eventually the server is not really running, because the startup failed. - r.Nil(startedTLSListener) + // The imperonatorFunc was called to construct an impersonator. r.Equal(impersonatorFuncWasCalled, 1) + // Without waiting too long because we don't want the test to be slow, check if it seems like the + // server never started. + r.Never(func() bool { + testHTTPServerMutex.RLock() // this is to satisfy the race detector + defer testHTTPServerMutex.RUnlock() + return testHTTPServer != nil + }, 2*time.Second, 50*time.Millisecond) r.Len(kubeAPIClient.Actions(), 3) requireNodesListed(kubeAPIClient.Actions()[0]) requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) From 1078bf4dfbe4eaf041248290bffa6019d32dd266 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 10 Mar 2021 13:08:15 -0800 Subject: [PATCH 122/203] Don't pass credentials when testing impersonation proxy port is closed When testing that the impersonation proxy port was closed there is no need to include credentials in the request. At the point when we want to test that the impersonation proxy port is closed, it is possible that we cannot perform a TokenCredentialRequest to get a credential either. Also add a new assertion that the TokenCredentialRequest stops handing out credentials on clusters which have no successful strategies. Signed-off-by: Monis Khan --- .../concierge_impersonation_proxy_test.go | 132 +++++++++++------- 1 file changed, 80 insertions(+), 52 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index a48e3203..d3682e7d 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -64,7 +64,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) - expectedProxyServiceEndpointURL := "https://" + proxyServiceEndpoint // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) @@ -97,7 +96,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) require.NoError(t, err) - require.Empty(t, tokenCredentialRequestResponse.Status.Message) // no error message + require.Nil(t, tokenCredentialRequestResponse.Status.Message, + "expected no error message but got: %s", library.Sdump(tokenCredentialRequestResponse.Status.Message)) require.NotEmpty(t, tokenCredentialRequestResponse.Status.Credential.ClientCertificateData) require.NotEmpty(t, tokenCredentialRequestResponse.Status.Credential.ClientKeyData) @@ -132,15 +132,27 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl return &config } - impersonationProxyViaSquidClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { - t.Helper() - kubeconfig := impersonationProxyRestConfig(refreshCredential(), proxyURL, caData, doubleImpersonateUser) - kubeconfig.Proxy = func(req *http.Request) (*url.URL, error) { + kubeconfigProxyFunc := func() func(req *http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { proxyURL, err := url.Parse(env.Proxy) require.NoError(t, err) t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) return proxyURL, nil } + } + + impersonationProxyViaSquidClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { + t.Helper() + kubeconfig := impersonationProxyRestConfig(refreshCredential(), proxyURL, caData, doubleImpersonateUser) + kubeconfig.Proxy = kubeconfigProxyFunc() + return library.NewKubeclient(t, kubeconfig).Kubernetes + } + + impersonationProxyViaSquidClientWithoutCredential := func() kubernetes.Interface { + t.Helper() + proxyURL := "https://" + proxyServiceEndpoint + kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, "") + kubeconfig.Proxy = kubeconfigProxyFunc() return library.NewKubeclient(t, kubeconfig).Kubernetes } @@ -202,7 +214,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. - _, err = impersonationProxyViaSquidClient(expectedProxyServiceEndpointURL, nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableViaSquidError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). @@ -592,57 +604,73 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Equal(t, "configmap-1", createConfigMap.Name) }) - // Update configuration to force the proxy to disabled mode - configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) - if env.HasCapability(library.HasExternalLoadBalancerProvider) { - t.Logf("creating configmap %s", configMap.Name) - _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) - require.NoError(t, err) - } else { - t.Logf("updating configmap %s", configMap.Name) - _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) - require.NoError(t, err) - } + t.Run("manually disabling the impersonation proxy feature", func(t *testing.T) { + // Update configuration to force the proxy to disabled mode + configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + t.Logf("creating configmap %s", configMap.Name) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) + require.NoError(t, err) + } else { + t.Logf("updating configmap %s", configMap.Name) + _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) + require.NoError(t, err) + } - if env.HasCapability(library.HasExternalLoadBalancerProvider) { - // The load balancer should have been deleted when we disabled the impersonation proxy. - // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). - library.RequireEventuallyWithoutError(t, func() (bool, error) { - hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) - return !hasService, err - }, 2*time.Minute, 500*time.Millisecond) - } + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + // The load balancer should have been deleted when we disabled the impersonation proxy. + // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). + library.RequireEventuallyWithoutError(t, func() (bool, error) { + hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) + return !hasService, err + }, 2*time.Minute, 500*time.Millisecond) + } - // Check that the impersonation proxy port has shut down. - // Ideally we could always check that the impersonation proxy's port has shut down, but on clusters where we - // do not run the squid proxy we have no easy way to see beyond the load balancer to see inside the cluster, - // so we'll skip this check on clusters which have load balancers but don't run the squid proxy. - // The other cluster types that do run the squid proxy will give us sufficient coverage here. - if env.Proxy != "" { + // Check that the impersonation proxy port has shut down. + // Ideally we could always check that the impersonation proxy's port has shut down, but on clusters where we + // do not run the squid proxy we have no easy way to see beyond the load balancer to see inside the cluster, + // so we'll skip this check on clusters which have load balancers but don't run the squid proxy. + // The other cluster types that do run the squid proxy will give us sufficient coverage here. + if env.Proxy != "" { + require.Eventually(t, func() bool { + // It's okay if this returns RBAC errors because this user has no role bindings. + // What we want to see is that the proxy eventually shuts down entirely. + _, err = impersonationProxyViaSquidClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + return err.Error() == serviceUnavailableViaSquidError + }, 20*time.Second, 500*time.Millisecond) + } + + // Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up + // when we disable the impersonator. require.Eventually(t, func() bool { - // It's okay if this returns RBAC errors because this user has no role bindings. - // What we want to see is that the proxy eventually shuts down entirely. - _, err = impersonationProxyViaSquidClient(expectedProxyServiceEndpointURL, nil, "").CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - return err.Error() == serviceUnavailableViaSquidError - }, 20*time.Second, 500*time.Millisecond) - } + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + return k8serrors.IsNotFound(err) + }, 10*time.Second, 250*time.Millisecond) - // Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up - // when we disable the impersonator. - require.Eventually(t, func() bool { - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - return k8serrors.IsNotFound(err) - }, 10*time.Second, 250*time.Millisecond) + // Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this + // around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when + // possible because they make their way into kubeconfigs on client machines. + _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) + require.NoError(t, err) - // Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this - // around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when - // possible because they make their way into kubeconfigs on client machines. - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) - require.NoError(t, err) + // At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to + // include an unsuccessful impersonation strategy saying that it was manually configured to be disabled. + requireDisabledByConfigurationStrategy(ctx, t, env, adminConciergeClient) - // At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to - // include an unsuccessful impersonation strategy saying that it was manually configured to be disabled. - requireDisabledByConfigurationStrategy(ctx, t, env, adminConciergeClient) + if !env.HasCapability(library.ClusterSigningKeyIsAvailable) { + // This cluster does not support the cluster signing key strategy, so now that we've manually disabled the + // impersonation strategy, we should be left with no working strategies. + // Given that there are no working strategies, a TokenCredentialRequest which would otherwise work should now + // fail, because there is no point handing out credentials that are not going to work for any strategy. + tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, + loginv1alpha1.TokenCredentialRequestSpec{Token: env.TestUser.Token, Authenticator: authenticator}, + ) + require.NoError(t, err) + require.NotNil(t, tokenCredentialRequestResponse.Status.Message, "expected an error message but got nil") + require.Equal(t, "authentication failed", *tokenCredentialRequestResponse.Status.Message) + require.Nil(t, tokenCredentialRequestResponse.Status.Credential) + } + }) } func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient versioned.Interface) (string, []byte) { From 2a2e2f532b4d0d0786935448f76b077048bd098b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 10 Mar 2021 14:17:20 -0800 Subject: [PATCH 123/203] Remove an integration test that is covered elsewhere now The same coverage that was supplied by TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheClusterIsNotCapable is now provided by an assertion at the end of TestImpersonationProxy, so delete the duplicate test which was failing on GKE because the impersonation proxy is now active by default on GKE. --- .../concierge_credentialrequest_test.go | 39 ++++--------------- 1 file changed, 8 insertions(+), 31 deletions(-) diff --git a/test/integration/concierge_credentialrequest_test.go b/test/integration/concierge_credentialrequest_test.go index 196670d4..6dedfcac 100644 --- a/test/integration/concierge_credentialrequest_test.go +++ b/test/integration/concierge_credentialrequest_test.go @@ -31,11 +31,14 @@ func TestUnsuccessfulCredentialRequest(t *testing.T) { defer cancel() response, err := library.CreateTokenCredentialRequest(ctx, t, - validCredentialRequestSpecWithRealToken(t, corev1.TypedLocalObjectReference{ - APIGroup: &auth1alpha1.SchemeGroupVersion.Group, - Kind: "WebhookAuthenticator", - Name: "some-webhook-that-does-not-exist", - }), + loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: corev1.TypedLocalObjectReference{ + APIGroup: &auth1alpha1.SchemeGroupVersion.Group, + Kind: "WebhookAuthenticator", + Name: "some-webhook-that-does-not-exist", + }, + }, ) require.NoError(t, err) require.Nil(t, response.Status.Credential) @@ -182,32 +185,6 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T require.Nil(t, response.Status.Credential) } -func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheClusterIsNotCapable(t *testing.T) { - env := library.IntegrationEnv(t).WithoutCapability(library.ClusterSigningKeyIsAvailable) - - library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - defer cancel() - - testWebhook := library.CreateTestWebhookAuthenticator(ctx, t) - - response, err := library.CreateTokenCredentialRequest(ctx, t, validCredentialRequestSpecWithRealToken(t, testWebhook)) - - require.NoError(t, err) - - require.Empty(t, response.Spec) - require.Nil(t, response.Status.Credential) - require.Equal(t, stringPtr("authentication failed"), response.Status.Message) -} - -func validCredentialRequestSpecWithRealToken(t *testing.T, authenticator corev1.TypedLocalObjectReference) loginv1alpha1.TokenCredentialRequestSpec { - return loginv1alpha1.TokenCredentialRequestSpec{ - Token: library.IntegrationEnv(t).TestUser.Token, - Authenticator: authenticator, - } -} - func stringPtr(s string) *string { return &s } From 006dc8aa7979da342faddaa963d2f87753f7435d Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 10 Mar 2021 14:50:46 -0800 Subject: [PATCH 124/203] Small test refactor --- .../concierge_impersonation_proxy_test.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index d3682e7d..a4772140 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -67,6 +67,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) + credentialRequestSpecWithWorkingCredentials := loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: authenticator, + } + credentialAlmostExpired := func(credential *loginv1alpha1.TokenCredentialRequest) bool { pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) @@ -90,10 +95,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // // However, we issue short-lived certs, so this cert will only be valid for a few minutes. // Cache it until it is almost expired and then refresh it whenever it is close to expired. - tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, loginv1alpha1.TokenCredentialRequestSpec{ - Token: env.TestUser.Token, - Authenticator: authenticator, - }) + tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) require.NoError(t, err) require.Nil(t, tokenCredentialRequestResponse.Status.Message, @@ -662,10 +664,9 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // impersonation strategy, we should be left with no working strategies. // Given that there are no working strategies, a TokenCredentialRequest which would otherwise work should now // fail, because there is no point handing out credentials that are not going to work for any strategy. - tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, - loginv1alpha1.TokenCredentialRequestSpec{Token: env.TestUser.Token, Authenticator: authenticator}, - ) + tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) require.NoError(t, err) + require.NotNil(t, tokenCredentialRequestResponse.Status.Message, "expected an error message but got nil") require.Equal(t, "authentication failed", *tokenCredentialRequestResponse.Status.Message) require.Nil(t, tokenCredentialRequestResponse.Status.Credential) From 24396b6af1d717fab254d636bde87ae56d5051a7 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Wed, 10 Mar 2021 15:49:09 -0800 Subject: [PATCH 125/203] Use gorilla websocket library so squid proxy works --- go.mod | 1 + .../concierge_impersonation_proxy_test.go | 61 ++++++++++++------- 2 files changed, 41 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index de809464..bb3b8a9a 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/google/go-cmp v0.5.5 github.com/google/gofuzz v1.2.0 github.com/gorilla/securecookie v1.1.1 + github.com/gorilla/websocket v1.4.2 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/onsi/ginkgo v1.13.0 // indirect github.com/ory/fosite v0.38.0 diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index a4772140..93f6f343 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -6,7 +6,6 @@ package integration import ( "bytes" "context" - "crypto/tls" "crypto/x509" "encoding/base64" "encoding/json" @@ -22,8 +21,9 @@ import ( "testing" "time" + "github.com/gorilla/websocket" + "github.com/stretchr/testify/require" - "golang.org/x/net/websocket" v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -558,30 +558,44 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("websocket client", func(t *testing.T) { + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespace.Name, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) + + impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") + tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) + require.NoError(t, err) + dest, _ := url.Parse(impersonationProxyURL) dest.Scheme = "wss" dest.Path = "/api/v1/namespaces/" + namespace.Name + "/configmaps" dest.RawQuery = "watch=1&resourceVersion=0" - origin, _ := url.Parse("http://localhost") - rootCAs := x509.NewCertPool() - rootCAs.AppendCertsFromPEM(impersonationProxyCACertPEM) - tlsConfig := &tls.Config{ - MinVersion: tls.VersionTLS12, - RootCAs: rootCAs, + dialer := websocket.Dialer{ + TLSClientConfig: tlsConfig, } - - websocketConfig := websocket.Config{ - Location: dest, - Origin: origin, - TlsConfig: tlsConfig, - Version: 13, - Header: http.Header(make(map[string][]string)), + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + dialer.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + } } - ws, err := websocket.DialConfig(&websocketConfig) - if err != nil { - t.Fatalf("failed to dial websocket: %v", err) + c, r, err := dialer.Dial(dest.String(), nil) + if r != nil { + defer r.Body.Close() } + if err != nil && r != nil { + body, _ := ioutil.ReadAll(r.Body) + t.Logf("websocket dial failed: %d:%s", r.StatusCode, body) + } + require.NoError(t, err) // perform a create through the admin client _, err = adminClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, @@ -589,17 +603,22 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl metav1.CreateOptions{}, ) require.NoError(t, err) + t.Cleanup(func() { + err = adminClient.CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + require.NoError(t, err) + }) // see if the websocket client received an event for the create - var got watchJSON - err = websocket.JSON.Receive(ws, &got) + _, message, err := c.ReadMessage() if err != nil { t.Fatalf("Unexpected error: %v", err) } + var got watchJSON + err = json.Unmarshal(message, &got) + require.NoError(t, err) if got.Type != watch.Added { t.Errorf("Unexpected type: %v", got.Type) } - var createConfigMap corev1.ConfigMap err = json.Unmarshal(got.Object, &createConfigMap) require.NoError(t, err) From d13bb07b3e18aef6ec100fd0429630daaea3777f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Wed, 10 Mar 2021 16:57:15 -0800 Subject: [PATCH 126/203] Add integration test for using WhoAmIRequest through impersonator --- go.mod | 4 +- .../concierge_impersonation_proxy_test.go | 263 +++++++++++------- test/integration/whoami_test.go | 8 - test/library/client.go | 18 +- test/library/credential_request.go | 27 -- 5 files changed, 174 insertions(+), 146 deletions(-) delete mode 100644 test/library/credential_request.go diff --git a/go.mod b/go.mod index bb3b8a9a..c6cc4589 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/google/go-cmp v0.5.5 github.com/google/gofuzz v1.2.0 github.com/gorilla/securecookie v1.1.1 - github.com/gorilla/websocket v1.4.2 // indirect + github.com/gorilla/websocket v1.4.2 github.com/oleiade/reflections v1.0.1 // indirect github.com/onsi/ginkgo v1.13.0 // indirect github.com/ory/fosite v0.38.0 @@ -27,7 +27,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 93f6f343..520eda31 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -22,7 +22,6 @@ import ( "time" "github.com/gorilla/websocket" - "github.com/stretchr/testify/require" v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" @@ -38,9 +37,11 @@ import ( "sigs.k8s.io/yaml" "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" "go.pinniped.dev/test/library" ) @@ -59,35 +60,20 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl adminClient := library.NewKubernetesClientset(t) adminConciergeClient := library.NewConciergeClientset(t) - // Create a WebhookAuthenticator. - authenticator := library.CreateTestWebhookAuthenticator(ctx, t) + // Create a WebhookAuthenticator and prepare a TokenCredentialRequestSpec using the authenticator for use later. + credentialRequestSpecWithWorkingCredentials := loginv1alpha1.TokenCredentialRequestSpec{ + Token: env.TestUser.Token, + Authenticator: library.CreateTestWebhookAuthenticator(ctx, t), + } // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) - credentialRequestSpecWithWorkingCredentials := loginv1alpha1.TokenCredentialRequestSpec{ - Token: env.TestUser.Token, - Authenticator: authenticator, - } - - credentialAlmostExpired := func(credential *loginv1alpha1.TokenCredentialRequest) bool { - pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) - parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) - require.NoError(t, err) - timeRemaining := time.Until(parsedCredential.NotAfter) - if timeRemaining < 2*time.Minute { - t.Logf("The TokenCredentialRequest cred is almost expired and needs to be refreshed. Expires in %s.", timeRemaining) - return true - } - t.Logf("The TokenCredentialRequest cred is good for some more time (%s) so using it.", timeRemaining) - return false - } - - var tokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest + var mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest refreshCredential := func() *loginv1alpha1.ClusterCredential { - if tokenCredentialRequestResponse == nil || credentialAlmostExpired(tokenCredentialRequestResponse) { + if mostRecentTokenCredentialRequestResponse == nil || credentialAlmostExpired(t, mostRecentTokenCredentialRequestResponse) { var err error // Make a TokenCredentialRequest. This can either return a cert signed by the Kube API server's CA (e.g. on kind) // or a cert signed by the impersonator's signing CA (e.g. on GKE). Either should be accepted by the impersonation @@ -95,80 +81,46 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // // However, we issue short-lived certs, so this cert will only be valid for a few minutes. // Cache it until it is almost expired and then refresh it whenever it is close to expired. - tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) + mostRecentTokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) require.NoError(t, err) - require.Nil(t, tokenCredentialRequestResponse.Status.Message, - "expected no error message but got: %s", library.Sdump(tokenCredentialRequestResponse.Status.Message)) - require.NotEmpty(t, tokenCredentialRequestResponse.Status.Credential.ClientCertificateData) - require.NotEmpty(t, tokenCredentialRequestResponse.Status.Credential.ClientKeyData) + require.Nil(t, mostRecentTokenCredentialRequestResponse.Status.Message, + "expected no error message but got: %s", library.Sdump(mostRecentTokenCredentialRequestResponse.Status.Message)) + require.NotEmpty(t, mostRecentTokenCredentialRequestResponse.Status.Credential.ClientCertificateData) + require.NotEmpty(t, mostRecentTokenCredentialRequestResponse.Status.Credential.ClientKeyData) // At the moment the credential request should not have returned a token. In the future, if we make it return // tokens, we should revisit this test's rest config below. - require.Empty(t, tokenCredentialRequestResponse.Status.Credential.Token) + require.Empty(t, mostRecentTokenCredentialRequestResponse.Status.Credential.Token) } - return tokenCredentialRequestResponse.Status.Credential + return mostRecentTokenCredentialRequestResponse.Status.Credential } - impersonationProxyRestConfig := func(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, doubleImpersonateUser string) *rest.Config { - config := rest.Config{ - Host: host, - TLSClientConfig: rest.TLSClientConfig{ - Insecure: caData == nil, - CAData: caData, - CertData: []byte(credential.ClientCertificateData), - KeyData: []byte(credential.ClientKeyData), - }, - // kubectl would set both the client cert and the token, so we'll do that too. - // The Kube API server will ignore the token if the client cert successfully authenticates. - // Only if the client cert is not present or fails to authenticate will it use the token. - // Historically, it works that way because some web browsers will always send your - // corporate-assigned client cert even if it is not valid, and it doesn't want to treat - // that as a failure if you also sent a perfectly good token. - // We would like the impersonation proxy to imitate that behavior, so we test it here. - BearerToken: "this is not valid", - } - if doubleImpersonateUser != "" { - config.Impersonate = rest.ImpersonationConfig{UserName: doubleImpersonateUser} - } - return &config - } - - kubeconfigProxyFunc := func() func(req *http.Request) (*url.URL, error) { - return func(req *http.Request) (*url.URL, error) { - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil - } - } - - impersonationProxyViaSquidClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { - t.Helper() - kubeconfig := impersonationProxyRestConfig(refreshCredential(), proxyURL, caData, doubleImpersonateUser) - kubeconfig.Proxy = kubeconfigProxyFunc() - return library.NewKubeclient(t, kubeconfig).Kubernetes - } - - impersonationProxyViaSquidClientWithoutCredential := func() kubernetes.Interface { - t.Helper() + impersonationProxyViaSquidKubeClientWithoutCredential := func() kubernetes.Interface { proxyURL := "https://" + proxyServiceEndpoint kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, "") - kubeconfig.Proxy = kubeconfigProxyFunc() + kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) return library.NewKubeclient(t, kubeconfig).Kubernetes } - impersonationProxyViaLoadBalancerClient := func(proxyURL string, caData []byte, doubleImpersonateUser string) kubernetes.Interface { - t.Helper() - kubeconfig := impersonationProxyRestConfig(refreshCredential(), proxyURL, caData, doubleImpersonateUser) - return library.NewKubeclient(t, kubeconfig).Kubernetes - } - - newImpersonationProxyClient := func(proxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) kubernetes.Interface { - if env.HasCapability(library.HasExternalLoadBalancerProvider) { - return impersonationProxyViaLoadBalancerClient(proxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + newImpersonationProxyClientWithCredentials := func(credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { + kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + // Send traffic through the Squid proxy + kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) } - return impersonationProxyViaSquidClient(proxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + return library.NewKubeclient(t, kubeconfig) + } + + newImpersonationProxyClient := func(impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { + refreshedCredentials := refreshCredential() + refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests + return newImpersonationProxyClientWithCredentials(refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) + } + + newAnonymousImpersonationProxyClient := func(impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { + emptyCredentials := &loginv1alpha1.ClusterCredential{} + return newImpersonationProxyClientWithCredentials(emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) } oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName(env), metav1.GetOptions{}) @@ -216,7 +168,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. - _, err = impersonationProxyViaSquidClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) require.EqualError(t, err, serviceUnavailableViaSquidError) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). @@ -244,21 +196,21 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // credentials before they expire. Create a closure to capture the arguments to newImpersonationProxyClient // so we don't have to keep repeating them. // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. - impersonationProxyClient := func() kubernetes.Interface { - return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "") + impersonationProxyKubeClient := func() kubernetes.Interface { + return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "").Kubernetes } // Test that the user can perform basic actions through the client with their username and group membership // influencing RBAC checks correctly. t.Run( "access as user", - library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyClient()), + library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyKubeClient()), ) for _, group := range env.TestUser.ExpectedGroups { group := group t.Run( "access as group "+group, - library.AccessAsGroupTest(ctx, group, impersonationProxyClient()), + library.AccessAsGroupTest(ctx, group, impersonationProxyKubeClient()), ) } @@ -288,7 +240,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Create and start informer to exercise the "watch" verb for us. informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( - impersonationProxyClient(), + impersonationProxyKubeClient(), 0, k8sinformers.WithNamespace(namespace.Name)) informer := informerFactory.Core().V1().ConfigMaps() @@ -308,17 +260,17 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // Test "create" verb through the impersonation proxy. - _, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) - _, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) - _, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, metav1.CreateOptions{}, ) @@ -334,11 +286,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 50*time.Millisecond) // Test "get" verb through the impersonation proxy. - configMap3, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Get(ctx, "configmap-3", metav1.GetOptions{}) + configMap3, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Get(ctx, "configmap-3", metav1.GetOptions{}) require.NoError(t, err) // Test "list" verb through the impersonation proxy. - listResult, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + listResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) @@ -346,7 +298,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Test "update" verb through the impersonation proxy. configMap3.Data = map[string]string{"foo": "bar"} - updateResult, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Update(ctx, configMap3, metav1.UpdateOptions{}) + updateResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Update(ctx, configMap3, metav1.UpdateOptions{}) require.NoError(t, err) require.Equal(t, "bar", updateResult.Data["foo"]) @@ -357,7 +309,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 50*time.Millisecond) // Test "patch" verb through the impersonation proxy. - patchResult, err := impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Patch(ctx, + patchResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Patch(ctx, "configmap-3", types.MergePatchType, []byte(`{"data":{"baz":"42"}}`), @@ -374,7 +326,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 50*time.Millisecond) // Test "delete" verb through the impersonation proxy. - err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMap shows up in the informer's cache. @@ -385,7 +337,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 50*time.Millisecond) // Test "deletecollection" verb through the impersonation proxy. - err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMaps shows up in the informer's cache. @@ -395,7 +347,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 50*time.Millisecond) // There should be no ConfigMaps left. - listResult, err = impersonationProxyClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + listResult, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) @@ -415,16 +367,16 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. - doubleImpersonationClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate") + doubleImpersonationKubeClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. // We could use any resource for this, but we happen to know that this one should exist. - _, err = impersonationProxyClient().CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + _, err = impersonationProxyKubeClient().CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.NoError(t, err) // Now we'll see what happens when we add an impersonation header to the request. This should generate a - // request similar to the one above, except that it will have an impersonation header. - _, err = doubleImpersonationClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // request similar to the one above, except that it will also have an impersonation header. + _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) // Double impersonation is not supported yet, so we should get an error. require.EqualError(t, err, fmt.Sprintf( `users "other-user-to-impersonate" is forbidden: `+ @@ -433,6 +385,36 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl env.TestUser.ExpectedUsername)) }) + t.Run("using service account tokens to authenticate to impersonation proxy", func(t *testing.T) { + // TODO: test that this is not currently allowed + }) + + t.Run("WhoAmIRequests through the impersonation proxy", func(t *testing.T) { + // Test using the TokenCredentialRequest for authentication. + impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( + impersonationProxyURL, impersonationProxyCACertPEM, "", + ).PinnipedConcierge + whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse(env.TestUser.ExpectedUsername, append(env.TestUser.ExpectedGroups, "system:authenticated")), + whoAmI, + ) + + // Test an unauthenticated request which does not include any credentials. + impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient( + impersonationProxyURL, impersonationProxyCACertPEM, "", + ).PinnipedConcierge + whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse("system:anonymous", []string{"system:unauthenticated"}), + whoAmI, + ) + }) + t.Run("kubectl as a client", func(t *testing.T) { // Create an RBAC rule to allow this user to read/write everything. library.CreateTestClusterRoleBinding(t, @@ -526,7 +508,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "echo", echoString) - require.NoError(t, err) + require.NoError(t, err, `"kubectl exec" failed`) require.Equal(t, echoString+"\n", stdout) // run the kubectl port-forward command @@ -537,7 +519,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // start, but don't wait for the command to finish err = portForwardCmd.Start() - require.NoError(t, err) + require.NoError(t, err, `"kubectl port-forward" failed`) // then run curl something against it time.Sleep(time.Second) @@ -589,7 +571,9 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } c, r, err := dialer.Dial(dest.String(), nil) if r != nil { - defer r.Body.Close() + defer func() { + require.NoError(t, r.Body.Close()) + }() } if err != nil && r != nil { body, _ := ioutil.ReadAll(r.Body) @@ -656,7 +640,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Eventually(t, func() bool { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. - _, err = impersonationProxyViaSquidClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return err.Error() == serviceUnavailableViaSquidError }, 20*time.Second, 500*time.Millisecond) } @@ -683,7 +667,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // impersonation strategy, we should be left with no working strategies. // Given that there are no working strategies, a TokenCredentialRequest which would otherwise work should now // fail, because there is no point handing out credentials that are not going to work for any strategy. - tokenCredentialRequestResponse, err = library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) + tokenCredentialRequestResponse, err := library.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) require.NoError(t, err) require.NotNil(t, tokenCredentialRequestResponse.Status.Message, "expected an error message but got nil") @@ -693,6 +677,21 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) } +func expectedWhoAmIRequestResponse(username string, groups []string) *identityv1alpha1.WhoAmIRequest { + return &identityv1alpha1.WhoAmIRequest{ + Status: identityv1alpha1.WhoAmIRequestStatus{ + KubernetesUserInfo: identityv1alpha1.KubernetesUserInfo{ + User: identityv1alpha1.UserInfo{ + Username: username, + UID: "", // no way to impersonate UID: https://github.com/kubernetes/kubernetes/issues/93699 + Groups: groups, + Extra: nil, + }, + }, + }, + } +} + func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient versioned.Interface) (string, []byte) { t.Helper() var impersonationProxyURL string @@ -767,6 +766,54 @@ func requireDisabledByConfigurationStrategy(ctx context.Context, t *testing.T, e }, 1*time.Minute, 500*time.Millisecond) } +func credentialAlmostExpired(t *testing.T, credential *loginv1alpha1.TokenCredentialRequest) bool { + t.Helper() + pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) + parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) + require.NoError(t, err) + timeRemaining := time.Until(parsedCredential.NotAfter) + if timeRemaining < 2*time.Minute { + t.Logf("The TokenCredentialRequest cred is almost expired and needs to be refreshed. Expires in %s.", timeRemaining) + return true + } + t.Logf("The TokenCredentialRequest cred is good for some more time (%s) so using it.", timeRemaining) + return false +} + +func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, doubleImpersonateUser string) *rest.Config { + config := rest.Config{ + Host: host, + TLSClientConfig: rest.TLSClientConfig{ + Insecure: caData == nil, + CAData: caData, + CertData: []byte(credential.ClientCertificateData), + KeyData: []byte(credential.ClientKeyData), + }, + // kubectl would set both the client cert and the token, so we'll do that too. + // The Kube API server will ignore the token if the client cert successfully authenticates. + // Only if the client cert is not present or fails to authenticate will it use the token. + // Historically, it works that way because some web browsers will always send your + // corporate-assigned client cert even if it is not valid, and it doesn't want to treat + // that as a failure if you also sent a perfectly good token. + // We would like the impersonation proxy to imitate that behavior, so we test it here. + BearerToken: credential.Token, + } + if doubleImpersonateUser != "" { + config.Impersonate = rest.ImpersonationConfig{UserName: doubleImpersonateUser} + } + return &config +} + +func kubeconfigProxyFunc(t *testing.T, squidProxyURL string) func(req *http.Request) (*url.URL, error) { + return func(req *http.Request) (*url.URL, error) { + t.Helper() + parsedSquidProxyURL, err := url.Parse(squidProxyURL) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, parsedSquidProxyURL.String()) + return parsedSquidProxyURL, nil + } +} + func configMapForConfig(t *testing.T, env *library.TestEnv, config impersonator.Config) corev1.ConfigMap { t.Helper() configString, err := yaml.Marshal(config) diff --git a/test/integration/whoami_test.go b/test/integration/whoami_test.go index 345347d8..f13c7cc2 100644 --- a/test/integration/whoami_test.go +++ b/test/integration/whoami_test.go @@ -443,11 +443,3 @@ func TestWhoAmI_ImpersonateDirectly(t *testing.T) { whoAmIAnonymous, ) } - -func TestWhoAmI_ImpersonateViaProxy(t *testing.T) { - _ = library.IntegrationEnv(t) - - // TODO: add this test after the impersonation proxy is done - // this should test all forms of auth understood by the proxy (certs, SA token, token cred req, anonymous, etc) - // remember that impersonation does not support UID: https://github.com/kubernetes/kubernetes/issues/93699 -} diff --git a/test/library/client.go b/test/library/client.go index 1d5fd9e8..7371b974 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -28,6 +28,7 @@ import ( aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" auth1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" idpv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idp/v1alpha1" conciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" @@ -54,7 +55,9 @@ func NewClientsetForKubeConfig(t *testing.T, kubeConfig string) kubernetes.Inter func NewRestConfigFromKubeconfig(t *testing.T, kubeConfig string) *rest.Config { kubeConfigFile, err := ioutil.TempFile("", "pinniped-cli-test-*") require.NoError(t, err) - defer os.Remove(kubeConfigFile.Name()) + defer func() { + require.NoError(t, os.Remove(kubeConfigFile.Name())) + }() _, err = kubeConfigFile.Write([]byte(kubeConfig)) require.NoError(t, err) @@ -423,6 +426,19 @@ func CreateTestClusterRoleBinding(t *testing.T, subject rbacv1.Subject, roleRef return created } +func CreateTokenCredentialRequest(ctx context.Context, t *testing.T, spec v1alpha1.TokenCredentialRequestSpec) (*v1alpha1.TokenCredentialRequest, error) { + t.Helper() + + client := NewAnonymousConciergeClientset(t) + + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, + &v1alpha1.TokenCredentialRequest{Spec: spec}, metav1.CreateOptions{}, + ) +} + func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldHaveAccessTo *authorizationv1.ResourceAttributes) { t.Helper() client := NewKubernetesClientset(t) diff --git a/test/library/credential_request.go b/test/library/credential_request.go deleted file mode 100644 index 44aeaff2..00000000 --- a/test/library/credential_request.go +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright 2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package library - -import ( - "context" - "testing" - "time" - - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" -) - -func CreateTokenCredentialRequest(ctx context.Context, t *testing.T, spec v1alpha1.TokenCredentialRequestSpec) (*v1alpha1.TokenCredentialRequest, error) { - t.Helper() - - client := NewAnonymousConciergeClientset(t) - - ctx, cancel := context.WithTimeout(ctx, time.Minute) - defer cancel() - - return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, - &v1alpha1.TokenCredentialRequest{Spec: spec}, v1.CreateOptions{}, - ) -} From 32b038c639cb943d67351964bd5f9007ff3119c0 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 11 Mar 2021 10:02:28 -0500 Subject: [PATCH 127/203] test/integration: add 'kubectl cp' test to TestImpersonationProxy Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 520eda31..05d2b3f7 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -507,10 +507,22 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" - stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "echo", echoString) + remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) + stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) require.NoError(t, err, `"kubectl exec" failed`) require.Equal(t, echoString+"\n", stdout) + // run the kubectl cp command + localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) + _, _, err = runKubectl("cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, podName, remoteEchoFile), localEchoFile) + require.NoError(t, err, `"kubectl cp" failed`) + localEchoFileData, err := os.ReadFile(localEchoFile) + require.NoError(t, err) + require.Equal(t, echoString+"\n", string(localEchoFileData)) + defer func() { + _, _, _ = runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "rm", remoteEchoFile) // cleanup remote echo file + }() + // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() From 7b1ecf79a6f67dcf2edf08dd75006235ea9d1715 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Thu, 11 Mar 2021 10:13:07 -0500 Subject: [PATCH 128/203] Fix race between err chan send and re-queue Signed-off-by: Monis Khan --- .../impersonatorconfig/impersonator_config.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index a06e44dc..389905d7 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -378,16 +378,18 @@ func (c *impersonatorConfigController) ensureImpersonatorIsStarted(syncCtx contr } c.serverStopCh = make(chan struct{}) - c.errorCh = make(chan error) + // use a buffered channel so that startImpersonatorFunc can send + // on it without coordinating with the main controller go routine + c.errorCh = make(chan error, 1) // startImpersonatorFunc will block until the server shuts down (or fails to start), so run it in the background. go func() { - startOrStopErr := startImpersonatorFunc(c.serverStopCh) - // The server has stopped, so enqueue ourselves for another sync, so we can - // try to start the server again as quickly as possible. - syncCtx.Queue.AddRateLimited(syncCtx.Key) // TODO this a race because the main controller go routine could run and complete before we send on the err chan + // The server has stopped, so enqueue ourselves for another sync, + // so we can try to start the server again as quickly as possible. + defer syncCtx.Queue.AddRateLimited(syncCtx.Key) + // Forward any errors returned by startImpersonatorFunc on the errorCh. - c.errorCh <- startOrStopErr + c.errorCh <- startImpersonatorFunc(c.serverStopCh) }() return nil From b793b9a17e5dde0a926af7d81d9412a81329570e Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 11 Mar 2021 10:42:12 -0500 Subject: [PATCH 129/203] test/integration: add 'kubectl logs' test to TestImpersonationProxy Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 05d2b3f7..721c9253 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -523,6 +523,12 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl _, _, _ = runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "rm", remoteEchoFile) // cleanup remote echo file }() + // run the kubectl logs command + logLinesCount := 10 + stdout, _, err = runKubectl("logs", "--namespace", env.ConciergeNamespace, podName, fmt.Sprintf("--tail=%d", logLinesCount)) + require.NoError(t, err, `"kubectl logs" failed`) + require.Equalf(t, logLinesCount, strings.Count(stdout, "\n"), "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) + // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() From 61d64fc4c652cad97193384fbd3da47321818e8a Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 08:58:54 -0800 Subject: [PATCH 130/203] Use ioutil.ReadFile instead of os.ReadFile Because it works on older golang versions too. --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 721c9253..b1e25c62 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -516,7 +516,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) _, _, err = runKubectl("cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, podName, remoteEchoFile), localEchoFile) require.NoError(t, err, `"kubectl cp" failed`) - localEchoFileData, err := os.ReadFile(localEchoFile) + localEchoFileData, err := ioutil.ReadFile(localEchoFile) require.NoError(t, err) require.Equal(t, echoString+"\n", string(localEchoFileData)) defer func() { From 34accc3deefd31af58937cead74dce4cfe04b2d9 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 10:01:17 -0800 Subject: [PATCH 131/203] Test using a service account token to auth to the impersonator Also make each t.Run use its own namespace to slight reduce the interdependency between them. Use t.Cleanup instead of defer in whoami_test.go just to be consistent with other integration tests. --- .../concierge_impersonation_proxy_test.go | 155 +++++++++++++----- test/integration/whoami_test.go | 31 ++-- 2 files changed, 120 insertions(+), 66 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index b1e25c62..b701fb19 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -214,20 +214,10 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl ) } - // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. - namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, - }, metav1.CreateOptions{}) - require.NoError(t, err) - // Schedule the namespace for cleanup. - t.Cleanup(func() { - t.Logf("cleaning up test namespace %s", namespace.Name) - err = adminClient.CoreV1().Namespaces().Delete(context.Background(), namespace.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }) + t.Run("using and watching all the basic verbs", func(t *testing.T) { + // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. + namespaceName := createTestNamespace(t, adminClient) - // Try more Kube API verbs through the impersonation proxy. - t.Run("watching all the basic verbs", func(t *testing.T) { // Create an RBAC rule to allow this user to read/write everything. library.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, @@ -235,14 +225,14 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespace.Name, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", }) // Create and start informer to exercise the "watch" verb for us. informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( impersonationProxyKubeClient(), 0, - k8sinformers.WithNamespace(namespace.Name)) + k8sinformers.WithNamespace(namespaceName)) informer := informerFactory.Core().V1().ConfigMaps() informer.Informer() // makes sure that the informer will cache stopChannel := make(chan struct{}) @@ -260,17 +250,17 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // Test "create" verb through the impersonation proxy. - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, metav1.CreateOptions{}, ) @@ -279,18 +269,18 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Make sure that all of the created ConfigMaps show up in the informer's cache to // demonstrate that the informer's "watch" verb is working through the impersonation proxy. require.Eventually(t, func() bool { - _, err1 := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-1") - _, err2 := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-2") - _, err3 := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + _, err1 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-1") + _, err2 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-2") + _, err3 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") return err1 == nil && err2 == nil && err3 == nil }, 10*time.Second, 50*time.Millisecond) // Test "get" verb through the impersonation proxy. - configMap3, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Get(ctx, "configmap-3", metav1.GetOptions{}) + configMap3, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Get(ctx, "configmap-3", metav1.GetOptions{}) require.NoError(t, err) // Test "list" verb through the impersonation proxy. - listResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + listResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) @@ -298,18 +288,18 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Test "update" verb through the impersonation proxy. configMap3.Data = map[string]string{"foo": "bar"} - updateResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Update(ctx, configMap3, metav1.UpdateOptions{}) + updateResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Update(ctx, configMap3, metav1.UpdateOptions{}) require.NoError(t, err) require.Equal(t, "bar", updateResult.Data["foo"]) // Make sure that the updated ConfigMap shows up in the informer's cache. require.Eventually(t, func() bool { - configMap, err := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") return err == nil && configMap.Data["foo"] == "bar" }, 10*time.Second, 50*time.Millisecond) // Test "patch" verb through the impersonation proxy. - patchResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Patch(ctx, + patchResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Patch(ctx, "configmap-3", types.MergePatchType, []byte(`{"data":{"baz":"42"}}`), @@ -321,33 +311,33 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Make sure that the patched ConfigMap shows up in the informer's cache. require.Eventually(t, func() bool { - configMap, err := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") + configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") return err == nil && configMap.Data["foo"] == "bar" && configMap.Data["baz"] == "42" }, 10*time.Second, 50*time.Millisecond) // Test "delete" verb through the impersonation proxy. - err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMap shows up in the informer's cache. require.Eventually(t, func() bool { - _, getErr := informer.Lister().ConfigMaps(namespace.Name).Get("configmap-3") - list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(configMapLabels.AsSelector()) + _, getErr := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) return k8serrors.IsNotFound(getErr) && listErr == nil && len(list) == 2 }, 10*time.Second, 50*time.Millisecond) // Test "deletecollection" verb through the impersonation proxy. - err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMaps shows up in the informer's cache. require.Eventually(t, func() bool { - list, listErr := informer.Lister().ConfigMaps(namespace.Name).List(configMapLabels.AsSelector()) + list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) return listErr == nil && len(list) == 0 }, 10*time.Second, 50*time.Millisecond) // There should be no ConfigMaps left. - listResult, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespace.Name).List(ctx, metav1.ListOptions{ + listResult, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) @@ -385,11 +375,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl env.TestUser.ExpectedUsername)) }) - t.Run("using service account tokens to authenticate to impersonation proxy", func(t *testing.T) { - // TODO: test that this is not currently allowed - }) - - t.Run("WhoAmIRequests through the impersonation proxy", func(t *testing.T) { + t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { // Test using the TokenCredentialRequest for authentication. impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( impersonationProxyURL, impersonationProxyCACertPEM, "", @@ -398,7 +384,10 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, - expectedWhoAmIRequestResponse(env.TestUser.ExpectedUsername, append(env.TestUser.ExpectedGroups, "system:authenticated")), + expectedWhoAmIRequestResponse( + env.TestUser.ExpectedUsername, + append(env.TestUser.ExpectedGroups, "system:authenticated"), + ), whoAmI, ) @@ -410,9 +399,25 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, - expectedWhoAmIRequestResponse("system:anonymous", []string{"system:unauthenticated"}), + expectedWhoAmIRequestResponse( + "system:anonymous", + []string{"system:unauthenticated"}, + ), whoAmI, ) + + // Test using a service account token. Authenticating as Service Accounts through the impersonation + // proxy is not supported, so it should fail. + namespaceName := createTestNamespace(t, adminClient) + impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials( + &loginv1alpha1.ClusterCredential{Token: createServiceAccountToken(ctx, t, adminClient, namespaceName)}, + impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge + whoAmI, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.Error(t, err) + // The server checks that we have a UID in the request and rejects it with a 422 Unprocessable Entity. + // The API machinery turns 422's into this error text... + require.Contains(t, err.Error(), "the server rejected our request due to an error in our request") }) t.Run("kubectl as a client", func(t *testing.T) { @@ -558,13 +563,15 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("websocket client", func(t *testing.T) { + namespaceName := createTestNamespace(t, adminClient) + library.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespace.Name, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", }) impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") @@ -573,7 +580,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl dest, _ := url.Parse(impersonationProxyURL) dest.Scheme = "wss" - dest.Path = "/api/v1/namespaces/" + namespace.Name + "/configmaps" + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" dest.RawQuery = "watch=1&resourceVersion=0" dialer := websocket.Dialer{ @@ -600,14 +607,14 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.NoError(t, err) // perform a create through the admin client - _, err = adminClient.CoreV1().ConfigMaps(namespace.Name).Create(ctx, + _, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1"}}, metav1.CreateOptions{}, ) require.NoError(t, err) t.Cleanup(func() { - err = adminClient.CoreV1().ConfigMaps(namespace.Name).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) - require.NoError(t, err) + require.NoError(t, adminClient.CoreV1().ConfigMaps(namespaceName). + DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})) }) // see if the websocket client received an event for the create @@ -695,6 +702,64 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) } +func createTestNamespace(t *testing.T, adminClient kubernetes.Interface) string { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + namespace, err := adminClient.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "impersonation-integration-test-"}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + t.Logf("cleaning up test namespace %s", namespace.Name) + require.NoError(t, adminClient.CoreV1().Namespaces().Delete(ctx, namespace.Name, metav1.DeleteOptions{})) + }) + return namespace.Name +} + +func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, namespaceName string) string { + t.Helper() + + serviceAccount, err := adminClient.CoreV1().ServiceAccounts(namespaceName).Create(ctx, + &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{GenerateName: "int-test-service-account-"}}, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, adminClient.CoreV1().ServiceAccounts(namespaceName). + Delete(context.Background(), serviceAccount.Name, metav1.DeleteOptions{})) + }) + + secret, err := adminClient.CoreV1().Secrets(namespaceName).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "int-test-service-account-token-", + Annotations: map[string]string{ + corev1.ServiceAccountNameKey: serviceAccount.Name, + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + }, metav1.CreateOptions{}) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, adminClient.CoreV1().Secrets(namespaceName). + Delete(context.Background(), secret.Name, metav1.DeleteOptions{})) + }) + + library.RequireEventuallyWithoutError(t, func() (bool, error) { + secret, err = adminClient.CoreV1().Secrets(namespaceName).Get(ctx, secret.Name, metav1.GetOptions{}) + if err != nil { + return false, err + } + return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil + }, time.Minute, time.Second) + + return string(secret.Data[corev1.ServiceAccountTokenKey]) +} + func expectedWhoAmIRequestResponse(username string, groups []string) *identityv1alpha1.WhoAmIRequest { return &identityv1alpha1.WhoAmIRequest{ Status: identityv1alpha1.WhoAmIRequestStatus{ diff --git a/test/integration/whoami_test.go b/test/integration/whoami_test.go index f13c7cc2..f30175ec 100644 --- a/test/integration/whoami_test.go +++ b/test/integration/whoami_test.go @@ -75,13 +75,9 @@ func TestWhoAmI_ServiceAccount_Legacy(t *testing.T) { }, metav1.CreateOptions{}) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } - err := kubeClient.Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }() + t.Cleanup(func() { + require.NoError(t, kubeClient.Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{})) + }) sa, err := kubeClient.ServiceAccounts(ns.Name).Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -152,13 +148,9 @@ func TestWhoAmI_ServiceAccount_TokenRequest(t *testing.T) { }, metav1.CreateOptions{}) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } - err := kubeClient.Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}) - require.NoError(t, err) - }() + t.Cleanup(func() { + require.NoError(t, kubeClient.Namespaces().Delete(context.Background(), ns.Name, metav1.DeleteOptions{})) + }) sa, err := kubeClient.ServiceAccounts(ns.Name).Create(ctx, &corev1.ServiceAccount{ ObjectMeta: metav1.ObjectMeta{ @@ -287,13 +279,10 @@ func TestWhoAmI_CSR(t *testing.T) { ) require.NoError(t, err) - defer func() { - if t.Failed() { - return - } - err := kubeClient.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}) - require.NoError(t, err) - }() + t.Cleanup(func() { + require.NoError(t, kubeClient.CertificatesV1beta1().CertificateSigningRequests(). + Delete(context.Background(), csrName, metav1.DeleteOptions{})) + }) // this is a blind update with no resource version checks, which is only safe during tests // use the beta CSR API to support older clusters From a918e9fb97fb5c076f29d3c44e4b9d692321b2b1 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 10:04:24 -0800 Subject: [PATCH 132/203] concierge_impersonation_proxy_test.go: Fix lint error in previous commit --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index b701fb19..6d038181 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -412,7 +412,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials( &loginv1alpha1.ClusterCredential{Token: createServiceAccountToken(ctx, t, adminClient, namespaceName)}, impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge - whoAmI, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + _, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.Error(t, err) // The server checks that we have a UID in the request and rejects it with a 422 Unprocessable Entity. From fcd8c585c3b681f7d908f47e230abd269a578292 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 11 Mar 2021 13:04:36 -0500 Subject: [PATCH 133/203] test/integration: update "kubectl port-forward" test to use non-privileged port This was failing on our laptops because 443 is a privileged port. Signed-off-by: Margo Crawford --- test/integration/concierge_impersonation_proxy_test.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 6d038181..d3cce256 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/gorilla/websocket" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" @@ -537,18 +538,21 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - portForwardCmd, _, _ := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, podName, "443:8443") + portForwardCmd, _, stderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, podName, "8443:8443") portForwardCmd.Env = envVarsWithProxy // start, but don't wait for the command to finish err = portForwardCmd.Start() require.NoError(t, err, `"kubectl port-forward" failed`) + go func() { + assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, stderr.String()) + }() // then run curl something against it time.Sleep(time.Second) timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1") + curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") var curlStdOut, curlStdErr bytes.Buffer curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr From 22ca2da1fff9a202f9c3ded9a46f169694b095f9 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Thu, 11 Mar 2021 15:10:16 -0500 Subject: [PATCH 134/203] test/integration: add "kubectl attach" test to TestImpersonationProxy Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 82 ++++++++++++++++--- test/library/client.go | 29 +++++++ 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index d3cce256..e9b7175b 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -18,6 +18,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "testing" "time" @@ -47,6 +48,30 @@ import ( "go.pinniped.dev/test/library" ) +// syncBuffer wraps bytes.Buffer with a mutex so we don't have races in our test code. +type syncBuffer struct { + buf bytes.Buffer + mu sync.Mutex +} + +func (sb *syncBuffer) String() string { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.String() +} + +func (sb *syncBuffer) Read(b []byte) (int, error) { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.Read(b) +} + +func (sb *syncBuffer) Write(b []byte) (int, error) { + sb.mu.Lock() + defer sb.mu.Unlock() + return sb.buf.Write(b) +} + // Note that this test supports being run on all of our integration test cluster types: // - load balancers not supported, has squid proxy (e.g. kind) // - load balancers supported, has squid proxy (e.g. EKS) @@ -471,11 +496,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) // func to create kubectl commands with a kubeconfig - kubectlCommand := func(timeout context.Context, args ...string) (*exec.Cmd, *bytes.Buffer, *bytes.Buffer) { + kubectlCommand := func(timeout context.Context, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) { allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) //nolint:gosec // we are not performing malicious argument injection against ourselves kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) - var stdout, stderr bytes.Buffer + var stdout, stderr syncBuffer kubectlCmd.Stdout = &stdout kubectlCmd.Stderr = &stderr kubectlCmd.Env = envVarsWithProxy @@ -501,44 +526,45 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) require.NoError(t, err) require.Greater(t, len(pods.Items), 0) - var podName string + var conciergePod *corev1.Pod for _, pod := range pods.Items { + pod := pod if !strings.Contains(pod.Name, "kube-cert-agent") { - podName = pod.Name + conciergePod = &pod } } - if podName == "" { + if conciergePod == nil { t.Error("could not find a concierge pod") } // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) - stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) + stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) require.NoError(t, err, `"kubectl exec" failed`) require.Equal(t, echoString+"\n", stdout) // run the kubectl cp command localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) - _, _, err = runKubectl("cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, podName, remoteEchoFile), localEchoFile) + _, _, err = runKubectl("cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, conciergePod.Name, remoteEchoFile), localEchoFile) require.NoError(t, err, `"kubectl cp" failed`) localEchoFileData, err := ioutil.ReadFile(localEchoFile) require.NoError(t, err) require.Equal(t, echoString+"\n", string(localEchoFileData)) defer func() { - _, _, _ = runKubectl("exec", "--namespace", env.ConciergeNamespace, podName, "--", "rm", remoteEchoFile) // cleanup remote echo file + _, _, _ = runKubectl("exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "rm", remoteEchoFile) // cleanup remote echo file }() // run the kubectl logs command logLinesCount := 10 - stdout, _, err = runKubectl("logs", "--namespace", env.ConciergeNamespace, podName, fmt.Sprintf("--tail=%d", logLinesCount)) + stdout, _, err = runKubectl("logs", "--namespace", env.ConciergeNamespace, conciergePod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) require.NoError(t, err, `"kubectl logs" failed`) require.Equalf(t, logLinesCount, strings.Count(stdout, "\n"), "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - portForwardCmd, _, stderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, podName, "8443:8443") + portForwardCmd, _, stderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") portForwardCmd.Env = envVarsWithProxy // start, but don't wait for the command to finish @@ -564,6 +590,42 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // we expect this to 403, but all we care is that it gets through require.Contains(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + + // run the kubectl attach command + namespaceName := createTestNamespace(t, adminClient) + attachPod := library.CreatePod(ctx, t, "impersonation-proxy-attach", namespaceName, corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "impersonation-proxy-attach", + Image: conciergePod.Spec.Containers[0].Image, + Command: []string{"bash"}, + Args: []string{"-c", `while true; do read VAR; echo "VAR: $VAR"; done`}, + Stdin: true, + }, + }, + }) + attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) + attachCmd.Env = envVarsWithProxy + attachStdin, err := attachCmd.StdinPipe() + require.NoError(t, err) + + // start but don't wait for the attach command + err = attachCmd.Start() + require.NoError(t, err) + + // write to stdin on the attach process + _, err = attachStdin.Write([]byte(echoString + "\n")) + require.NoError(t, err) + + // see that we can read stdout and it spits out stdin output back to us + wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) + require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*30, time.Second, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) + + // close stdin and attach process should exit + err = attachStdin.Close() + require.NoError(t, err) + err = attachCmd.Wait() + require.NoError(t, err) }) t.Run("websocket client", func(t *testing.T) { diff --git a/test/library/client.go b/test/library/client.go index 7371b974..c53acc6f 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -439,6 +439,35 @@ func CreateTokenCredentialRequest(ctx context.Context, t *testing.T, spec v1alph ) } +func CreatePod(ctx context.Context, t *testing.T, name, namespace string, spec corev1.PodSpec) *corev1.Pod { + t.Helper() + + client := NewKubernetesClientset(t) + pods := client.CoreV1().Pods(namespace) + + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + created, err := pods.Create(ctx, &corev1.Pod{ObjectMeta: testObjectMeta(t, name), Spec: spec}, metav1.CreateOptions{}) + require.NoError(t, err) + t.Logf("created test Pod %s", created.Name) + + t.Cleanup(func() { + t.Logf("cleaning up test Pod %s", created.Name) + err := pods.Delete(context.Background(), created.Name, metav1.DeleteOptions{}) + require.NoError(t, err) + }) + + var result *corev1.Pod + require.Eventuallyf(t, func() bool { + var err error + result, err = pods.Get(ctx, created.Name, metav1.GetOptions{}) + require.NoError(t, err) + return result.Status.Phase == corev1.PodRunning + }, 15*time.Second, 1*time.Second, "expected the Pod to go into phase %s", corev1.PodRunning) + return result +} + func WaitForUserToHaveAccess(t *testing.T, user string, groups []string, shouldHaveAccessTo *authorizationv1.ResourceAttributes) { t.Helper() client := NewKubernetesClientset(t) From 29d7f406f727335aa83d20c357cf2525aa609d29 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 12:52:39 -0800 Subject: [PATCH 135/203] Test double impersonation as the cluster admin --- .../concierge/impersonator/impersonator.go | 55 +++++++++---------- .../concierge_impersonation_proxy_test.go | 34 +++++++++++- 2 files changed, 60 insertions(+), 29 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index f943db80..e6b5d5fe 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -61,12 +61,12 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. var listener net.Listener constructServer := func() (func(stopCh <-chan struct{}) error, error) { - // bare minimum server side scheme to allow for status messages to be encoded + // Bare minimum server side scheme to allow for status messages to be encoded. scheme := runtime.NewScheme() metav1.AddToGroupVersion(scheme, metav1.Unversioned) codecs := serializer.NewCodecFactory(scheme) - // this is unused for now but it is a safe value that we could use in the future + // This is unused for now but it is a safe value that we could use in the future. defaultEtcdPathPrefix := "/pinniped-impersonation-proxy-registry" recommendedOptions := genericoptions.NewRecommendedOptions( @@ -77,18 +77,22 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider // serving certs (end user facing) recommendedOptions.SecureServing.BindPort = port - // wire up the impersonation proxy signer CA as a valid authenticator for client cert auth - // TODO fix comments + // Wire up the impersonation proxy signer CA as another valid authenticator for client cert auth, + // along with the Kube API server's CA. kubeClient, err := kubeclient.New(clientOpts...) if err != nil { return nil, err } - kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClient.Kubernetes) + kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController( + "client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClient.Kubernetes, + ) if err != nil { return nil, err } recommendedOptions.Authentication.ClientCert.ClientCA = "---irrelevant-but-needs-to-be-non-empty---" // drop when we pick up https://github.com/kubernetes/kubernetes/pull/100055 - recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider(impersonationProxySignerCA, kubeClientCA) + recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider( + impersonationProxySignerCA, kubeClientCA, + ) if recOpts != nil { recOpts(recommendedOptions) @@ -108,16 +112,16 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. return nil, err } - // loopback authentication to this server does not really make sense since we just proxy everything to KAS - // thus we replace loopback connection config with one that does direct connections to KAS - // loopback config is mainly used by post start hooks, so this is mostly future proofing + // Loopback authentication to this server does not really make sense since we just proxy everything to + // the Kube API server, thus we replace loopback connection config with one that does direct connections + // the Kube API server. Loopback config is mainly used by post start hooks, so this is mostly future proofing. serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClient.ProtoConfig) // assume proto is safe (hooks can override) - // remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken - // see sanity checks at the end of this function + // Remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken. + // See sanity checks at the end of this function. serverConfig.LoopbackClientConfig.BearerToken = "" - // assume proto config is safe because transport level configs do not use rest.ContentConfig - // thus if we are interacting with actual APIs, they should be using pre-built clients + // Assume proto config is safe because transport level configs do not use rest.ContentConfig. + // Thus if we are interacting with actual APIs, they should be using pre-built clients. impersonationProxy, err := newImpersonationReverseProxy(rest.CopyConfig(kubeClient.ProtoConfig)) if err != nil { return nil, err @@ -125,23 +129,22 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler { - // we ignore the passed in handler because we never have any REST APIs to delegate to + // We ignore the passed in handler because we never have any REST APIs to delegate to. handler := defaultBuildHandlerChainFunc(impersonationProxy, c) handler = securityheader.Wrap(handler) return handler } - // TODO integration test this authorizer logic with system:masters + double impersonation - // overwrite the delegating authorizer with one that only cares about impersonation - // empty string is disallowed because request info has had bugs in the past where it would leave it empty + // Overwrite the delegating authorizer with one that only cares about impersonation. + // Empty string is disallowed because request info has had bugs in the past where it would leave it empty. disallowedVerbs := sets.NewString("", "impersonate") noImpersonationAuthorizer := &comparableAuthorizer{ AuthorizerFunc: func(a authorizer.Attributes) (authorizer.Decision, string, error) { - // supporting impersonation is not hard, it would just require a bunch of testing - // and configuring the audit layer (to preserve the caller) which we can do later - // we would also want to delete the incoming impersonation headers + // Supporting impersonation is not hard, it would just require a bunch of testing + // and configuring the audit layer (to preserve the caller) which we can do later. + // We would also want to delete the incoming impersonation headers // instead of overwriting the delegating authorizer, we would - // actually use it to make the impersonation authorization checks + // actually use it to make the impersonation authorization checks. if disallowedVerbs.Has(a.GetVerb()) { return authorizer.DecisionDeny, "impersonation is not allowed or invalid verb", nil } @@ -149,7 +152,7 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. return authorizer.DecisionAllow, "deferring authorization to kube API server", nil }, } - // TODO write a big comment explaining wth this is doing + // Set our custom authorizer before calling Compete(), which will use it. serverConfig.Authorization.Authorizer = noImpersonationAuthorizer impersonationProxyServer, err := serverConfig.Complete().New("impersonation-proxy", genericapiserver.NewEmptyDelegate()) @@ -159,20 +162,16 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. preparedRun := impersonationProxyServer.PrepareRun() - // wait until the very end to do sanity checks - + // Sanity check. Make sure that our custom authorizer is still in place and did not get changed or wrapped. if preparedRun.Authorizer != noImpersonationAuthorizer { return nil, constable.Error("invalid mutation of impersonation authorizer detected") } - // assert that we have a functioning token file to use and no bearer token + // Sanity check. Assert that we have a functioning token file to use and no bearer token. if len(preparedRun.LoopbackClientConfig.BearerToken) != 0 || len(preparedRun.LoopbackClientConfig.BearerTokenFile) == 0 { return nil, constable.Error("invalid impersonator loopback rest config has wrong bearer token semantics") } - // TODO make sure this is closed on error - _ = preparedRun.SecureServingInfo.Listener - return preparedRun.Run, nil } diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index e9b7175b..cfc33a6c 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -370,7 +370,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Len(t, listResult.Items, 0) }) - t.Run("double impersonation is blocked", func(t *testing.T) { + t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { // Create an RBAC rule to allow this user to read/write everything. library.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, @@ -401,6 +401,38 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl env.TestUser.ExpectedUsername)) }) + // This is a separate test from the above double impersonation test because the cluster admin user gets special + // authorization treatment from the Kube API server code that we are using, and we want to ensure that we are blocking + // double impersonation even for the cluster admin. + t.Run("double impersonation as a cluster admin user is blocked", func(t *testing.T) { + // Copy the admin credentials from the admin kubeconfig. + adminClientRestConfig := library.NewClientConfig(t) + + if adminClientRestConfig.BearerToken == "" && adminClientRestConfig.CertData == nil && adminClientRestConfig.KeyData == nil { + t.Skip("The admin kubeconfig does not include credentials, so skipping this test.") + } + + clusterAdminCredentials := &loginv1alpha1.ClusterCredential{ + Token: adminClientRestConfig.BearerToken, + ClientCertificateData: string(adminClientRestConfig.CertData), + ClientKeyData: string(adminClientRestConfig.KeyData), + } + + // Make a client using the admin credentials which will send requests through the impersonation proxy + // and will also add impersonate headers to the request. + doubleImpersonationKubeClient := newImpersonationProxyClientWithCredentials( + clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate", + ).Kubernetes + + _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // Double impersonation is not supported yet, so we should get an error. + require.EqualError(t, err, fmt.Sprintf( + `users "other-user-to-impersonate" is forbidden: `+ + `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `impersonation is not allowed or invalid verb`, + "kubernetes-admin")) + }) + t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { // Test using the TokenCredentialRequest for authentication. impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( From 2d28d1da19e22d6adca60c39ee6916426e229c50 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Thu, 11 Mar 2021 16:20:25 -0500 Subject: [PATCH 136/203] Implement all optional methods in dynamic certs provider Signed-off-by: Monis Khan --- cmd/local-user-authenticator/main.go | 2 +- cmd/local-user-authenticator/main_test.go | 5 +- .../dynamiccertauthority_test.go | 42 ++++--- .../concierge/impersonator/impersonator.go | 16 +-- .../impersonator/impersonator_test.go | 13 ++- internal/concierge/server/server.go | 6 +- .../controller/apicerts/certs_observer.go | 7 +- .../apicerts/certs_observer_test.go | 52 ++++++--- .../impersonatorconfig/impersonator_config.go | 25 +++-- .../impersonator_config_test.go | 14 +-- internal/controller/kubecertagent/execer.go | 27 +++-- .../controller/kubecertagent/execer_test.go | 67 +++++++++++- internal/dynamiccert/provider.go | 103 +++++++++++++----- 13 files changed, 268 insertions(+), 111 deletions(-) diff --git a/cmd/local-user-authenticator/main.go b/cmd/local-user-authenticator/main.go index 379cde21..f4f2249d 100644 --- a/cmd/local-user-authenticator/main.go +++ b/cmd/local-user-authenticator/main.go @@ -355,7 +355,7 @@ func run() error { kubeinformers.WithNamespace(namespace), ) - dynamicCertProvider := dynamiccert.New() + dynamicCertProvider := dynamiccert.New("local-user-authenticator-tls-serving-certificate") startControllers(ctx, dynamicCertProvider, client.Kubernetes, kubeInformers) plog.Debug("controllers are ready") diff --git a/cmd/local-user-authenticator/main_test.go b/cmd/local-user-authenticator/main_test.go index b755f1ac..632fbde0 100644 --- a/cmd/local-user-authenticator/main_test.go +++ b/cmd/local-user-authenticator/main_test.go @@ -473,8 +473,9 @@ func newCertProvider(t *testing.T) (dynamiccert.Provider, []byte, string) { certPEM, keyPEM, err := certauthority.ToPEM(cert) require.NoError(t, err) - certProvider := dynamiccert.New() - certProvider.Set(certPEM, keyPEM) + certProvider := dynamiccert.New(t.Name()) + err = certProvider.SetCertKeyContent(certPEM, keyPEM) + require.NoError(t, err) return certProvider, ca.Bundle(), serverName } diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go index 5482af71..41e0f291 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go @@ -17,7 +17,7 @@ import ( func TestCAIssuePEM(t *testing.T) { t.Parallel() - provider := dynamiccert.New() + provider := dynamiccert.New(t.Name()) ca := New(provider) goodCACrtPEM0, goodCAKeyPEM0, err := testutil.CreateCertificate( @@ -44,12 +44,12 @@ func TestCAIssuePEM(t *testing.T) { { name: "only cert", caCrtPEM: goodCACrtPEM0, - wantError: "could not load CA: tls: failed to find any PEM data in key input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in key input", }, { name: "only key", caKeyPEM: goodCAKeyPEM0, - wantError: "could not load CA: tls: failed to find any PEM data in certificate input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input", }, { name: "new cert+key", @@ -68,19 +68,19 @@ func TestCAIssuePEM(t *testing.T) { name: "bad cert", caCrtPEM: []byte("this is not a cert"), caKeyPEM: goodCAKeyPEM0, - wantError: "could not load CA: tls: failed to find any PEM data in certificate input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input", }, { name: "bad key", caCrtPEM: goodCACrtPEM0, caKeyPEM: []byte("this is not a key"), - wantError: "could not load CA: tls: failed to find any PEM data in key input", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: failed to find any PEM data in key input", }, { name: "mismatch cert+key", caCrtPEM: goodCACrtPEM0, caKeyPEM: goodCAKeyPEM1, - wantError: "could not load CA: tls: private key does not match public key", + wantError: "TestCAIssuePEM: attempt to set invalid key pair: tls: private key does not match public key", }, { name: "good cert+key again", @@ -94,17 +94,7 @@ func TestCAIssuePEM(t *testing.T) { // Can't run these steps in parallel, because each one depends on the previous steps being // run. - if step.caCrtPEM != nil || step.caKeyPEM != nil { - provider.Set(step.caCrtPEM, step.caKeyPEM) - } - - crtPEM, keyPEM, err := ca.IssuePEM( - pkix.Name{ - CommonName: "some-common-name", - }, - []string{"some-dns-name", "some-other-dns-name"}, - time.Hour*24, - ) + crtPEM, keyPEM, err := issuePEM(provider, ca, step.caCrtPEM, step.caKeyPEM) if step.wantError != "" { require.EqualError(t, err, step.wantError) @@ -126,3 +116,21 @@ func TestCAIssuePEM(t *testing.T) { }) } } + +func issuePEM(provider dynamiccert.Provider, ca *CA, caCrt, caKey []byte) ([]byte, []byte, error) { + // if setting fails, look at that error + if caCrt != nil || caKey != nil { + if err := provider.SetCertKeyContent(caCrt, caKey); err != nil { + return nil, nil, err + } + } + + // otherwise check to see if their is an issuing error + return ca.IssuePEM( + pkix.Name{ + CommonName: "some-common-name", + }, + []string{"some-dns-name", "some-other-dns-name"}, + time.Hour*24, + ) +} diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index e6b5d5fe..a14e87bc 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -12,11 +12,10 @@ import ( "strings" "time" - "k8s.io/apimachinery/pkg/util/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/request" @@ -27,6 +26,7 @@ import ( "k8s.io/client-go/transport" "go.pinniped.dev/internal/constable" + "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/httputil/securityheader" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/plog" @@ -39,22 +39,22 @@ import ( // Instead, call the factory function again to get a new start function. type FactoryFunc func( port int, - dynamicCertProvider dynamiccertificates.CertKeyContentProvider, - impersonationProxySignerCA dynamiccertificates.CAContentProvider, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCA dynamiccert.Public, ) (func(stopCh <-chan struct{}) error, error) func New( port int, - dynamicCertProvider dynamiccertificates.CertKeyContentProvider, // TODO: we need to check those optional interfaces and see what we need to implement - impersonationProxySignerCA dynamiccertificates.CAContentProvider, // TODO: we need to check those optional interfaces and see what we need to implement + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCA dynamiccert.Public, ) (func(stopCh <-chan struct{}) error, error) { return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, nil, nil) } func newInternal( //nolint:funlen // yeah, it's kind of long. port int, - dynamicCertProvider dynamiccertificates.CertKeyContentProvider, - impersonationProxySignerCA dynamiccertificates.CAContentProvider, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCA dynamiccert.Public, clientOpts []kubeclient.Option, // for unit testing, should always be nil in production recOpts func(*genericoptions.RecommendedOptions), // for unit testing, should always be nil in production ) (func(stopCh <-chan struct{}) error, error) { diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 31e9aae9..66cbd3d3 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -18,7 +18,6 @@ import ( "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" - "k8s.io/apiserver/pkg/server/dynamiccertificates" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/rest" @@ -26,6 +25,7 @@ import ( featuregatetesting "k8s.io/component-base/featuregate/testing" "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" ) @@ -35,11 +35,16 @@ func TestNew(t *testing.T) { ca, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) require.NoError(t, err) + caKey, err := ca.PrivateKeyToPEM() + require.NoError(t, err) + caContent := dynamiccert.New("ca") + err = caContent.SetCertKeyContent(ca.Bundle(), caKey) + require.NoError(t, err) + cert, key, err := ca.IssuePEM(pkix.Name{CommonName: "example.com"}, []string{"example.com"}, time.Hour) require.NoError(t, err) - certKeyContent, err := dynamiccertificates.NewStaticCertKeyContent("cert-key", cert, key) - require.NoError(t, err) - caContent, err := dynamiccertificates.NewStaticCAContent("ca", ca.Bundle()) + certKeyContent := dynamiccert.New("cert-key") + err = certKeyContent.SetCertKeyContent(cert, key) require.NoError(t, err) // Punch out just enough stuff to make New actually run without error. diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index d485ee19..8602d07d 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -114,15 +114,15 @@ func (a *App) runServer(ctx context.Context) error { // is stored in a k8s Secret. Therefore it also effectively acting as // an in-memory cache of what is stored in the k8s Secret, helping to // keep incoming requests fast. - dynamicServingCertProvider := dynamiccert.New() + dynamicServingCertProvider := dynamiccert.New("concierge-serving-cert") // This cert provider will be used to provide the Kube signing key to the // cert issuer used to issue certs to Pinniped clients wishing to login. - dynamicSigningCertProvider := dynamiccert.New() + dynamicSigningCertProvider := dynamiccert.New("concierge-kube-signing-cert") // This cert provider will be used to provide the impersonation proxy signing key to the // cert issuer used to issue certs to Pinniped clients wishing to login. - impersonationProxySigningCertProvider := dynamiccert.New() + impersonationProxySigningCertProvider := dynamiccert.New("impersonation-proxy-signing-cert") // Get the "real" name of the login concierge API group (i.e., the API group name with the // injected suffix). diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go index c6380741..c8b6f2b8 100644 --- a/internal/controller/apicerts/certs_observer.go +++ b/internal/controller/apicerts/certs_observer.go @@ -57,12 +57,15 @@ func (c *certsObserverController) Sync(_ controllerlib.Context) error { if notFound { klog.Info("certsObserverController Sync found that the secret does not exist yet or was deleted") // The secret does not exist yet or was deleted. - c.dynamicCertProvider.Set(nil, nil) + c.dynamicCertProvider.UnsetCertKeyContent() return nil } // Mutate the in-memory cert provider to update with the latest cert values. - c.dynamicCertProvider.Set(certSecret.Data[TLSCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]) + if err := c.dynamicCertProvider.SetCertKeyContent(certSecret.Data[TLSCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]); err != nil { + return fmt.Errorf("failed to set serving cert/key content from secret %s/%s: %w", c.namespace, c.certsSecretResourceName, err) + } + klog.Info("certsObserverController Sync updated certs in the dynamic cert provider") return nil } diff --git a/internal/controller/apicerts/certs_observer_test.go b/internal/controller/apicerts/certs_observer_test.go index bd1540cc..553e66f3 100644 --- a/internal/controller/apicerts/certs_observer_test.go +++ b/internal/controller/apicerts/certs_observer_test.go @@ -5,7 +5,9 @@ package apicerts import ( "context" + "strings" "testing" + "time" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -94,6 +96,7 @@ func TestObserverControllerInformerFilters(t *testing.T) { } func TestObserverControllerSync(t *testing.T) { + name := t.Name() spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" const certsSecretResourceName = "some-resource-name" @@ -142,7 +145,7 @@ func TestObserverControllerSync(t *testing.T) { kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) - dynamicCertProvider = dynamiccert.New() + dynamicCertProvider = dynamiccert.New(name) }) it.After(func() { @@ -160,7 +163,14 @@ func TestObserverControllerSync(t *testing.T) { err := kubeInformerClient.Tracker().Add(unrelatedSecret) r.NoError(err) - dynamicCertProvider.Set([]byte("some cert"), []byte("some private key")) + crt, key, err := testutil.CreateCertificate( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + ) + require.NoError(t, err) + + err = dynamicCertProvider.SetCertKeyContent(crt, key) + r.NoError(err) }) it("sets the dynamicCertProvider's cert and key to nil", func() { @@ -176,6 +186,12 @@ func TestObserverControllerSync(t *testing.T) { when("there is a serving cert Secret with the expected keys already in the installation namespace", func() { it.Before(func() { + crt, key, err := testutil.CreateCertificate( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + ) + require.NoError(t, err) + apiServingCertSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: certsSecretResourceName, @@ -183,24 +199,29 @@ func TestObserverControllerSync(t *testing.T) { }, Data: map[string][]byte{ "caCertificate": []byte("fake cert"), - "tlsPrivateKey": []byte("fake private key"), - "tlsCertificateChain": []byte("fake cert chain"), + "tlsPrivateKey": key, + "tlsCertificateChain": crt, }, } - err := kubeInformerClient.Tracker().Add(apiServingCertSecret) + err = kubeInformerClient.Tracker().Add(apiServingCertSecret) r.NoError(err) - dynamicCertProvider.Set(nil, nil) + dynamicCertProvider.UnsetCertKeyContent() }) it("updates the dynamicCertProvider's cert and key", func() { startInformersAndController() + + actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() + r.Nil(actualCertChain) + r.Nil(actualKey) + err := controllerlib.TestSync(t, subject, *syncContext) r.NoError(err) - actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() - r.Equal("fake cert chain", string(actualCertChain)) - r.Equal("fake private key", string(actualKey)) + actualCertChain, actualKey = dynamicCertProvider.CurrentCertKeyContent() + r.True(strings.HasPrefix(string(actualCertChain), `-----BEGIN CERTIFICATE-----`), "not a cert:\n%s", string(actualCertChain)) + r.True(strings.HasPrefix(string(actualKey), `-----BEGIN PRIVATE KEY-----`), "not a key:\n%s", string(actualKey)) }) }) @@ -216,17 +237,22 @@ func TestObserverControllerSync(t *testing.T) { err := kubeInformerClient.Tracker().Add(apiServingCertSecret) r.NoError(err) - dynamicCertProvider.Set(nil, nil) + dynamicCertProvider.UnsetCertKeyContent() }) - it("set the missing values in the dynamicCertProvider as nil", func() { + it("returns an error and does not change the dynamicCertProvider", func() { startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.NoError(err) actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() r.Nil(actualCertChain) r.Nil(actualKey) + + err := controllerlib.TestSync(t, subject, *syncContext) + r.EqualError(err, "failed to set serving cert/key content from secret some-namespace/some-resource-name: TestObserverControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input") + + actualCertChain, actualKey = dynamicCertProvider.CurrentCertKeyContent() + r.Nil(actualCertChain) + r.Nil(actualKey) }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 389905d7..6f81901d 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -117,7 +117,7 @@ func NewImpersonatorConfigController( clock: clock, impersonationSigningCertProvider: impersonationSigningCertProvider, impersonatorFunc: impersonatorFunc, - tlsServingCertDynamicCertProvider: dynamiccert.New(), + tlsServingCertDynamicCertProvider: dynamiccert.New("impersonation-proxy-serving-cert"), }, }, withInformer( @@ -238,7 +238,7 @@ func (c *impersonatorConfigController) doSync(syncCtx controllerlib.Context) (*v nameInfo, err := c.findDesiredTLSCertificateName(config) if err != nil { // Unexpected error while determining the name that should go into the certs, so clear any existing certs. - c.tlsServingCertDynamicCertProvider.Set(nil, nil) + c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent() return nil, err } @@ -371,7 +371,7 @@ func (c *impersonatorConfigController) ensureImpersonatorIsStarted(syncCtx contr startImpersonatorFunc, err := c.impersonatorFunc( impersonationProxyPort, c.tlsServingCertDynamicCertProvider, - dynamiccert.NewCAProvider(c.impersonationSigningCertProvider), + c.impersonationSigningCertProvider, ) if err != nil { return err @@ -750,16 +750,17 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secret) error { certPEM := tlsSecret.Data[v1.TLSCertKey] keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] - _, err := tls.X509KeyPair(certPEM, keyPEM) - if err != nil { - c.tlsServingCertDynamicCertProvider.Set(nil, nil) + + if err := c.tlsServingCertDynamicCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil { + c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent() return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) } + plog.Info("Loading TLS certificates for impersonation proxy", "certPEM", string(certPEM), "secret", c.tlsSecretName, "namespace", c.namespace) - c.tlsServingCertDynamicCertProvider.Set(certPEM, keyPEM) + return nil } @@ -779,7 +780,7 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return err } - c.tlsServingCertDynamicCertProvider.Set(nil, nil) + c.tlsServingCertDynamicCertProvider.UnsetCertKeyContent() return nil } @@ -798,8 +799,8 @@ func (c *impersonatorConfigController) loadSignerCA(status v1alpha1.StrategyStat certPEM := signingCertSecret.Data[apicerts.CACertificateSecretKey] keyPEM := signingCertSecret.Data[apicerts.CACertificatePrivateKeySecretKey] - _, err = tls.X509KeyPair(certPEM, keyPEM) - if err != nil { + + if err := c.impersonationSigningCertProvider.SetCertKeyContent(certPEM, keyPEM); err != nil { return fmt.Errorf("could not load the impersonator's credential signing secret: %w", err) } @@ -807,13 +808,13 @@ func (c *impersonatorConfigController) loadSignerCA(status v1alpha1.StrategyStat "certPEM", string(certPEM), "fromSecret", c.impersonationSignerSecretName, "namespace", c.namespace) - c.impersonationSigningCertProvider.Set(certPEM, keyPEM) + return nil } func (c *impersonatorConfigController) clearSignerCA() { plog.Info("Clearing credential signing certificate for impersonation proxy") - c.impersonationSigningCertProvider.Set(nil, nil) + c.impersonationSigningCertProvider.UnsetCertKeyContent() } func (c *impersonatorConfigController) doSyncResult(nameInfo *certNameInfo, config *impersonator.Config, ca *certauthority.CA) *v1alpha1.CredentialIssuerStrategy { diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index 60d029fb..f926208b 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -30,7 +30,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/clock" - "k8s.io/apiserver/pkg/server/dynamiccertificates" kubeinformers "k8s.io/client-go/informers" kubernetesfake "k8s.io/client-go/kubernetes/fake" coretesting "k8s.io/client-go/testing" @@ -266,6 +265,7 @@ func TestImpersonatorConfigControllerOptions(t *testing.T) { } func TestImpersonatorConfigControllerSync(t *testing.T) { + name := t.Name() spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" const configMapResourceName = "some-configmap-resource-name" @@ -306,8 +306,8 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { var impersonatorFunc = func( port int, - dynamicCertProvider dynamiccertificates.CertKeyContentProvider, - impersonationProxySignerCAProvider dynamiccertificates.CAContentProvider, + dynamicCertProvider dynamiccert.Private, + impersonationProxySignerCAProvider dynamiccert.Public, ) (func(stopCh <-chan struct{}) error, error) { impersonatorFuncWasCalled++ r.Equal(8444, port) @@ -972,7 +972,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { kubeAPIClient = kubernetesfake.NewSimpleClientset() pinnipedAPIClient = pinnipedfake.NewSimpleClientset() frozenNow = time.Date(2021, time.March, 2, 7, 42, 0, 0, time.Local) - signingCertProvider = dynamiccert.New() + signingCertProvider = dynamiccert.New(name) ca := newCA() signingCACertPEM = ca.Bundle() @@ -2408,7 +2408,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns the error", func() { startInformersAndController() - errString := `could not load the impersonator's credential signing secret: tls: failed to find any PEM data in certificate input` + errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input` r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) requireSigningCertProviderIsEmpty() @@ -2423,7 +2423,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it("returns the error", func() { startInformersAndController() - errString := `could not load the impersonator's credential signing secret: tls: failed to find any PEM data in certificate input` + errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input` r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) requireSigningCertProviderIsEmpty() @@ -2459,7 +2459,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addSecretToTrackers(updatedSigner, kubeInformerClient) waitForObjectToAppearInInformer(updatedSigner, kubeInformers.Core().V1().Secrets()) - errString := `could not load the impersonator's credential signing secret: tls: failed to find any PEM data in certificate input` + errString := `could not load the impersonator's credential signing secret: TestImpersonatorConfigControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in certificate input` r.EqualError(runControllerSync(), errString) requireCredentialIssuer(newErrorStrategy(errString)) requireSigningCertProviderIsEmpty() diff --git a/internal/controller/kubecertagent/execer.go b/internal/controller/kubecertagent/execer.go index b8ff1c83..691af1c4 100644 --- a/internal/controller/kubecertagent/execer.go +++ b/internal/controller/kubecertagent/execer.go @@ -11,9 +11,9 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/apimachinery/pkg/util/errors" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/tools/clientcmd" - "k8s.io/klog/v2" configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" @@ -119,8 +119,7 @@ func (c *execerController) Sync(ctx controllerlib.Context) error { c.pinnipedAPIClient, strategyError(c.clock, err), ) - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer with strategy success") - return err + return newAggregate(err, strategyResultUpdateErr) } keyPEM, err := c.podCommandExecutor.Exec(agentPod.Namespace, agentPod.Name, "cat", keyPath) @@ -132,11 +131,20 @@ func (c *execerController) Sync(ctx controllerlib.Context) error { c.pinnipedAPIClient, strategyError(c.clock, err), ) - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer with strategy success") - return err + return newAggregate(err, strategyResultUpdateErr) } - c.dynamicCertProvider.Set([]byte(certPEM), []byte(keyPEM)) + if err := c.dynamicCertProvider.SetCertKeyContent([]byte(certPEM), []byte(keyPEM)); err != nil { + err = fmt.Errorf("failed to set signing cert/key content from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err) + strategyResultUpdateErr := issuerconfig.UpdateStrategy( + ctx.Context, + c.credentialIssuerLocationConfig.Name, + c.credentialIssuerLabels, + c.pinnipedAPIClient, + strategyError(c.clock, err), + ) + return newAggregate(err, strategyResultUpdateErr) + } apiInfo, err := c.getTokenCredentialRequestAPIInfo() if err != nil { @@ -153,8 +161,7 @@ func (c *execerController) Sync(ctx controllerlib.Context) error { LastUpdateTime: metav1.NewTime(c.clock.Now()), }, ) - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer with strategy success") - return err + return newAggregate(err, strategyResultUpdateErr) } return issuerconfig.UpdateStrategy( @@ -219,3 +226,7 @@ func (c *execerController) getKeypairFilePaths(pod *v1.Pod) (string, string) { return certPath, keyPath } + +func newAggregate(errs ...error) error { + return errors.NewAggregate(errs) +} diff --git a/internal/controller/kubecertagent/execer_test.go b/internal/controller/kubecertagent/execer_test.go index 7d1092c9..fe19d198 100644 --- a/internal/controller/kubecertagent/execer_test.go +++ b/internal/controller/kubecertagent/execer_test.go @@ -132,6 +132,7 @@ func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndAr } func TestManagerControllerSync(t *testing.T) { + name := t.Name() spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { const agentPodNamespace = "some-namespace" const agentPodName = "some-agent-pod-name-123" @@ -139,8 +140,6 @@ func TestManagerControllerSync(t *testing.T) { const keyPathAnnotationName = "kube-cert-agent.pinniped.dev/key-path" const fakeCertPath = "/some/cert/path" const fakeKeyPath = "/some/key/path" - const defaultDynamicCertProviderCert = "initial-cert" - const defaultDynamicCertProviderKey = "initial-key" const credentialIssuerResourceName = "ci-resource-name" var r *require.Assertions @@ -159,6 +158,8 @@ func TestManagerControllerSync(t *testing.T) { var fakeCertPEM, fakeKeyPEM string var credentialIssuerGVR schema.GroupVersionResource var frozenNow time.Time + var defaultDynamicCertProviderCert string + var defaultDynamicCertProviderKey string // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. @@ -228,14 +229,23 @@ func TestManagerControllerSync(t *testing.T) { it.Before(func() { r = require.New(t) + crt, key, err := testutil.CreateCertificate( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + ) + require.NoError(t, err) + defaultDynamicCertProviderCert = string(crt) + defaultDynamicCertProviderKey = string(key) + cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) pinnipedAPIClient = pinnipedfake.NewSimpleClientset() kubeClientset = kubernetesfake.NewSimpleClientset() kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClientset, 0) fakeExecutor = &fakePodExecutor{r: r} frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - dynamicCertProvider = dynamiccert.New() - dynamicCertProvider.Set([]byte(defaultDynamicCertProviderCert), []byte(defaultDynamicCertProviderKey)) + dynamicCertProvider = dynamiccert.New(name) + err = dynamicCertProvider.SetCertKeyContent([]byte(defaultDynamicCertProviderCert), []byte(defaultDynamicCertProviderKey)) + r.NoError(err) loadFile := func(filename string) string { bytes, err := ioutil.ReadFile(filename) @@ -669,6 +679,55 @@ func TestManagerControllerSync(t *testing.T) { r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) }) }) + + when("the third resulting pod exec has invalid key data", func() { + var keyParseErrorMessage string + + it.Before(func() { + keyParseErrorMessage = "failed to set signing cert/key content from agent pod some-namespace/some-agent-pod-name-123: TestManagerControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in key input" + fakeExecutor.errorsToReturn = []error{nil, nil} + fakeExecutor.resultsToReturn = []string{fakeCertPEM, ""} + startInformersAndController() + }) + + it("does not update the dynamic certificates provider", func() { + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), keyParseErrorMessage) + requireDynamicCertProviderHasDefaultValues() + }) + + it("creates or updates the the CredentialIssuer status field with an error", func() { + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), keyParseErrorMessage) + + expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: credentialIssuerResourceName, + }, + } + + expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: credentialIssuerResourceName, + }, + Status: configv1alpha1.CredentialIssuerStatus{ + Strategies: []configv1alpha1.CredentialIssuerStrategy{ + { + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: keyParseErrorMessage, + LastUpdateTime: metav1.NewTime(frozenNow), + }, + }, + }, + } + expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) + expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) + expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) + r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) + }) + }) }) }, spec.Parallel(), spec.Report(report.Terminal{})) } diff --git a/internal/dynamiccert/provider.go b/internal/dynamiccert/provider.go index 77687fb1..8cef78bd 100644 --- a/internal/dynamiccert/provider.go +++ b/internal/dynamiccert/provider.go @@ -4,69 +4,112 @@ package dynamiccert import ( + "crypto/tls" "crypto/x509" + "fmt" "sync" "k8s.io/apiserver/pkg/server/dynamiccertificates" ) -// Provider provides a getter, CurrentCertKeyContent(), and a setter, Set(), for a PEM-formatted -// certificate and matching key. type Provider interface { + Private + Public +} + +type Private interface { dynamiccertificates.CertKeyContentProvider - // TODO dynamiccertificates.Notifier - // TODO dynamiccertificates.ControllerRunner ??? - Set(certPEM, keyPEM []byte) + SetCertKeyContent(certPEM, keyPEM []byte) error + UnsetCertKeyContent() + + notifier +} + +type Public interface { + dynamiccertificates.CAContentProvider + + notifier +} + +type notifier interface { + dynamiccertificates.Notifier + dynamiccertificates.ControllerRunner // we do not need this today, but it could grow and change in the future } type provider struct { - certPEM []byte - keyPEM []byte - mutex sync.RWMutex + name string + + // mutex guards all the fields below it + mutex sync.RWMutex + certPEM []byte + keyPEM []byte + listeners []dynamiccertificates.Listener } // New returns an empty Provider. The returned Provider is thread-safe. -func New() Provider { - return &provider{} -} - -func (p *provider) Set(certPEM, keyPEM []byte) { - p.mutex.Lock() // acquire a write lock - defer p.mutex.Unlock() - p.certPEM = certPEM - p.keyPEM = keyPEM +func New(name string) Provider { + return &provider{name: name} } func (p *provider) Name() string { - return "DynamicCertProvider" + return p.name // constant after struct initialization and thus does not need locking } func (p *provider) CurrentCertKeyContent() (cert []byte, key []byte) { - p.mutex.RLock() // acquire a read lock + p.mutex.RLock() defer p.mutex.RUnlock() + return p.certPEM, p.keyPEM } -func NewCAProvider(delegate dynamiccertificates.CertKeyContentProvider) dynamiccertificates.CAContentProvider { - return &caContentProvider{delegate: delegate} +func (p *provider) SetCertKeyContent(certPEM, keyPEM []byte) error { + // always make sure that we have valid PEM data, otherwise + // dynamiccertificates.NewUnionCAContentProvider.VerifyOptions will panic + if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil { + return fmt.Errorf("%s: attempt to set invalid key pair: %w", p.name, err) + } + + p.setCertKeyContent(certPEM, keyPEM) + + return nil } -type caContentProvider struct { - delegate dynamiccertificates.CertKeyContentProvider +func (p *provider) UnsetCertKeyContent() { + p.setCertKeyContent(nil, nil) } -func (c *caContentProvider) Name() string { - return "DynamicCAProvider" +func (p *provider) setCertKeyContent(certPEM, keyPEM []byte) { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.certPEM = certPEM + p.keyPEM = keyPEM + + for _, listener := range p.listeners { + listener.Enqueue() + } } -func (c *caContentProvider) CurrentCABundleContent() []byte { - ca, _ := c.delegate.CurrentCertKeyContent() +func (p *provider) CurrentCABundleContent() []byte { + ca, _ := p.CurrentCertKeyContent() return ca } -func (c *caContentProvider) VerifyOptions() (x509.VerifyOptions, bool) { +func (p *provider) VerifyOptions() (x509.VerifyOptions, bool) { return x509.VerifyOptions{}, false // assume we are unioned via dynamiccertificates.NewUnionCAContentProvider } -// TODO look at both the serving side union struct and the ca side union struct for all optional interfaces -// and then implement everything that makes sense for us to implement +func (p *provider) AddListener(listener dynamiccertificates.Listener) { + p.mutex.Lock() + defer p.mutex.Unlock() + + p.listeners = append(p.listeners, listener) +} + +func (p *provider) RunOnce() error { + return nil // no-op, but we want to make sure to stay in sync with dynamiccertificates.ControllerRunner +} + +func (p *provider) Run(workers int, stopCh <-chan struct{}) { + // no-op, but we want to make sure to stay in sync with dynamiccertificates.ControllerRunner +} From a64786a7289305bff62c41514c42eb120d07ae94 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 11 Mar 2021 15:47:39 -0600 Subject: [PATCH 137/203] Fix TestCLIGetKubeconfigStaticToken for new CLI log output. Signed-off-by: Matt Moyer --- test/integration/cli_test.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index f2479d0c..c5e18907 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "gopkg.in/square/go-jose.v2" @@ -47,9 +48,9 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { pinnipedExe := library.PinnipedCLIPath(t) for _, tt := range []struct { - name string - args []string - expectStderr string + name string + args []string + expectStderrContains []string }{ { name: "newer command, but still using static parameters", @@ -60,12 +61,20 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { "--concierge-authenticator-type", "webhook", "--concierge-authenticator-name", authenticator.Name, }, + expectStderrContains: []string{ + "discovered CredentialIssuer", + "discovered Concierge endpoint", + "discovered Concierge certificate authority bundle", + "validated connection to the cluster", + }, }, } { tt := tt t.Run(tt.name, func(t *testing.T) { stdout, stderr := runPinnipedCLI(t, nil, pinnipedExe, tt.args...) - require.Equal(t, tt.expectStderr, stderr) + for _, s := range tt.expectStderrContains { + assert.Contains(t, stderr, s) + } // Even the deprecated command should now generate a kubeconfig with the new "pinniped login static" command. restConfig := library.NewRestConfigFromKubeconfig(t, stdout) From c9ce067a0eed51479eac887381b9cc379ce282f8 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 11 Mar 2021 16:11:46 -0600 Subject: [PATCH 138/203] Captialize "API" in this error message. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 2 +- cmd/pinniped/cmd/kubeconfig_test.go | 4 ++-- cmd/pinniped/cmd/login_oidc_test.go | 4 ++-- cmd/pinniped/cmd/login_static_test.go | 4 ++-- pkg/conciergeclient/conciergeclient.go | 2 +- pkg/conciergeclient/conciergeclient_test.go | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index 4d6f1da7..cde12fff 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -171,7 +171,7 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f // Validate api group suffix and immediately return an error if it is invalid. if err := groupsuffix.Validate(flags.concierge.apiGroupSuffix); err != nil { - return fmt.Errorf("invalid api group suffix: %w", err) + return fmt.Errorf("invalid API group suffix: %w", err) } execConfig := clientcmdapi.ExecConfig{ diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 53b83830..7a200c6f 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -536,13 +536,13 @@ func TestGetKubeconfig(t *testing.T) { `), }, { - name: "invalid api group suffix", + name: "invalid API group suffix", args: []string{ "--concierge-api-group-suffix", ".starts.with.dot", }, wantError: true, wantStderr: here.Doc(` - Error: invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index c6a4d78e..f36d67cd 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -125,7 +125,7 @@ func TestLoginOIDCCommand(t *testing.T) { `), }, { - name: "invalid api group suffix", + name: "invalid API group suffix", args: []string{ "--issuer", "test-issuer", "--enable-concierge", @@ -136,7 +136,7 @@ func TestLoginOIDCCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index 993313d5..9ee5a969 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -131,7 +131,7 @@ func TestLoginStaticCommand(t *testing.T) { `), }, { - name: "invalid api group suffix", + name: "invalid API group suffix", args: []string{ "--token", "test-token", "--enable-concierge", @@ -142,7 +142,7 @@ func TestLoginStaticCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { diff --git a/pkg/conciergeclient/conciergeclient.go b/pkg/conciergeclient/conciergeclient.go index 0e5fa6f7..895089bd 100644 --- a/pkg/conciergeclient/conciergeclient.go +++ b/pkg/conciergeclient/conciergeclient.go @@ -109,7 +109,7 @@ func WithEndpoint(endpoint string) Option { func WithAPIGroupSuffix(apiGroupSuffix string) Option { return func(c *Client) error { if err := groupsuffix.Validate(apiGroupSuffix); err != nil { - return fmt.Errorf("invalid api group suffix: %w", err) + return fmt.Errorf("invalid API group suffix: %w", err) } c.apiGroupSuffix = apiGroupSuffix return nil diff --git a/pkg/conciergeclient/conciergeclient_test.go b/pkg/conciergeclient/conciergeclient_test.go index 028dda9f..162ecfee 100644 --- a/pkg/conciergeclient/conciergeclient_test.go +++ b/pkg/conciergeclient/conciergeclient_test.go @@ -111,16 +111,16 @@ func TestNew(t *testing.T) { WithEndpoint("https://example.com"), WithAPIGroupSuffix(""), }, - wantErr: "invalid api group suffix: [must contain '.', a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')]", + wantErr: "invalid API group suffix: [must contain '.', a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')]", }, { - name: "invalid api group suffix", + name: "invalid API group suffix", opts: []Option{ WithAuthenticator("jwt", "test-authenticator"), WithEndpoint("https://example.com"), WithAPIGroupSuffix(".starts.with.dot"), }, - wantErr: "invalid api group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", + wantErr: "invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*')", }, { name: "valid", From d2d9b1e49eb40d1112817a01163a764eaa645eff Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 11 Mar 2021 16:13:29 -0600 Subject: [PATCH 139/203] Stop outputting "--concierge-mode" from "pinniped get kubeconfig". Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/kubeconfig.go | 1 - cmd/pinniped/cmd/kubeconfig_test.go | 6 ------ 2 files changed, 7 deletions(-) diff --git a/cmd/pinniped/cmd/kubeconfig.go b/cmd/pinniped/cmd/kubeconfig.go index cde12fff..0528bffa 100644 --- a/cmd/pinniped/cmd/kubeconfig.go +++ b/cmd/pinniped/cmd/kubeconfig.go @@ -230,7 +230,6 @@ func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, f "--concierge-authenticator-type="+flags.concierge.authenticatorType, "--concierge-endpoint="+flags.concierge.endpoint, "--concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(flags.concierge.caBundle), - "--concierge-mode="+flags.concierge.mode.String(), ) // Point kubectl at the concierge endpoint. diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index 7a200c6f..f3b27d5d 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -608,7 +608,6 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=webhook - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== - - --concierge-mode=TokenCredentialRequestAPI - --token=test-token command: '.../path/to/pinniped' env: [] @@ -678,7 +677,6 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=webhook - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== - - --concierge-mode=TokenCredentialRequestAPI - --token-env=TEST_TOKEN command: '.../path/to/pinniped' env: [] @@ -759,7 +757,6 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=jwt - --concierge-endpoint=https://fake-server-url-value - --concierge-ca-bundle-data=ZmFrZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YS12YWx1ZQ== - - --concierge-mode=TokenCredentialRequestAPI - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience @@ -842,7 +839,6 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=webhook - --concierge-endpoint=https://explicit-concierge-endpoint.example.com - --concierge-ca-bundle-data=%s - - --concierge-mode=TokenCredentialRequestAPI - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience @@ -957,7 +953,6 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=jwt - --concierge-endpoint=https://impersonation-proxy-endpoint.test - --concierge-ca-bundle-data=%s - - --concierge-mode=ImpersonationProxy - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience @@ -1064,7 +1059,6 @@ func TestGetKubeconfig(t *testing.T) { - --concierge-authenticator-type=jwt - --concierge-endpoint=https://impersonation-proxy-endpoint.test - --concierge-ca-bundle-data=dGVzdC1jb25jaWVyZ2UtY2E= - - --concierge-mode=ImpersonationProxy - --issuer=https://example.com/issuer - --client-id=pinniped-cli - --scopes=offline_access,openid,pinniped:request-audience From 4f154100ff31808d4c68a4cb0806f8bb4d823156 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 11 Mar 2021 16:14:56 -0600 Subject: [PATCH 140/203] Remove "--concierge-mode" flag from "pinniped login [...]" commands. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/login_oidc.go | 90 ++------------------------- cmd/pinniped/cmd/login_oidc_test.go | 38 ----------- cmd/pinniped/cmd/login_static.go | 30 ++------- cmd/pinniped/cmd/login_static_test.go | 13 ---- 4 files changed, 11 insertions(+), 160 deletions(-) diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 7dd29943..3887ccf7 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -14,14 +14,8 @@ import ( "net/http" "os" "path/filepath" - "strings" "time" - corev1 "k8s.io/api/core/v1" - - authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" - "github.com/coreos/go-oidc/v3/oidc" "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -71,7 +65,6 @@ type oidcLoginFlags struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string - conciergeMode conciergeModeFlag } func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { @@ -102,7 +95,6 @@ func oidcLoginCommand(deps oidcLoginCommandDeps) *cobra.Command { cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the Concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") - cmd.Flags().Var(&flags.conciergeMode, "concierge-mode", "Concierge mode of operation") mustMarkHidden(cmd, "debug-session-cache") mustMarkRequired(cmd, "issuer") @@ -179,37 +171,17 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin } cred := tokenCredential(token) - // If there is no concierge configuration, return the credential directly. - if concierge == nil { - return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) - } + // If the concierge was configured, exchange the credential for a separate short-lived, cluster-specific credential. + if concierge != nil { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() - // If the concierge was configured, we need to do extra steps to make the credential usable. - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - // The exact behavior depends on in which mode the Concierge is operating. - switch flags.conciergeMode { - case modeUnknown, modeTokenCredentialRequestAPI: - // do a credential exchange request - cred, err := deps.exchangeToken(ctx, concierge, token.IDToken.Token) + cred, err = deps.exchangeToken(ctx, concierge, token.IDToken.Token) if err != nil { return fmt.Errorf("could not complete concierge credential exchange: %w", err) } - return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) - - case modeImpersonationProxy: - // Put the token into a TokenCredentialRequest - // put the TokenCredentialRequest in an ExecCredential - req, err := execCredentialForImpersonationProxy(token.IDToken.Token, flags.conciergeAuthenticatorType, flags.conciergeAuthenticatorName, &token.IDToken.Expiry) - if err != nil { - return err - } - return json.NewEncoder(cmd.OutOrStdout()).Encode(req) - - default: - return fmt.Errorf("unsupported Concierge mode %q", flags.conciergeMode.String()) } + return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) } func makeClient(caBundlePaths []string, caBundleData []string) (*http.Client, error) { @@ -271,53 +243,3 @@ func mustGetConfigDir() string { } return filepath.Join(home, ".config", xdgAppName) } - -func execCredentialForImpersonationProxy( - idToken string, - conciergeAuthenticatorType string, - conciergeAuthenticatorName string, - tokenExpiry *metav1.Time, -) (*clientauthv1beta1.ExecCredential, error) { - // TODO maybe de-dup this with conciergeclient.go - // TODO reuse code from internal/testutil/impersonationtoken here to create token - var kind string - switch strings.ToLower(conciergeAuthenticatorType) { - case "webhook": - kind = "WebhookAuthenticator" - case "jwt": - kind = "JWTAuthenticator" - default: - return nil, fmt.Errorf(`invalid authenticator type: %q, supported values are "webhook" and "jwt"`, kind) - } - reqJSON, err := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginv1alpha1.GroupName + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: idToken, // TODO - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, - Kind: kind, - Name: conciergeAuthenticatorName, - }, - }, - }) - if err != nil { - return nil, fmt.Errorf("Error creating TokenCredentialRequest for impersonation proxy: %w", err) - } - encodedToken := base64.StdEncoding.EncodeToString(reqJSON) - cred := &clientauthv1beta1.ExecCredential{ - TypeMeta: metav1.TypeMeta{ - Kind: "ExecCredential", - APIVersion: "client.authentication.k8s.io/v1beta1", - }, - Status: &clientauthv1beta1.ExecCredentialStatus{ - Token: encodedToken, - }, - } - if !tokenExpiry.IsZero() { - cred.Status.ExpirationTimestamp = tokenExpiry - } - return cred, nil -} diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index f36d67cd..7bba6d9b 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -8,7 +8,6 @@ import ( "context" "crypto/x509/pkix" "encoding/base64" - "encoding/json" "fmt" "io/ioutil" "path/filepath" @@ -16,12 +15,9 @@ import ( "time" "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" clientauthv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" - authenticationv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/testutil" @@ -69,7 +65,6 @@ func TestLoginOIDCCommand(t *testing.T) { --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') --concierge-ca-bundle-data string CA bundle to use when connecting to the Concierge --concierge-endpoint string API base for the Concierge endpoint - --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) --enable-concierge Use the Concierge to login -h, --help help for oidc --issuer string OpenID Connect issuer URL @@ -199,21 +194,6 @@ func TestLoginOIDCCommand(t *testing.T) { wantOptionsCount: 7, wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"token":"exchanged-token"}}` + "\n", }, - { - name: "success with impersonation proxy", - args: []string{ - "--client-id", "test-client-id", - "--issuer", "test-issuer", - "--enable-concierge", - "--concierge-mode", "ImpersonationProxy", - "--concierge-authenticator-type", "webhook", - "--concierge-authenticator-name", "test-authenticator", - "--concierge-endpoint", "https://127.0.0.1:1234/", - "--concierge-ca-bundle-data", base64.StdEncoding.EncodeToString(testCA.Bundle()), - }, - wantOptionsCount: 3, - wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"expirationTimestamp":"3020-10-12T13:14:15Z","token":"` + impersonationProxyTestToken("test-id-token") + `"}}` + "\n", - }, } for _, tt := range tests { tt := tt @@ -270,21 +250,3 @@ func TestLoginOIDCCommand(t *testing.T) { }) } } - -func impersonationProxyTestToken(token string) string { - reqJSON, _ := json.Marshal(&loginv1alpha1.TokenCredentialRequest{ - TypeMeta: metav1.TypeMeta{ - Kind: "TokenCredentialRequest", - APIVersion: loginv1alpha1.GroupName + "/v1alpha1", - }, - Spec: loginv1alpha1.TokenCredentialRequestSpec{ - Token: token, - Authenticator: corev1.TypedLocalObjectReference{ - APIGroup: &authenticationv1alpha1.SchemeGroupVersion.Group, - Kind: "WebhookAuthenticator", - Name: "test-authenticator", - }, - }, - }) - return base64.StdEncoding.EncodeToString(reqJSON) -} diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index c9942551..4e8bef67 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -47,7 +47,6 @@ type staticLoginParams struct { conciergeEndpoint string conciergeCABundle string conciergeAPIGroupSuffix string - conciergeMode conciergeModeFlag } func staticLoginCommand(deps staticLoginDeps) *cobra.Command { @@ -70,7 +69,6 @@ func staticLoginCommand(deps staticLoginDeps) *cobra.Command { cmd.Flags().StringVar(&flags.conciergeEndpoint, "concierge-endpoint", "", "API base for the Concierge endpoint") cmd.Flags().StringVar(&flags.conciergeCABundle, "concierge-ca-bundle-data", "", "CA bundle to use when connecting to the Concierge") cmd.Flags().StringVar(&flags.conciergeAPIGroupSuffix, "concierge-api-group-suffix", groupsuffix.PinnipedDefaultSuffix, "Concierge API group suffix") - cmd.Flags().Var(&flags.conciergeMode, "concierge-mode", "Concierge mode of operation") cmd.RunE = func(cmd *cobra.Command, args []string) error { return runStaticLogin(cmd.OutOrStdout(), deps, flags) } @@ -115,34 +113,16 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams } cred := tokenCredential(&oidctypes.Token{IDToken: &oidctypes.IDToken{Token: token}}) - // If there is no concierge configuration, return the credential directly. - if concierge == nil { - return json.NewEncoder(out).Encode(cred) - } - - // If the concierge is enabled, we need to do extra steps. - switch flags.conciergeMode { - case modeUnknown, modeTokenCredentialRequestAPI: - // do a credential exchange request + // If the concierge was configured, exchange the credential for a separate short-lived, cluster-specific credential. + if concierge != nil { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() - cred, err := deps.exchangeToken(ctx, concierge, token) + var err error + cred, err = deps.exchangeToken(ctx, concierge, token) if err != nil { return fmt.Errorf("could not complete concierge credential exchange: %w", err) } - return json.NewEncoder(out).Encode(cred) - - case modeImpersonationProxy: - // Put the token into a TokenCredentialRequest - // put the TokenCredentialRequest in an ExecCredential - req, err := execCredentialForImpersonationProxy(token, flags.conciergeAuthenticatorType, flags.conciergeAuthenticatorName, nil) - if err != nil { - return err - } - return json.NewEncoder(out).Encode(req) - - default: - return fmt.Errorf("unsupported Concierge mode %q", flags.conciergeMode.String()) } + return json.NewEncoder(out).Encode(cred) } diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index 9ee5a969..fc81edc4 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -56,7 +56,6 @@ func TestLoginStaticCommand(t *testing.T) { --concierge-authenticator-type string Concierge authenticator type (e.g., 'webhook', 'jwt') --concierge-ca-bundle-data string CA bundle to use when connecting to the Concierge --concierge-endpoint string API base for the Concierge endpoint - --concierge-mode mode Concierge mode of operation (default TokenCredentialRequestAPI) --enable-concierge Use the Concierge to login -h, --help help for static --token string Static token to present during login @@ -152,18 +151,6 @@ func TestLoginStaticCommand(t *testing.T) { }, wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"token":"test-token"}}` + "\n", }, - { - name: "impersonation proxy success", - args: []string{ - "--enable-concierge", - "--concierge-mode", "ImpersonationProxy", - "--token", "test-token", - "--concierge-endpoint", "https://127.0.0.1/", - "--concierge-authenticator-type", "webhook", - "--concierge-authenticator-name", "test-authenticator", - }, - wantStdout: `{"kind":"ExecCredential","apiVersion":"client.authentication.k8s.io/v1beta1","spec":{},"status":{"token":"` + impersonationProxyTestToken("test-token") + `"}}` + "\n", - }, } for _, tt := range tests { tt := tt From a52455504fb6591c30fdb80cb8496a6aa7e5f3a6 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 11 Mar 2021 16:18:15 -0600 Subject: [PATCH 141/203] Capitalize "Concierge" in these error messages as well, for consistency. Signed-off-by: Matt Moyer --- cmd/pinniped/cmd/login_oidc.go | 4 ++-- cmd/pinniped/cmd/login_oidc_test.go | 6 +++--- cmd/pinniped/cmd/login_static.go | 4 ++-- cmd/pinniped/cmd/login_static_test.go | 6 +++--- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/pinniped/cmd/login_oidc.go b/cmd/pinniped/cmd/login_oidc.go index 3887ccf7..a3125475 100644 --- a/cmd/pinniped/cmd/login_oidc.go +++ b/cmd/pinniped/cmd/login_oidc.go @@ -144,7 +144,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin conciergeclient.WithAPIGroupSuffix(flags.conciergeAPIGroupSuffix), ) if err != nil { - return fmt.Errorf("invalid concierge parameters: %w", err) + return fmt.Errorf("invalid Concierge parameters: %w", err) } } @@ -178,7 +178,7 @@ func runOIDCLogin(cmd *cobra.Command, deps oidcLoginCommandDeps, flags oidcLogin cred, err = deps.exchangeToken(ctx, concierge, token.IDToken.Token) if err != nil { - return fmt.Errorf("could not complete concierge credential exchange: %w", err) + return fmt.Errorf("could not complete Concierge credential exchange: %w", err) } } return json.NewEncoder(cmd.OutOrStdout()).Encode(cred) diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index 7bba6d9b..976e3adf 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -92,7 +92,7 @@ func TestLoginOIDCCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: endpoint must not be empty + Error: invalid Concierge parameters: endpoint must not be empty `), }, { @@ -131,7 +131,7 @@ func TestLoginOIDCCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid Concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { @@ -161,7 +161,7 @@ func TestLoginOIDCCommand(t *testing.T) { wantOptionsCount: 3, wantError: true, wantStderr: here.Doc(` - Error: could not complete concierge credential exchange: some concierge error + Error: could not complete Concierge credential exchange: some concierge error `), }, { diff --git a/cmd/pinniped/cmd/login_static.go b/cmd/pinniped/cmd/login_static.go index 4e8bef67..4aa6d5eb 100644 --- a/cmd/pinniped/cmd/login_static.go +++ b/cmd/pinniped/cmd/login_static.go @@ -93,7 +93,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams conciergeclient.WithAPIGroupSuffix(flags.conciergeAPIGroupSuffix), ) if err != nil { - return fmt.Errorf("invalid concierge parameters: %w", err) + return fmt.Errorf("invalid Concierge parameters: %w", err) } } @@ -121,7 +121,7 @@ func runStaticLogin(out io.Writer, deps staticLoginDeps, flags staticLoginParams var err error cred, err = deps.exchangeToken(ctx, concierge, token) if err != nil { - return fmt.Errorf("could not complete concierge credential exchange: %w", err) + return fmt.Errorf("could not complete Concierge credential exchange: %w", err) } } return json.NewEncoder(out).Encode(cred) diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index fc81edc4..1d097123 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -78,7 +78,7 @@ func TestLoginStaticCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: endpoint must not be empty + Error: invalid Concierge parameters: endpoint must not be empty `), }, { @@ -126,7 +126,7 @@ func TestLoginStaticCommand(t *testing.T) { conciergeErr: fmt.Errorf("some concierge error"), wantError: true, wantStderr: here.Doc(` - Error: could not complete concierge credential exchange: some concierge error + Error: could not complete Concierge credential exchange: some concierge error `), }, { @@ -141,7 +141,7 @@ func TestLoginStaticCommand(t *testing.T) { }, wantError: true, wantStderr: here.Doc(` - Error: invalid concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') + Error: invalid Concierge parameters: invalid API group suffix: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character (e.g. 'example.com', regex used for validation is '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*') `), }, { From 71712b2d00098d3be4938bafb2cae2f563d3642d Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 11 Mar 2021 15:49:24 -0800 Subject: [PATCH 142/203] Add test for http2 Signed-off-by: Margo Crawford --- .../concierge_impersonation_proxy_test.go | 112 ++++++++++++++++-- 1 file changed, 104 insertions(+), 8 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index cfc33a6c..233d9795 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -25,6 +25,7 @@ import ( "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "golang.org/x/net/http2" v1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -662,7 +663,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Run("websocket client", func(t *testing.T) { namespaceName := createTestNamespace(t, adminClient) - library.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, @@ -676,11 +676,15 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) require.NoError(t, err) + wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" dest, _ := url.Parse(impersonationProxyURL) dest.Scheme = "wss" dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" - dest.RawQuery = "watch=1&resourceVersion=0" - + dest.RawQuery = url.Values{ + "watch": {"1"}, + "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, + "resourceVersion": {"0"}, + }.Encode() dialer := websocket.Dialer{ TLSClientConfig: tlsConfig, } @@ -705,8 +709,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.NoError(t, err) // perform a create through the admin client - _, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1"}}, + wantConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, + } + wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, + wantConfigMap, metav1.CreateOptions{}, ) require.NoError(t, err) @@ -726,10 +733,99 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl if got.Type != watch.Added { t.Errorf("Unexpected type: %v", got.Type) } - var createConfigMap corev1.ConfigMap - err = json.Unmarshal(got.Object, &createConfigMap) + var actualConfigMap corev1.ConfigMap + require.NoError(t, json.Unmarshal(got.Object, &actualConfigMap)) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + }) + + t.Run("http2 client", func(t *testing.T) { + namespaceName := createTestNamespace(t, adminClient) + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) + + wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" + wantConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, + } + wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, + wantConfigMap, + metav1.CreateOptions{}, + ) require.NoError(t, err) - require.Equal(t, "configmap-1", createConfigMap.Name) + t.Cleanup(func() { + _ = adminClient.CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + }) + + // create rest client + restConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") + + tlsConfig, err := rest.TLSConfigFor(restConfig) + require.NoError(t, err) + httpTransport := http.Transport{ + TLSClientConfig: tlsConfig, + } + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + httpTransport.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + } + } + err = http2.ConfigureTransport(&httpTransport) + require.NoError(t, err) + + httpClient := http.Client{ + Transport: &httpTransport, + } + + dest, _ := url.Parse(impersonationProxyURL) + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1" + response, err := httpClient.Get(dest.String()) + require.NoError(t, err) + body, _ := ioutil.ReadAll(response.Body) + t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body) + require.Equal(t, "HTTP/2.0", response.Proto) + require.Equal(t, http.StatusOK, response.StatusCode) + defer response.Body.Close() + var actualConfigMap corev1.ConfigMap + require.NoError(t, json.Unmarshal(body, &actualConfigMap)) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + + // watch configmaps + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" + dest.RawQuery = url.Values{ + "watch": {"1"}, + "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, + "resourceVersion": {"0"}, + }.Encode() + response, err = httpClient.Get(dest.String()) + require.NoError(t, err) + require.Equal(t, "HTTP/2.0", response.Proto) + require.Equal(t, http.StatusOK, response.StatusCode) + defer response.Body.Close() + + // decode + decoder := json.NewDecoder(response.Body) + var got watchJSON + err = decoder.Decode(&got) + require.NoError(t, err) + if got.Type != watch.Added { + t.Errorf("Unexpected type: %v", got.Type) + } + err = json.Unmarshal(got.Object, &actualConfigMap) + require.NoError(t, err) + require.Equal(t, "configmap-1", actualConfigMap.Name) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) }) t.Run("manually disabling the impersonation proxy feature", func(t *testing.T) { From c12a23725d4a271c778863925b670f2ca6644d59 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 16:21:40 -0800 Subject: [PATCH 143/203] Fix lint errors from a previous commit --- go.mod | 2 +- .../concierge_impersonation_proxy_test.go | 20 ++++++++++++------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index b8f1a732..004c3c08 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20201217014255-9d1352758620 - golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 // indirect + golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d golang.org/x/sync v0.0.0-20201207232520-09787c993a3a golang.org/x/tools v0.0.0-20200825202427-b303f430e36d // indirect diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 233d9795..095f8bb0 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -566,9 +566,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl conciergePod = &pod } } - if conciergePod == nil { - t.Error("could not find a concierge pod") - } + require.NotNil(t, conciergePod, "could not find a concierge pod") // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" @@ -788,13 +786,17 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl dest, _ := url.Parse(impersonationProxyURL) dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1" - response, err := httpClient.Get(dest.String()) + getConfigmapRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) + require.NoError(t, err) + response, err := httpClient.Do(getConfigmapRequest) require.NoError(t, err) body, _ := ioutil.ReadAll(response.Body) t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body) require.Equal(t, "HTTP/2.0", response.Proto) require.Equal(t, http.StatusOK, response.StatusCode) - defer response.Body.Close() + defer func() { + require.NoError(t, response.Body.Close()) + }() var actualConfigMap corev1.ConfigMap require.NoError(t, json.Unmarshal(body, &actualConfigMap)) actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. @@ -807,11 +809,15 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, "resourceVersion": {"0"}, }.Encode() - response, err = httpClient.Get(dest.String()) + watchConfigmapsRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) + require.NoError(t, err) + response, err = httpClient.Do(watchConfigmapsRequest) require.NoError(t, err) require.Equal(t, "HTTP/2.0", response.Proto) require.Equal(t, http.StatusOK, response.StatusCode) - defer response.Body.Close() + defer func() { + require.NoError(t, response.Body.Close()) + }() // decode decoder := json.NewDecoder(response.Body) From f77c92560fe563aa6f8ebe5545d22c489fc9f3bb Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 16:27:16 -0800 Subject: [PATCH 144/203] Rewrite impersonator_test.go, add missing argument to IssuePEM() The impersonator_test.go unit test now starts the impersonation server and makes real HTTP requests against it using client-go. It is backed by a fake Kube API server. The CA IssuePEM() method was missing the argument to allow a slice of IP addresses to be passed in. --- internal/certauthority/certauthority.go | 4 +- internal/certauthority/certauthority_test.go | 2 +- .../dynamiccertauthority.go | 2 +- .../impersonator/impersonator_test.go | 233 +++++++++++++----- .../controllermanager/prepare_controllers.go | 12 +- .../concierge_impersonation_proxy_test.go | 8 +- 6 files changed, 196 insertions(+), 65 deletions(-) diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index 806d6498..ad81e38a 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -220,8 +220,8 @@ func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time. // IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of // PEM-formatted byte slices for the certificate and private key. -func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { - return toPEM(c.Issue(subject, dnsNames, nil, ttl)) +func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) ([]byte, []byte, error) { + return toPEM(c.Issue(subject, dnsNames, ips, ttl)) } func toPEM(cert *tls.Certificate, err error) ([]byte, []byte, error) { diff --git a/internal/certauthority/certauthority_test.go b/internal/certauthority/certauthority_test.go index 4193990b..7f9e3571 100644 --- a/internal/certauthority/certauthority_test.go +++ b/internal/certauthority/certauthority_test.go @@ -325,7 +325,7 @@ func TestIssuePEM(t *testing.T) { realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) - certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, 10*time.Minute) + certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, nil, 10*time.Minute) require.NoError(t, err) require.NotEmpty(t, certPEM) require.NotEmpty(t, keyPEM) diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go index c15e4cc0..3c0c4b72 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go @@ -36,5 +36,5 @@ func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ( return nil, nil, err } - return ca.IssuePEM(subject, dnsNames, ttl) + return ca.IssuePEM(subject, dnsNames, nil, ttl) } diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 66cbd3d3..3a7911de 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -15,6 +15,8 @@ import ( "time" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" @@ -26,12 +28,13 @@ import ( "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/here" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" ) -func TestNew(t *testing.T) { - const port = 8444 +func TestImpersonator(t *testing.T) { + const port = 9444 ca, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) require.NoError(t, err) @@ -41,7 +44,7 @@ func TestNew(t *testing.T) { err = caContent.SetCertKeyContent(ca.Bundle(), caKey) require.NoError(t, err) - cert, key, err := ca.IssuePEM(pkix.Name{CommonName: "example.com"}, []string{"example.com"}, time.Hour) + cert, key, err := ca.IssuePEM(pkix.Name{}, nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) require.NoError(t, err) certKeyContent := dynamiccert.New("cert-key") err = certKeyContent.SetCertKeyContent(cert, key) @@ -57,77 +60,195 @@ func TestNew(t *testing.T) { defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIPriorityAndFairness, false)() tests := []struct { - name string - clientOpts []kubeclient.Option - wantErr string + name string + clientUsername string + clientGroups []string + clientImpersonateUser rest.ImpersonationConfig + kubeAPIServerClientBearerTokenFile string + kubeAPIServerStatusCode int + wantError string + wantConstructionError string }{ { - name: "happy path", - clientOpts: []kubeclient.Option{ - kubeclient.WithConfig(&rest.Config{ - BearerToken: "should-be-ignored", - BearerTokenFile: "required-to-be-set", - }), - }, + name: "happy path", + clientUsername: "test-username", + clientGroups: []string{"test-group1", "test-group2"}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", }, { - name: "no bearer token file", - clientOpts: []kubeclient.Option{ - kubeclient.WithConfig(&rest.Config{ - BearerToken: "should-be-ignored", - }), - }, - wantErr: "invalid impersonator loopback rest config has wrong bearer token semantics", + name: "no bearer token file in Kube API server client config", + wantConstructionError: "invalid impersonator loopback rest config has wrong bearer token semantics", + }, + { + name: "user is authenticated but the kube API request returns an error", + kubeAPIServerStatusCode: http.StatusNotFound, + clientUsername: "test-username", + clientGroups: []string{"test-group1", "test-group2"}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `the server could not find the requested resource (get namespaces)`, + }, + { + name: "double impersonation is not allowed by regular users", + clientUsername: "test-username", + clientGroups: []string{"test-group1", "test-group2"}, + clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `users "some-other-username" is forbidden: User "test-username" ` + + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + }, + { + name: "double impersonation is not allowed by admin users", + clientUsername: "test-admin", + clientGroups: []string{"system:masters", "test-group2"}, + clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `users "some-other-username" is forbidden: User "test-admin" ` + + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, }, } for _, tt := range tests { tt := tt + // This is a serial test because the production code binds to the port. t.Run(tt.name, func(t *testing.T) { - // This is a serial test because the production code binds to the port. - runner, constructionErr := newInternal(port, certKeyContent, caContent, tt.clientOpts, recOpts) - - if len(tt.wantErr) != 0 { - require.EqualError(t, constructionErr, tt.wantErr) - require.Nil(t, runner) - } else { - require.NoError(t, constructionErr) - require.NotNil(t, runner) - - stopCh := make(chan struct{}) - errCh := make(chan error) - go func() { - stopErr := runner(stopCh) - errCh <- stopErr - }() - - select { - case unexpectedExit := <-errCh: - t.Errorf("unexpected exit, err=%v (even nil error is failure)", unexpectedExit) - case <-time.After(10 * time.Second): - } - - close(stopCh) - exitErr := <-errCh - require.NoError(t, exitErr) + if tt.kubeAPIServerStatusCode == 0 { + tt.kubeAPIServerStatusCode = http.StatusOK } - // assert listener is closed is both cases above by trying to make another one on the same port - ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) - defer func() { - if ln == nil { + // Set up a fake Kube API server which will stand in for the real one. The impersonator + // will proxy incoming calls to this fake server. + testKubeAPIServerWasCalled := false + testKubeAPIServerSawHeaders := http.Header{} + testKubeAPIServerCA, testKubeAPIServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + require.Equal(t, http.MethodGet, r.Method) + switch r.URL.Path { + case "/api/v1/namespaces/kube-system/configmaps": + // The production code uses NewDynamicCAFromConfigMapController which fetches a ConfigMap, + // so treat that differently. It wants to read the Kube API server CA from that ConfigMap + // to use it to validate client certs. We don't need it for this test, so return NotFound. + http.NotFound(w, r) return + case "/api/v1/namespaces": + testKubeAPIServerWasCalled = true + testKubeAPIServerSawHeaders = r.Header + if tt.kubeAPIServerStatusCode != http.StatusOK { + w.WriteHeader(tt.kubeAPIServerStatusCode) + } else { + w.Header().Add("Content-Type", "application/json; charset=UTF-8") + _, _ = w.Write([]byte(here.Doc(` + { + "kind": "NamespaceList", + "apiVersion":"v1", + "items": [ + {"metadata":{"name": "namespace1"}}, + {"metadata":{"name": "namespace2"}} + ] + } + `))) + } + default: + require.Fail(t, "fake Kube API server got an unexpected request") } - require.NoError(t, ln.Close()) - }() - require.NoError(t, listenErr) + }) + testKubeAPIServerKubeconfig := rest.Config{ + Host: testKubeAPIServerURL, + BearerToken: "some-service-account-token", + TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testKubeAPIServerCA)}, + BearerTokenFile: tt.kubeAPIServerClientBearerTokenFile, + } + clientOpts := []kubeclient.Option{kubeclient.WithConfig(&testKubeAPIServerKubeconfig)} - // TODO: create some client certs and assert the authorizer works correctly with system:masters - // and nested impersonation - we could also try to test what headers are sent to KAS + // Create an impersonator. + runner, constructionErr := newInternal(port, certKeyContent, caContent, clientOpts, recOpts) + if len(tt.wantConstructionError) > 0 { + require.EqualError(t, constructionErr, tt.wantConstructionError) + require.Nil(t, runner) + // After failing to start, the impersonator port should be available again. + requireCanBindToPort(t, port) + // The rest of the test doesn't make sense when you expect a construction error, so stop here. + return + } + require.NoError(t, constructionErr) + require.NotNil(t, runner) + + // Start the impersonator. + stopCh := make(chan struct{}) + errCh := make(chan error) + go func() { + stopErr := runner(stopCh) + errCh <- stopErr + }() + + // Create client certs for authentication to the impersonator. + clientCertPEM, clientKeyPEM, err := ca.IssuePEM( + pkix.Name{CommonName: tt.clientUsername, Organization: tt.clientGroups}, nil, nil, time.Hour, + ) + require.NoError(t, err) + + // Create a kubeconfig to talk to the impersonator as a client. + clientKubeconfig := &rest.Config{ + Host: "https://127.0.0.1:" + strconv.Itoa(port), + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca.Bundle(), + CertData: clientCertPEM, + KeyData: clientKeyPEM, + }, + UserAgent: "test-agent", + Impersonate: tt.clientImpersonateUser, + } + + // Create a real Kube client to make API requests to the impersonator. + client, err := kubeclient.New(kubeclient.WithConfig(clientKubeconfig)) + require.NoError(t, err) + + // The fake Kube API server knows how to to list namespaces, so make that request using the client + // through the impersonator. + listResponse, err := client.Kubernetes.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{}) + if len(tt.wantError) > 0 { + require.EqualError(t, err, tt.wantError) + } else { + require.NoError(t, err) + require.Equal(t, &v1.NamespaceList{ + Items: []v1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: "namespace1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "namespace2"}}, + }, + }, listResponse) + + // The impersonator should have proxied the request to the fake Kube API server, which should have seen + // the headers of the original request mutated by the impersonator. + require.True(t, testKubeAPIServerWasCalled) + require.Equal(t, + http.Header{ + "Impersonate-User": {tt.clientUsername}, + "Impersonate-Group": append(tt.clientGroups, "system:authenticated"), + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + testKubeAPIServerSawHeaders, + ) + } + + // Stop the impersonator server. + close(stopCh) + exitErr := <-errCh + require.NoError(t, exitErr) + + // After shutdown, the impersonator port should be available again. + requireCanBindToPort(t, port) }) } } -func TestImpersonator(t *testing.T) { +func requireCanBindToPort(t *testing.T, port int) { + ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) + require.NoError(t, listenErr) + require.NoError(t, ln.Close()) +} + +func TestImpersonatorHTTPHandler(t *testing.T) { const testUser = "test-user" testGroups := []string{"test-group-1", "test-group-2"} diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 5af528e4..aef64f28 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -64,14 +64,22 @@ type Config struct { // DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert. DynamicServingCertProvider dynamiccert.Provider - // DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's // TODO fix comment + + // DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's // signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login. + // This is filled with the Kube API server's signing cert by a controller, if it can be found. DynamicSigningCertProvider dynamiccert.Provider - // TODO fix comment + + // ImpersonationSigningCertProvider provides a setter and a getter to the CA cert that should be + // used to sign client certs for authentication to the impersonation proxy. This CA is used by + // the TokenCredentialRequest to sign certs and by the impersonation proxy to check certs. + // When the impersonation proxy is not running, the getter will return nil cert and nil key. + // (Note that the impersonation proxy also accepts client certs signed by the Kube API server's cert.) ImpersonationSigningCertProvider dynamiccert.Provider // ServingCertDuration is the validity period, in seconds, of the API serving certificate. ServingCertDuration time.Duration + // ServingCertRenewBefore is the period of time, in seconds, that pinniped will wait before // rotating the serving certificate. This period of time starts upon issuance of the serving // certificate. diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 095f8bb0..6c3efeb0 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -427,11 +427,13 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) // Double impersonation is not supported yet, so we should get an error. - require.EqualError(t, err, fmt.Sprintf( + require.Error(t, err) + require.Regexp(t, `users "other-user-to-impersonate" is forbidden: `+ - `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `User ".*" cannot impersonate resource "users" in API group "" at the cluster scope: `+ `impersonation is not allowed or invalid verb`, - "kubernetes-admin")) + err.Error(), + ) }) t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { From 1d68841c785bb8195e28d4f01c37164f50dd2280 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 16:44:08 -0800 Subject: [PATCH 145/203] impersonator_test.go: Test one more thing and small refactors --- .../concierge/impersonator/impersonator.go | 1 - .../impersonator/impersonator_test.go | 41 +++++++++++-------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index a14e87bc..ac97016f 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -210,7 +210,6 @@ func newImpersonationReverseProxy(restConfig *rest.Config) (http.Handler, error) } return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // TODO integration test using a bearer token if len(r.Header.Values("Authorization")) != 0 { plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so", "url", r.URL.String(), diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 3a7911de..a37308dc 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -149,6 +149,8 @@ func TestImpersonator(t *testing.T) { require.Fail(t, "fake Kube API server got an unexpected request") } }) + + // Create the client config that the impersonation server should use to talk to the Kube API server. testKubeAPIServerKubeconfig := rest.Config{ Host: testKubeAPIServerURL, BearerToken: "some-service-account-token", @@ -192,7 +194,10 @@ func TestImpersonator(t *testing.T) { CertData: clientCertPEM, KeyData: clientKeyPEM, }, - UserAgent: "test-agent", + UserAgent: "test-agent", + // BearerToken should be ignored during auth because there are valid client certs, + // and it should not passed into the impersonator handler func as an authorization header. + BearerToken: "must-be-ignored", Impersonate: tt.clientImpersonateUser, } @@ -405,40 +410,44 @@ func TestImpersonatorHTTPHandler(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if tt.kubeAPIServerStatusCode == 0 { tt.kubeAPIServerStatusCode = http.StatusOK } - serverWasCalled := false - serverSawHeaders := http.Header{} - testServerCA, testServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { - serverWasCalled = true - serverSawHeaders = r.Header + testKubeAPIServerWasCalled := false + testKubeAPIServerSawHeaders := http.Header{} + testKubeAPIServerCA, testKubeAPIServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { + testKubeAPIServerWasCalled = true + testKubeAPIServerSawHeaders = r.Header if tt.kubeAPIServerStatusCode != http.StatusOK { w.WriteHeader(tt.kubeAPIServerStatusCode) } else { _, _ = w.Write([]byte("successful proxied response")) } }) - testServerKubeconfig := rest.Config{ - Host: testServerURL, + testKubeAPIServerKubeconfig := rest.Config{ + Host: testKubeAPIServerURL, BearerToken: "some-service-account-token", - TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testServerCA)}, + TLSClientConfig: rest.TLSClientConfig{CAData: []byte(testKubeAPIServerCA)}, } if tt.restConfig == nil { - tt.restConfig = &testServerKubeconfig + tt.restConfig = &testKubeAPIServerKubeconfig } - proxy, err := newImpersonationReverseProxy(tt.restConfig) + impersonatorHTTPHandler, err := newImpersonationReverseProxy(tt.restConfig) if tt.wantCreationErr != "" { require.EqualError(t, err, tt.wantCreationErr) return } require.NoError(t, err) - require.NotNil(t, proxy) + require.NotNil(t, impersonatorHTTPHandler) + w := httptest.NewRecorder() requestBeforeServe := tt.request.Clone(tt.request.Context()) - proxy.ServeHTTP(w, tt.request) + impersonatorHTTPHandler.ServeHTTP(w, tt.request) + require.Equal(t, requestBeforeServe, tt.request, "ServeHTTP() mutated the request, and it should not per http.Handler docs") if tt.wantHTTPStatus != 0 { require.Equalf(t, tt.wantHTTPStatus, w.Code, "fyi, response body was %q", w.Body.String()) @@ -448,10 +457,10 @@ func TestImpersonatorHTTPHandler(t *testing.T) { } if tt.wantHTTPStatus == http.StatusOK || tt.kubeAPIServerStatusCode != http.StatusOK { - require.True(t, serverWasCalled, "Should have proxied the request to the Kube API server, but didn't") - require.Equal(t, tt.wantKubeAPIServerRequestHeaders, serverSawHeaders) + require.True(t, testKubeAPIServerWasCalled, "Should have proxied the request to the Kube API server, but didn't") + require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) } else { - require.False(t, serverWasCalled, "Should not have proxied the request to the Kube API server, but did") + require.False(t, testKubeAPIServerWasCalled, "Should not have proxied the request to the Kube API server, but did") } }) } From 6ddf4c04e62468bbf57cc4434f40ef18f95b338f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 17:11:38 -0800 Subject: [PATCH 146/203] impersonator_test.go: Test failed and anonymous auth --- .../impersonator/impersonator_test.go | 87 ++++++++++++++----- 1 file changed, 66 insertions(+), 21 deletions(-) diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index a37308dc..85b6af97 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -50,6 +50,13 @@ func TestImpersonator(t *testing.T) { err = certKeyContent.SetCertKeyContent(cert, key) require.NoError(t, err) + unrelatedCA, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) + require.NoError(t, err) + badClientCertPEM, badClientKeyPEM, err := unrelatedCA.IssuePEM( + pkix.Name{CommonName: "test-user", Organization: []string{"test-group1"}}, nil, nil, time.Hour, + ) + require.NoError(t, err) + // Punch out just enough stuff to make New actually run without error. recOpts := func(options *genericoptions.RecommendedOptions) { options.Authentication.RemoteKubeConfigFileOptional = true @@ -63,9 +70,12 @@ func TestImpersonator(t *testing.T) { name string clientUsername string clientGroups []string + clientCertPEM string + clientKeyPEM string clientImpersonateUser rest.ImpersonationConfig kubeAPIServerClientBearerTokenFile string kubeAPIServerStatusCode int + wantKubeAPIServerRequestHeaders http.Header wantError string wantConstructionError string }{ @@ -74,10 +84,15 @@ func TestImpersonator(t *testing.T) { clientUsername: "test-username", clientGroups: []string{"test-group1", "test-group2"}, kubeAPIServerClientBearerTokenFile: "required-to-be-set", - }, - { - name: "no bearer token file in Kube API server client config", - wantConstructionError: "invalid impersonator loopback rest config has wrong bearer token semantics", + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username"}, + "Impersonate-Group": {"test-group1", "test-group2", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, }, { name: "user is authenticated but the kube API request returns an error", @@ -86,6 +101,37 @@ func TestImpersonator(t *testing.T) { clientGroups: []string{"test-group1", "test-group2"}, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: `the server could not find the requested resource (get namespaces)`, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username"}, + "Impersonate-Group": {"test-group1", "test-group2", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "when there is no client cert on request, it is an anonymous request", + clientCertPEM: "present so we will use the empty keyPEM", + clientKeyPEM: "", + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"system:anonymous"}, + "Impersonate-Group": {"system:unauthenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "failed client cert authentication", + clientCertPEM: string(badClientCertPEM), + clientKeyPEM: string(badClientKeyPEM), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Unauthorized", }, { name: "double impersonation is not allowed by regular users", @@ -105,6 +151,10 @@ func TestImpersonator(t *testing.T) { wantError: `users "some-other-username" is forbidden: User "test-admin" ` + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, }, + { + name: "no bearer token file in Kube API server client config", + wantConstructionError: "invalid impersonator loopback rest config has wrong bearer token semantics", + }, } for _, tt := range tests { tt := tt @@ -181,10 +231,16 @@ func TestImpersonator(t *testing.T) { }() // Create client certs for authentication to the impersonator. - clientCertPEM, clientKeyPEM, err := ca.IssuePEM( - pkix.Name{CommonName: tt.clientUsername, Organization: tt.clientGroups}, nil, nil, time.Hour, - ) - require.NoError(t, err) + var clientCertPEM, clientKeyPEM []byte + if len(tt.clientCertPEM) == 0 { + clientCertPEM, clientKeyPEM, err = ca.IssuePEM( + pkix.Name{CommonName: tt.clientUsername, Organization: tt.clientGroups}, nil, nil, time.Hour, + ) + require.NoError(t, err) + } else { + clientCertPEM = []byte(tt.clientCertPEM) + clientKeyPEM = []byte(tt.clientKeyPEM) + } // Create a kubeconfig to talk to the impersonator as a client. clientKubeconfig := &rest.Config{ @@ -195,7 +251,7 @@ func TestImpersonator(t *testing.T) { KeyData: clientKeyPEM, }, UserAgent: "test-agent", - // BearerToken should be ignored during auth because there are valid client certs, + // BearerToken should be ignored during auth when there are valid client certs, // and it should not passed into the impersonator handler func as an authorization header. BearerToken: "must-be-ignored", Impersonate: tt.clientImpersonateUser, @@ -222,18 +278,7 @@ func TestImpersonator(t *testing.T) { // The impersonator should have proxied the request to the fake Kube API server, which should have seen // the headers of the original request mutated by the impersonator. require.True(t, testKubeAPIServerWasCalled) - require.Equal(t, - http.Header{ - "Impersonate-User": {tt.clientUsername}, - "Impersonate-Group": append(tt.clientGroups, "system:authenticated"), - "Authorization": {"Bearer some-service-account-token"}, - "User-Agent": {"test-agent"}, - "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, - "Accept-Encoding": {"gzip"}, - "X-Forwarded-For": {"127.0.0.1"}, - }, - testKubeAPIServerSawHeaders, - ) + require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) } // Stop the impersonator server. From 87f2899047ce5c72b7ddced9677a4d394ea8296f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 11 Mar 2021 17:24:52 -0800 Subject: [PATCH 147/203] impersonator_test.go: small refactor of previous commit --- .../impersonator/impersonator_test.go | 70 ++++++++----------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 85b6af97..5095963d 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -52,10 +52,6 @@ func TestImpersonator(t *testing.T) { unrelatedCA, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) require.NoError(t, err) - badClientCertPEM, badClientKeyPEM, err := unrelatedCA.IssuePEM( - pkix.Name{CommonName: "test-user", Organization: []string{"test-group1"}}, nil, nil, time.Hour, - ) - require.NoError(t, err) // Punch out just enough stuff to make New actually run without error. recOpts := func(options *genericoptions.RecommendedOptions) { @@ -68,10 +64,7 @@ func TestImpersonator(t *testing.T) { tests := []struct { name string - clientUsername string - clientGroups []string - clientCertPEM string - clientKeyPEM string + clientCert *clientCert clientImpersonateUser rest.ImpersonationConfig kubeAPIServerClientBearerTokenFile string kubeAPIServerStatusCode int @@ -81,8 +74,7 @@ func TestImpersonator(t *testing.T) { }{ { name: "happy path", - clientUsername: "test-username", - clientGroups: []string{"test-group1", "test-group2"}, + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantKubeAPIServerRequestHeaders: http.Header{ "Impersonate-User": {"test-username"}, @@ -97,8 +89,7 @@ func TestImpersonator(t *testing.T) { { name: "user is authenticated but the kube API request returns an error", kubeAPIServerStatusCode: http.StatusNotFound, - clientUsername: "test-username", - clientGroups: []string{"test-group1", "test-group2"}, + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: `the server could not find the requested resource (get namespaces)`, wantKubeAPIServerRequestHeaders: http.Header{ @@ -113,8 +104,7 @@ func TestImpersonator(t *testing.T) { }, { name: "when there is no client cert on request, it is an anonymous request", - clientCertPEM: "present so we will use the empty keyPEM", - clientKeyPEM: "", + clientCert: &clientCert{}, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantKubeAPIServerRequestHeaders: http.Header{ "Impersonate-User": {"system:anonymous"}, @@ -128,15 +118,13 @@ func TestImpersonator(t *testing.T) { }, { name: "failed client cert authentication", - clientCertPEM: string(badClientCertPEM), - clientKeyPEM: string(badClientKeyPEM), + clientCert: newClientCert(t, unrelatedCA, "test-username", []string{"test-group1"}), kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: "Unauthorized", }, { name: "double impersonation is not allowed by regular users", - clientUsername: "test-username", - clientGroups: []string{"test-group1", "test-group2"}, + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: `users "some-other-username" is forbidden: User "test-username" ` + @@ -144,8 +132,7 @@ func TestImpersonator(t *testing.T) { }, { name: "double impersonation is not allowed by admin users", - clientUsername: "test-admin", - clientGroups: []string{"system:masters", "test-group2"}, + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: `users "some-other-username" is forbidden: User "test-admin" ` + @@ -230,25 +217,13 @@ func TestImpersonator(t *testing.T) { errCh <- stopErr }() - // Create client certs for authentication to the impersonator. - var clientCertPEM, clientKeyPEM []byte - if len(tt.clientCertPEM) == 0 { - clientCertPEM, clientKeyPEM, err = ca.IssuePEM( - pkix.Name{CommonName: tt.clientUsername, Organization: tt.clientGroups}, nil, nil, time.Hour, - ) - require.NoError(t, err) - } else { - clientCertPEM = []byte(tt.clientCertPEM) - clientKeyPEM = []byte(tt.clientKeyPEM) - } - // Create a kubeconfig to talk to the impersonator as a client. clientKubeconfig := &rest.Config{ Host: "https://127.0.0.1:" + strconv.Itoa(port), TLSClientConfig: rest.TLSClientConfig{ CAData: ca.Bundle(), - CertData: clientCertPEM, - KeyData: clientKeyPEM, + CertData: tt.clientCert.certPEM, + KeyData: tt.clientCert.keyPEM, }, UserAgent: "test-agent", // BearerToken should be ignored during auth when there are valid client certs, @@ -292,12 +267,6 @@ func TestImpersonator(t *testing.T) { } } -func requireCanBindToPort(t *testing.T, port int) { - ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) - require.NoError(t, listenErr) - require.NoError(t, ln.Close()) -} - func TestImpersonatorHTTPHandler(t *testing.T) { const testUser = "test-user" @@ -510,3 +479,24 @@ func TestImpersonatorHTTPHandler(t *testing.T) { }) } } + +type clientCert struct { + certPEM, keyPEM []byte +} + +func newClientCert(t *testing.T, ca *certauthority.CA, username string, groups []string) *clientCert { + certPEM, keyPEM, err := ca.IssuePEM( + pkix.Name{CommonName: username, Organization: groups}, nil, nil, time.Hour, + ) + require.NoError(t, err) + return &clientCert{ + certPEM: certPEM, + keyPEM: keyPEM, + } +} + +func requireCanBindToPort(t *testing.T, port int) { + ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) + require.NoError(t, listenErr) + require.NoError(t, ln.Close()) +} From 253e0f8e9a578ae17159ca34c8b10b8f88f2ba6a Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 12 Mar 2021 09:54:59 -0500 Subject: [PATCH 148/203] test/integration: TestImpersonationProxy/websocket_client passes on my machine now I'm kinda surprised this is working with our current implementation of the impersonator, but regardless this seems like a step forward. Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 6c3efeb0..327e8a9a 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -696,7 +696,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl return proxyURL, nil } } - c, r, err := dialer.Dial(dest.String(), nil) + c, r, err := dialer.Dial(dest.String(), http.Header{"Origin": {dest.String()}}) if r != nil { defer func() { require.NoError(t, r.Body.Close()) From 5b1dc0abdfd9e99261dd70d04f36b1f2a49b1f45 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 12 Mar 2021 10:45:36 -0500 Subject: [PATCH 149/203] test/integration: add some more debugging to kubectl impersonation test I think this is nondeterministic... Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 327e8a9a..76a8cd7d 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -597,14 +597,14 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - portForwardCmd, _, stderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd, portForwardStdout, portForwardStderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") portForwardCmd.Env = envVarsWithProxy // start, but don't wait for the command to finish err = portForwardCmd.Start() require.NoError(t, err, `"kubectl port-forward" failed`) go func() { - assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, stderr.String()) + assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() // then run curl something against it @@ -622,7 +622,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Log("stdout: " + curlStdOut.String()) } // we expect this to 403, but all we care is that it gets through - require.Contains(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + require.Containsf(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"", "unexpected curl response\nport-forward stdout:\n%q\nport-forward stderr:\n%q", portForwardStdout.String(), portForwardStderr.String()) // run the kubectl attach command namespaceName := createTestNamespace(t, adminClient) From 12b13b1ea5e0fa7d1b25eb539a162632faf2e2d7 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Fri, 12 Mar 2021 09:56:34 -0500 Subject: [PATCH 150/203] impersonator: wire in genericapiserver.Config Signed-off-by: Monis Khan --- .../concierge/impersonator/impersonator.go | 115 +++++++++--------- .../impersonator/impersonator_test.go | 7 +- 2 files changed, 63 insertions(+), 59 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index ac97016f..52e8029c 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -122,7 +122,7 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. // Assume proto config is safe because transport level configs do not use rest.ContentConfig. // Thus if we are interacting with actual APIs, they should be using pre-built clients. - impersonationProxy, err := newImpersonationReverseProxy(rest.CopyConfig(kubeClient.ProtoConfig)) + impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClient.ProtoConfig)) if err != nil { return nil, err } @@ -130,7 +130,8 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler { // We ignore the passed in handler because we never have any REST APIs to delegate to. - handler := defaultBuildHandlerChainFunc(impersonationProxy, c) + handler := impersonationProxyFunc(c) + handler = defaultBuildHandlerChainFunc(handler, c) handler = securityheader.Wrap(handler) return handler } @@ -192,7 +193,7 @@ type comparableAuthorizer struct { authorizer.AuthorizerFunc } -func newImpersonationReverseProxy(restConfig *rest.Config) (http.Handler, error) { +func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapiserver.Config) http.Handler, error) { serverURL, err := url.Parse(restConfig.Host) if err != nil { return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err) @@ -209,63 +210,65 @@ func newImpersonationReverseProxy(restConfig *rest.Config) (http.Handler, error) return nil, fmt.Errorf("could not get in-cluster transport: %w", err) } - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if len(r.Header.Values("Authorization")) != 0 { - plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so", + return func(c *genericapiserver.Config) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if len(r.Header.Values("Authorization")) != 0 { + plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "invalid authorization header", http.StatusInternalServerError) + return + } + + if err := ensureNoImpersonationHeaders(r); err != nil { + plog.Error("noImpersonationAuthorizer logic did not prevent nested impersonation but it is always supposed to do so", + err, + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "invalid impersonation", http.StatusInternalServerError) + return + } + + userInfo, ok := request.UserFrom(r.Context()) + if !ok { + plog.Warning("aggregated API server logic did not set user info but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "invalid user", http.StatusInternalServerError) + return + } + + if len(userInfo.GetUID()) > 0 { + plog.Warning("rejecting request with UID since we cannot impersonate UIDs", + "url", r.URL.String(), + "method", r.Method, + ) + http.Error(w, "unexpected uid", http.StatusUnprocessableEntity) + return + } + + plog.Trace("proxying authenticated request", "url", r.URL.String(), "method", r.Method, + "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) - http.Error(w, "invalid authorization header", http.StatusInternalServerError) - return - } - if err := ensureNoImpersonationHeaders(r); err != nil { - plog.Error("noImpersonationAuthorizer logic did not prevent nested impersonation but it is always supposed to do so", - err, - "url", r.URL.String(), - "method", r.Method, - ) - http.Error(w, "invalid impersonation", http.StatusInternalServerError) - return - } - - userInfo, ok := request.UserFrom(r.Context()) - if !ok { - plog.Warning("aggregated API server logic did not set user info but it is always supposed to do so", - "url", r.URL.String(), - "method", r.Method, - ) - http.Error(w, "invalid user", http.StatusInternalServerError) - return - } - - if len(userInfo.GetUID()) > 0 { - plog.Warning("rejecting request with UID since we cannot impersonate UIDs", - "url", r.URL.String(), - "method", r.Method, - ) - http.Error(w, "unexpected uid", http.StatusUnprocessableEntity) - return - } - - plog.Trace("proxying authenticated request", - "url", r.URL.String(), - "method", r.Method, - "username", userInfo.GetName(), // this info leak seems fine for trace level logs - ) - - reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) - impersonateConfig := transport.ImpersonationConfig{ - UserName: userInfo.GetName(), - Groups: userInfo.GetGroups(), - Extra: userInfo.GetExtra(), - } - reverseProxy.Transport = transport.NewImpersonatingRoundTripper(impersonateConfig, kubeRoundTripper) - reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line - // transport.NewImpersonatingRoundTripper clones the request before setting headers - // so this call will not accidentally mutate the input request (see http.Handler docs) - reverseProxy.ServeHTTP(w, r) - }), nil + reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) + impersonateConfig := transport.ImpersonationConfig{ + UserName: userInfo.GetName(), + Groups: userInfo.GetGroups(), + Extra: userInfo.GetExtra(), + } + reverseProxy.Transport = transport.NewImpersonatingRoundTripper(impersonateConfig, kubeRoundTripper) + reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line + // transport.NewImpersonatingRoundTripper clones the request before setting headers + // so this call will not accidentally mutate the input request (see http.Handler docs) + reverseProxy.ServeHTTP(w, r) + }) + }, nil } func ensureNoImpersonationHeaders(r *http.Request) error { diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 5095963d..cf3f7846 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -450,17 +450,18 @@ func TestImpersonatorHTTPHandler(t *testing.T) { tt.restConfig = &testKubeAPIServerKubeconfig } - impersonatorHTTPHandler, err := newImpersonationReverseProxy(tt.restConfig) + impersonatorHTTPHandlerFunc, err := newImpersonationReverseProxyFunc(tt.restConfig) if tt.wantCreationErr != "" { require.EqualError(t, err, tt.wantCreationErr) + require.Nil(t, impersonatorHTTPHandlerFunc) return } require.NoError(t, err) - require.NotNil(t, impersonatorHTTPHandler) + require.NotNil(t, impersonatorHTTPHandlerFunc) w := httptest.NewRecorder() requestBeforeServe := tt.request.Clone(tt.request.Context()) - impersonatorHTTPHandler.ServeHTTP(w, tt.request) + impersonatorHTTPHandlerFunc(nil).ServeHTTP(w, tt.request) require.Equal(t, requestBeforeServe, tt.request, "ServeHTTP() mutated the request, and it should not per http.Handler docs") if tt.wantHTTPStatus != 0 { From 8c0bafd5beb9de4a27a189cd80f1191806313cd3 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Fri, 12 Mar 2021 10:33:30 -0500 Subject: [PATCH 151/203] impersonator: prep work for future SA token support Signed-off-by: Monis Khan --- .../concierge/impersonator/impersonator.go | 44 ++++++++++++++----- .../impersonator/impersonator_test.go | 2 +- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 52e8029c..169d3aaf 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/request" genericapiserver "k8s.io/apiserver/pkg/server" @@ -79,6 +80,8 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. // Wire up the impersonation proxy signer CA as another valid authenticator for client cert auth, // along with the Kube API server's CA. + // Note: any changes to the the Authentication stack need to be kept in sync with any assumptions made + // by getTransportForUser, especially if we ever update the TCR API to start returning bearer tokens. kubeClient, err := kubeclient.New(clientOpts...) if err != nil { return nil, err @@ -241,12 +244,13 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi return } - if len(userInfo.GetUID()) > 0 { - plog.Warning("rejecting request with UID since we cannot impersonate UIDs", + rt, err := getTransportForUser(userInfo, kubeRoundTripper) + if err != nil { + plog.WarningErr("rejecting request as we cannot act as the current user", err, "url", r.URL.String(), "method", r.Method, ) - http.Error(w, "unexpected uid", http.StatusUnprocessableEntity) + http.Error(w, "unable to act as user", http.StatusUnprocessableEntity) return } @@ -257,15 +261,8 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi ) reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) - impersonateConfig := transport.ImpersonationConfig{ - UserName: userInfo.GetName(), - Groups: userInfo.GetGroups(), - Extra: userInfo.GetExtra(), - } - reverseProxy.Transport = transport.NewImpersonatingRoundTripper(impersonateConfig, kubeRoundTripper) + reverseProxy.Transport = rt reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line - // transport.NewImpersonatingRoundTripper clones the request before setting headers - // so this call will not accidentally mutate the input request (see http.Handler docs) reverseProxy.ServeHTTP(w, r) }) }, nil @@ -280,3 +277,28 @@ func ensureNoImpersonationHeaders(r *http.Request) error { return nil } + +func getTransportForUser(userInfo user.Info, delegate http.RoundTripper) (http.RoundTripper, error) { + if len(userInfo.GetUID()) == 0 { + impersonateConfig := transport.ImpersonationConfig{ + UserName: userInfo.GetName(), + Groups: userInfo.GetGroups(), + Extra: userInfo.GetExtra(), + } + // transport.NewImpersonatingRoundTripper clones the request before setting headers + // thus it will not accidentally mutate the input request (see http.Handler docs) + return transport.NewImpersonatingRoundTripper(impersonateConfig, delegate), nil + } + + // 0. in the case of a request that is not attempting to do nested impersonation + // 1. if we make the assumption that the TCR API does not issue tokens (or pass the TCR API bearer token + // authenticator into this func - we need to know the authentication cred is something KAS would honor) + // 2. then if preserve the incoming authorization header into the request's context + // 3. we could reauthenticate it here (it would be a free cache hit) + // 4. confirm that it matches the passed in user info (i.e. it was actually the cred used to authenticate and not a client cert) + // 5. then we could issue a reverse proxy request using an anonymous rest config and the bearer token + // 6. thus instead of impersonating the user, we would just be passing their request through + // 7. this would preserve the UID info and thus allow us to safely support all token based auth + // 8. the above would be safe even if in the future Kube started supporting UIDs asserted by client certs + return nil, constable.Error("unexpected uid") +} diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index cf3f7846..9d1e231c 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -361,7 +361,7 @@ func TestImpersonatorHTTPHandler(t *testing.T) { { name: "unexpected UID", request: newRequest(map[string][]string{}, &user.DefaultInfo{UID: "007"}), - wantHTTPBody: "unexpected uid\n", + wantHTTPBody: "unable to act as user\n", wantHTTPStatus: http.StatusUnprocessableEntity, }, // happy path From d509e7012ed6f1f8de9caa68dab9de2bbb3a73ad Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 12 Mar 2021 10:44:11 -0800 Subject: [PATCH 152/203] Add eventually loop to port-forward test Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 76a8cd7d..6132ffaf 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -597,7 +597,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - portForwardCmd, portForwardStdout, portForwardStderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") portForwardCmd.Env = envVarsWithProxy // start, but don't wait for the command to finish @@ -607,22 +607,23 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() - // then run curl something against it - time.Sleep(time.Second) - timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") - var curlStdOut, curlStdErr bytes.Buffer - curlCmd.Stdout = &curlStdOut - curlCmd.Stderr = &curlStdErr - err = curlCmd.Run() - if err != nil { - t.Log("curl error: " + err.Error()) - t.Log("curlStdErr: " + curlStdErr.String()) - t.Log("stdout: " + curlStdOut.String()) - } - // we expect this to 403, but all we care is that it gets through - require.Containsf(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"", "unexpected curl response\nport-forward stdout:\n%q\nport-forward stderr:\n%q", portForwardStdout.String(), portForwardStderr.String()) + require.Eventually(t, func() bool { + // then run curl something against it + timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") + var curlStdOut, curlStdErr bytes.Buffer + curlCmd.Stdout = &curlStdOut + curlCmd.Stderr = &curlStdErr + err = curlCmd.Run() + if err != nil { + t.Log("curl error: " + err.Error()) + t.Log("curlStdErr: " + curlStdErr.String()) + t.Log("stdout: " + curlStdOut.String()) + } + // we expect this to 403, but all we care is that it gets through + return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + }, 5*time.Minute, 500*time.Millisecond) // run the kubectl attach command namespaceName := createTestNamespace(t, adminClient) @@ -637,6 +638,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, }, }) + timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) attachCmd.Env = envVarsWithProxy attachStdin, err := attachCmd.StdinPipe() From 077aa8a42e278d7e8e94e472736e2b90529a307f Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 12 Mar 2021 13:23:24 -0600 Subject: [PATCH 153/203] Fix a copy-paste typo in the ImpersonationProxyInfo JSON field name. Signed-off-by: Matt Moyer --- apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl | 2 +- .../config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.17/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.18/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.19/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- generated/1.20/README.adoc | 2 +- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- .../crds/config.concierge.pinniped.dev_credentialissuers.yaml | 4 ++-- .../apis/concierge/config/v1alpha1/types_credentialissuer.go | 2 +- go.sum | 1 - 16 files changed, 20 insertions(+), 21 deletions(-) diff --git a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl index e38b207b..3c0c8ba0 100644 --- a/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl +++ b/apis/concierge/config/v1alpha1/types_credentialissuer.go.tmpl @@ -111,7 +111,7 @@ type ImpersonationProxyInfo struct { // Endpoint is the HTTPS endpoint of the impersonation proxy. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://` - Endpoint string `json:"server"` + Endpoint string `json:"endpoint"` // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 diff --git a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml index 7436f6e2..8123f238 100644 --- a/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/deploy/concierge/config.concierge.pinniped.dev_credentialissuers.yaml @@ -77,7 +77,7 @@ spec: PEM CA bundle of the impersonation proxy. minLength: 1 type: string - server: + endpoint: description: Endpoint is the HTTPS endpoint of the impersonation proxy. minLength: 1 @@ -85,7 +85,7 @@ spec: type: string required: - certificateAuthorityData - - server + - endpoint type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the diff --git a/generated/1.17/README.adoc b/generated/1.17/README.adoc index b062b7f4..25267e8c 100644 --- a/generated/1.17/README.adoc +++ b/generated/1.17/README.adoc @@ -328,7 +328,7 @@ Status of a credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. | *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. |=== diff --git a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go index e38b207b..3c0c8ba0 100644 --- a/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.17/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -111,7 +111,7 @@ type ImpersonationProxyInfo struct { // Endpoint is the HTTPS endpoint of the impersonation proxy. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://` - Endpoint string `json:"server"` + Endpoint string `json:"endpoint"` // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 diff --git a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 7436f6e2..8123f238 100644 --- a/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.17/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -77,7 +77,7 @@ spec: PEM CA bundle of the impersonation proxy. minLength: 1 type: string - server: + endpoint: description: Endpoint is the HTTPS endpoint of the impersonation proxy. minLength: 1 @@ -85,7 +85,7 @@ spec: type: string required: - certificateAuthorityData - - server + - endpoint type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the diff --git a/generated/1.18/README.adoc b/generated/1.18/README.adoc index 18ec63f7..2c01f927 100644 --- a/generated/1.18/README.adoc +++ b/generated/1.18/README.adoc @@ -328,7 +328,7 @@ Status of a credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. | *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. |=== diff --git a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go index e38b207b..3c0c8ba0 100644 --- a/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.18/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -111,7 +111,7 @@ type ImpersonationProxyInfo struct { // Endpoint is the HTTPS endpoint of the impersonation proxy. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://` - Endpoint string `json:"server"` + Endpoint string `json:"endpoint"` // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 diff --git a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 7436f6e2..8123f238 100644 --- a/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.18/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -77,7 +77,7 @@ spec: PEM CA bundle of the impersonation proxy. minLength: 1 type: string - server: + endpoint: description: Endpoint is the HTTPS endpoint of the impersonation proxy. minLength: 1 @@ -85,7 +85,7 @@ spec: type: string required: - certificateAuthorityData - - server + - endpoint type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the diff --git a/generated/1.19/README.adoc b/generated/1.19/README.adoc index 612620fe..0c2527ae 100644 --- a/generated/1.19/README.adoc +++ b/generated/1.19/README.adoc @@ -328,7 +328,7 @@ Status of a credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. | *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. |=== diff --git a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go index e38b207b..3c0c8ba0 100644 --- a/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.19/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -111,7 +111,7 @@ type ImpersonationProxyInfo struct { // Endpoint is the HTTPS endpoint of the impersonation proxy. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://` - Endpoint string `json:"server"` + Endpoint string `json:"endpoint"` // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 diff --git a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 7436f6e2..8123f238 100644 --- a/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.19/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -77,7 +77,7 @@ spec: PEM CA bundle of the impersonation proxy. minLength: 1 type: string - server: + endpoint: description: Endpoint is the HTTPS endpoint of the impersonation proxy. minLength: 1 @@ -85,7 +85,7 @@ spec: type: string required: - certificateAuthorityData - - server + - endpoint type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the diff --git a/generated/1.20/README.adoc b/generated/1.20/README.adoc index 97b02d4e..91b4eef6 100644 --- a/generated/1.20/README.adoc +++ b/generated/1.20/README.adoc @@ -328,7 +328,7 @@ Status of a credential issuer. [cols="25a,75a", options="header"] |=== | Field | Description -| *`server`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. +| *`endpoint`* __string__ | Endpoint is the HTTPS endpoint of the impersonation proxy. | *`certificateAuthorityData`* __string__ | CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. |=== diff --git a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go index e38b207b..3c0c8ba0 100644 --- a/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/1.20/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -111,7 +111,7 @@ type ImpersonationProxyInfo struct { // Endpoint is the HTTPS endpoint of the impersonation proxy. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://` - Endpoint string `json:"server"` + Endpoint string `json:"endpoint"` // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 diff --git a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml index 7436f6e2..8123f238 100644 --- a/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml +++ b/generated/1.20/crds/config.concierge.pinniped.dev_credentialissuers.yaml @@ -77,7 +77,7 @@ spec: PEM CA bundle of the impersonation proxy. minLength: 1 type: string - server: + endpoint: description: Endpoint is the HTTPS endpoint of the impersonation proxy. minLength: 1 @@ -85,7 +85,7 @@ spec: type: string required: - certificateAuthorityData - - server + - endpoint type: object tokenCredentialRequestInfo: description: TokenCredentialRequestAPIInfo describes the diff --git a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go index e38b207b..3c0c8ba0 100644 --- a/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go +++ b/generated/latest/apis/concierge/config/v1alpha1/types_credentialissuer.go @@ -111,7 +111,7 @@ type ImpersonationProxyInfo struct { // Endpoint is the HTTPS endpoint of the impersonation proxy. // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:Pattern=`^https://` - Endpoint string `json:"server"` + Endpoint string `json:"endpoint"` // CertificateAuthorityData is the base64-encoded PEM CA bundle of the impersonation proxy. // +kubebuilder:validation:MinLength=1 diff --git a/go.sum b/go.sum index 8a2191e1..74408122 100644 --- a/go.sum +++ b/go.sum @@ -1176,7 +1176,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= From 5e4746e96b72948e0708489efea2474139c1a983 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Fri, 12 Mar 2021 16:36:37 -0500 Subject: [PATCH 154/203] impersonator: match kube API server long running func Signed-off-by: Monis Khan --- internal/concierge/impersonator/impersonator.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 169d3aaf..87f2d1b4 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -22,6 +22,7 @@ import ( "k8s.io/apiserver/pkg/endpoints/request" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/rest" "k8s.io/client-go/transport" @@ -123,6 +124,12 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. // See sanity checks at the end of this function. serverConfig.LoopbackClientConfig.BearerToken = "" + // match KAS exactly since our long running operations are just a proxy to it + serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( + sets.NewString("watch", "proxy"), + sets.NewString("attach", "exec", "proxy", "log", "portforward"), + ) + // Assume proto config is safe because transport level configs do not use rest.ContentConfig. // Thus if we are interacting with actual APIs, they should be using pre-built clients. impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClient.ProtoConfig)) From c82f568b2c14c16eb918f66ed7b4235db52b87e3 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 12 Mar 2021 16:09:16 -0800 Subject: [PATCH 155/203] certauthority.go: Refactor issuing client versus server certs We were previously issuing both client certs and server certs with both extended key usages included. Split the Issue*() methods into separate methods for issuing server certs versus client certs so they can have different extended key usages tailored for each use case. Also took the opportunity to clean up the parameters of the Issue*() methods and New() methods to more closely match how we prefer to call them. We were always only passing the common name part of the pkix.Name to New(), so now the New() method just takes the common name as a string. When making a server cert, we don't need to set the deprecated common name field, so remove that param. When making a client cert, we're always making it in the format expected by the Kube API server, so just accept the username and group as parameters directly. --- cmd/local-user-authenticator/main_test.go | 5 +- cmd/pinniped/cmd/flag_types_test.go | 3 +- cmd/pinniped/cmd/kubeconfig_test.go | 5 +- cmd/pinniped/cmd/login_oidc_test.go | 3 +- cmd/pinniped/cmd/login_static_test.go | 3 +- internal/certauthority/certauthority.go | 57 ++++--- internal/certauthority/certauthority_test.go | 146 ++++++++++++++---- .../dynamiccertauthority.go | 7 +- .../dynamiccertauthority_test.go | 18 +-- internal/concierge/apiserver/apiserver.go | 2 +- .../impersonator/impersonator_test.go | 11 +- internal/concierge/server/server.go | 4 +- internal/controller/apicerts/certs_manager.go | 10 +- .../controller/apicerts/certs_manager_test.go | 6 +- .../impersonatorconfig/impersonator_config.go | 19 ++- .../impersonator_config_test.go | 14 +- internal/issuer/issuer.go | 13 +- .../credentialrequestmocks.go | 43 +----- .../mocks/credentialrequestmocks/generate.go | 4 +- internal/mocks/issuermocks/generate.go | 6 + internal/mocks/issuermocks/issuermocks.go | 55 +++++++ internal/registry/credentialrequest/rest.go | 22 +-- .../registry/credentialrequest/rest_test.go | 32 ++-- internal/testutil/certs.go | 54 ++++++- pkg/conciergeclient/conciergeclient_test.go | 3 +- test/integration/e2e_test.go | 10 +- test/integration/supervisor_discovery_test.go | 5 +- test/integration/supervisor_login_test.go | 10 +- 28 files changed, 346 insertions(+), 224 deletions(-) create mode 100644 internal/mocks/issuermocks/generate.go create mode 100644 internal/mocks/issuermocks/issuermocks.go diff --git a/cmd/local-user-authenticator/main_test.go b/cmd/local-user-authenticator/main_test.go index 632fbde0..5a6fb0bd 100644 --- a/cmd/local-user-authenticator/main_test.go +++ b/cmd/local-user-authenticator/main_test.go @@ -8,7 +8,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/json" "fmt" "io" @@ -464,10 +463,10 @@ func newCertProvider(t *testing.T) (dynamiccert.Provider, []byte, string) { serverName := "local-user-authenticator" - ca, err := certauthority.New(pkix.Name{CommonName: serverName + " CA"}, time.Hour*24) + ca, err := certauthority.New(serverName+" CA", time.Hour*24) require.NoError(t, err) - cert, err := ca.Issue(pkix.Name{CommonName: serverName}, []string{serverName}, nil, time.Hour*24) + cert, err := ca.IssueServerCert([]string{serverName}, nil, time.Hour*24) require.NoError(t, err) certPEM, keyPEM, err := certauthority.ToPEM(cert) diff --git a/cmd/pinniped/cmd/flag_types_test.go b/cmd/pinniped/cmd/flag_types_test.go index 6d967969..101191d5 100644 --- a/cmd/pinniped/cmd/flag_types_test.go +++ b/cmd/pinniped/cmd/flag_types_test.go @@ -5,7 +5,6 @@ package cmd import ( "bytes" - "crypto/x509/pkix" "fmt" "io/ioutil" "path/filepath" @@ -51,7 +50,7 @@ func TestConciergeModeFlag(t *testing.T) { } func TestCABundleFlag(t *testing.T) { - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) emptyFilePath := filepath.Join(tmpdir, "empty") diff --git a/cmd/pinniped/cmd/kubeconfig_test.go b/cmd/pinniped/cmd/kubeconfig_test.go index f3b27d5d..879e2d71 100644 --- a/cmd/pinniped/cmd/kubeconfig_test.go +++ b/cmd/pinniped/cmd/kubeconfig_test.go @@ -5,7 +5,6 @@ package cmd import ( "bytes" - "crypto/x509/pkix" "encoding/base64" "fmt" "io/ioutil" @@ -30,13 +29,13 @@ import ( ) func TestGetKubeconfig(t *testing.T) { - testOIDCCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testOIDCCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) testOIDCCABundlePath := filepath.Join(tmpdir, "testca.pem") require.NoError(t, ioutil.WriteFile(testOIDCCABundlePath, testOIDCCA.Bundle(), 0600)) - testConciergeCA, err := certauthority.New(pkix.Name{CommonName: "Test Concierge CA"}, 1*time.Hour) + testConciergeCA, err := certauthority.New("Test Concierge CA", 1*time.Hour) require.NoError(t, err) testConciergeCABundlePath := filepath.Join(tmpdir, "testconciergeca.pem") require.NoError(t, ioutil.WriteFile(testConciergeCABundlePath, testConciergeCA.Bundle(), 0600)) diff --git a/cmd/pinniped/cmd/login_oidc_test.go b/cmd/pinniped/cmd/login_oidc_test.go index 976e3adf..2b1e8469 100644 --- a/cmd/pinniped/cmd/login_oidc_test.go +++ b/cmd/pinniped/cmd/login_oidc_test.go @@ -6,7 +6,6 @@ package cmd import ( "bytes" "context" - "crypto/x509/pkix" "encoding/base64" "fmt" "io/ioutil" @@ -29,7 +28,7 @@ import ( func TestLoginOIDCCommand(t *testing.T) { cfgDir := mustGetConfigDir() - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) testCABundlePath := filepath.Join(tmpdir, "testca.pem") diff --git a/cmd/pinniped/cmd/login_static_test.go b/cmd/pinniped/cmd/login_static_test.go index 1d097123..caf41df5 100644 --- a/cmd/pinniped/cmd/login_static_test.go +++ b/cmd/pinniped/cmd/login_static_test.go @@ -6,7 +6,6 @@ package cmd import ( "bytes" "context" - "crypto/x509/pkix" "fmt" "io/ioutil" "path/filepath" @@ -24,7 +23,7 @@ import ( ) func TestLoginStaticCommand(t *testing.T) { - testCA, err := certauthority.New(pkix.Name{CommonName: "Test CA"}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tmpdir := testutil.TempDir(t) testCABundlePath := filepath.Join(tmpdir, "testca.pem") diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index ad81e38a..b64bf0fa 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -89,13 +89,13 @@ func Load(certPEM string, keyPEM string) (*CA, error) { }, nil } -// New generates a fresh certificate authority with the given subject and ttl. -func New(subject pkix.Name, ttl time.Duration) (*CA, error) { - return newInternal(subject, ttl, secureEnv()) +// New generates a fresh certificate authority with the given Common Name and TTL. +func New(commonName string, ttl time.Duration) (*CA, error) { + return newInternal(commonName, ttl, secureEnv()) } // newInternal is the internal guts of New, broken out for easier testing. -func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { +func newInternal(commonName string, ttl time.Duration, env env) (*CA, error) { ca := CA{env: env} // Generate a random serial for the CA serialNumber, err := randomSerial(env.serialRNG) @@ -118,7 +118,7 @@ func newInternal(subject pkix.Name, ttl time.Duration, env env) (*CA, error) { // Create CA cert template caTemplate := x509.Certificate{ SerialNumber: serialNumber, - Subject: subject, + Subject: pkix.Name{CommonName: commonName}, NotBefore: notBefore, NotAfter: notAfter, IsCA: true, @@ -160,8 +160,31 @@ func (c *CA) Pool() *x509.CertPool { return pool } -// Issue a new server certificate for the given identity and duration. -func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) { +// IssueClientCert issues a new client certificate with username and groups included in the Kube-style +// certificate subject for the given identity and duration. +func (c *CA) IssueClientCert(username string, groups []string, ttl time.Duration) (*tls.Certificate, error) { + return c.issueCert(x509.ExtKeyUsageClientAuth, pkix.Name{CommonName: username, Organization: groups}, nil, nil, ttl) +} + +// IssueServerCert issues a new server certificate for the given identity and duration. +// The dnsNames and ips are each optional, but at least one of them should be specified. +func (c *CA) IssueServerCert(dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) { + return c.issueCert(x509.ExtKeyUsageServerAuth, pkix.Name{}, dnsNames, ips, ttl) +} + +// Similar to IssueClientCert, but returning the new cert as a pair of PEM-formatted byte slices +// for the certificate and private key. +func (c *CA) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { + return toPEM(c.IssueClientCert(username, groups, ttl)) +} + +// Similar to IssueServerCert, but returning the new cert as a pair of PEM-formatted byte slices +// for the certificate and private key. +func (c *CA) IssueServerCertPEM(dnsNames []string, ips []net.IP, ttl time.Duration) ([]byte, []byte, error) { + return toPEM(c.IssueServerCert(dnsNames, ips, ttl)) +} + +func (c *CA) issueCert(extKeyUsage x509.ExtKeyUsage, subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) (*tls.Certificate, error) { // Choose a random 128 bit serial number. serialNumber, err := randomSerial(c.env.serialRNG) if err != nil { @@ -187,13 +210,11 @@ func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time. // Sign a cert, getting back the DER-encoded certificate bytes. template := x509.Certificate{ - SerialNumber: serialNumber, - Subject: subject, - NotBefore: notBefore, - NotAfter: notAfter, - KeyUsage: x509.KeyUsageDigitalSignature, - // TODO split this function into two funcs that handle client and serving certs differently - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + SerialNumber: serialNumber, + Subject: subject, + NotBefore: notBefore, + NotAfter: notAfter, + ExtKeyUsage: []x509.ExtKeyUsage{extKeyUsage}, BasicConstraintsValid: true, IsCA: false, DNSNames: dnsNames, @@ -218,14 +239,8 @@ func (c *CA) Issue(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time. }, nil } -// IssuePEM issues a new server certificate for the given identity and duration, returning it as a pair of -// PEM-formatted byte slices for the certificate and private key. -func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ips []net.IP, ttl time.Duration) ([]byte, []byte, error) { - return toPEM(c.Issue(subject, dnsNames, ips, ttl)) -} - func toPEM(cert *tls.Certificate, err error) ([]byte, []byte, error) { - // If the wrapped Issue() returned an error, pass it back. + // If the wrapped IssueServerCert() returned an error, pass it back. if err != nil { return nil, nil, err } diff --git a/internal/certauthority/certauthority_test.go b/internal/certauthority/certauthority_test.go index 7f9e3571..80948a77 100644 --- a/internal/certauthority/certauthority_test.go +++ b/internal/certauthority/certauthority_test.go @@ -7,7 +7,6 @@ import ( "crypto" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "fmt" "io" "io/ioutil" @@ -17,6 +16,8 @@ import ( "time" "github.com/stretchr/testify/require" + + "go.pinniped.dev/internal/testutil" ) func loadFromFiles(t *testing.T, certPath string, keyPath string) (*CA, error) { @@ -87,7 +88,7 @@ func TestLoad(t *testing.T) { func TestNew(t *testing.T) { now := time.Now() - ca, err := New(pkix.Name{CommonName: "Test CA"}, time.Minute) + ca, err := New("Test CA", time.Minute) require.NoError(t, err) require.NotNil(t, ca) @@ -158,7 +159,7 @@ func TestNewInternal(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := newInternal(pkix.Name{CommonName: "Test CA"}, tt.ttl, tt.env) + got, err := newInternal("Test CA", tt.ttl, tt.env) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) require.Nil(t, got) @@ -184,7 +185,7 @@ func TestBundle(t *testing.T) { } func TestPrivateKeyToPEM(t *testing.T) { - ca, err := New(pkix.Name{CommonName: "Test CA"}, time.Hour) + ca, err := New("Test CA", time.Hour) require.NoError(t, err) keyPEM, err := ca.PrivateKeyToPEM() require.NoError(t, err) @@ -201,7 +202,7 @@ func TestPrivateKeyToPEM(t *testing.T) { } func TestPool(t *testing.T) { - ca, err := New(pkix.Name{CommonName: "test"}, 1*time.Hour) + ca, err := New("test", 1*time.Hour) require.NoError(t, err) pool := ca.Pool() @@ -220,6 +221,8 @@ func (e *errSigner) Sign(_ io.Reader, _ []byte, _ crypto.SignerOpts) ([]byte, er } func TestIssue(t *testing.T) { + const numRandBytes = 64 * 2 // each call to issue a cert will consume 64 bytes from the reader + now := time.Date(2020, 7, 10, 12, 41, 12, 1234, time.UTC) realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") @@ -243,7 +246,7 @@ func TestIssue(t *testing.T) { name: "failed to generate keypair", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), keygenRNG: strings.NewReader(""), }, }, @@ -253,8 +256,8 @@ func TestIssue(t *testing.T) { name: "invalid CA certificate", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, }, }, @@ -264,8 +267,8 @@ func TestIssue(t *testing.T) { name: "signing error", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, }, caCertBytes: realCA.caCertBytes, @@ -277,11 +280,11 @@ func TestIssue(t *testing.T) { wantErr: "could not sign certificate: some signer error", }, { - name: "success", + name: "parse certificate error", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, parseCert: func(_ []byte) (*x509.Certificate, error) { return nil, fmt.Errorf("some parse certificate error") @@ -296,8 +299,8 @@ func TestIssue(t *testing.T) { name: "success", ca: CA{ env: env{ - serialRNG: strings.NewReader(strings.Repeat("x", 64)), - keygenRNG: strings.NewReader(strings.Repeat("x", 64)), + serialRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), + keygenRNG: strings.NewReader(strings.Repeat("x", numRandBytes)), clock: func() time.Time { return now }, parseCert: x509.ParseCertificate, }, @@ -309,28 +312,26 @@ func TestIssue(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := tt.ca.Issue(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, []net.IP{net.IPv4(1, 2, 3, 4)}, 10*time.Minute) + got, err := tt.ca.IssueServerCert([]string{"example.com"}, []net.IP{net.IPv4(1, 2, 3, 4)}, 10*time.Minute) if tt.wantErr != "" { require.EqualError(t, err, tt.wantErr) require.Nil(t, got) - return + } else { + require.NoError(t, err) + require.NotNil(t, got) + } + got, err = tt.ca.IssueClientCert("test-user", []string{"group1", "group2"}, 10*time.Minute) + if tt.wantErr != "" { + require.EqualError(t, err, tt.wantErr) + require.Nil(t, got) + } else { + require.NoError(t, err) + require.NotNil(t, got) } - require.NoError(t, err) - require.NotNil(t, got) }) } } -func TestIssuePEM(t *testing.T) { - realCA, err := loadFromFiles(t, "./testdata/test.crt", "./testdata/test.key") - require.NoError(t, err) - - certPEM, keyPEM, err := realCA.IssuePEM(pkix.Name{CommonName: "Test Server"}, []string{"example.com"}, nil, 10*time.Minute) - require.NoError(t, err) - require.NotEmpty(t, certPEM) - require.NotEmpty(t, keyPEM) -} - func TestToPEM(t *testing.T) { realCert, err := tls.LoadX509KeyPair("./testdata/test.crt", "./testdata/test.key") require.NoError(t, err) @@ -358,3 +359,90 @@ func TestToPEM(t *testing.T) { require.NotEmpty(t, keyPEM) }) } + +func TestIssueMethods(t *testing.T) { + // One CA can be used to issue both kinds of certs. + ca, err := New("Test CA", time.Hour) + require.NoError(t, err) + + ttl := 121 * time.Hour + + t.Run("client certs", func(t *testing.T) { + user := "test-username" + groups := []string{"group1", "group2"} + + clientCert, err := ca.IssueClientCert(user, groups, ttl) + require.NoError(t, err) + certPEM, keyPEM, err := ToPEM(clientCert) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, groups, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM(user, groups, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, groups, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM(user, nil, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, nil, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM(user, []string{}, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, user, nil, ttl) + + certPEM, keyPEM, err = ca.IssueClientCertPEM("", []string{}, ttl) + require.NoError(t, err) + validateClientCert(t, ca.Bundle(), certPEM, keyPEM, "", nil, ttl) + }) + + t.Run("server certs", func(t *testing.T) { + dnsNames := []string{"example.com", "pinniped.dev"} + ips := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("1.2.3.4")} + + serverCert, err := ca.IssueServerCert(dnsNames, ips, ttl) + require.NoError(t, err) + certPEM, keyPEM, err := ToPEM(serverCert) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, ips, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(nil, ips, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, nil, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, nil, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, nil, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM([]string{}, ips, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, nil, ips, ttl) + + certPEM, keyPEM, err = ca.IssueServerCertPEM(dnsNames, []net.IP{}, ttl) + require.NoError(t, err) + validateServerCert(t, ca.Bundle(), certPEM, keyPEM, dnsNames, nil, ttl) + }) +} + +func validateClientCert(t *testing.T, caBundle []byte, certPEM []byte, keyPEM []byte, expectedUser string, expectedGroups []string, expectedTTL time.Duration) { + const fudgeFactor = 10 * time.Second + v := testutil.ValidateClientCertificate(t, string(caBundle), string(certPEM)) + v.RequireLifetime(time.Now(), time.Now().Add(expectedTTL), certBackdate+fudgeFactor) + v.RequireMatchesPrivateKey(string(keyPEM)) + v.RequireCommonName(expectedUser) + v.RequireOrganizations(expectedGroups) + v.RequireEmptyDNSNames() + v.RequireEmptyIPs() +} + +func validateServerCert(t *testing.T, caBundle []byte, certPEM []byte, keyPEM []byte, expectedDNSNames []string, expectedIPs []net.IP, expectedTTL time.Duration) { + const fudgeFactor = 10 * time.Second + v := testutil.ValidateServerCertificate(t, string(caBundle), string(certPEM)) + v.RequireLifetime(time.Now(), time.Now().Add(expectedTTL), certBackdate+fudgeFactor) + v.RequireMatchesPrivateKey(string(keyPEM)) + v.RequireCommonName("") + v.RequireDNSNames(expectedDNSNames) + v.RequireIPs(expectedIPs) +} diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go index 3c0c4b72..78eb97c6 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go @@ -6,7 +6,6 @@ package dynamiccertauthority import ( - "crypto/x509/pkix" "time" "k8s.io/apiserver/pkg/server/dynamiccertificates" @@ -27,14 +26,14 @@ func New(provider dynamiccertificates.CertKeyContentProvider) *CA { } } -// IssuePEM issues a new server certificate for the given identity and duration, returning it as a +// IssueClientCertPEM issues a new client certificate for the given identity and duration, returning it as a // pair of PEM-formatted byte slices for the certificate and private key. -func (c *CA) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { +func (c *CA) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { caCrtPEM, caKeyPEM := c.provider.CurrentCertKeyContent() ca, err := certauthority.Load(string(caCrtPEM), string(caKeyPEM)) if err != nil { return nil, nil, err } - return ca.IssuePEM(subject, dnsNames, nil, ttl) + return ca.IssueClientCertPEM(username, groups, ttl) } diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go index 41e0f291..47bc6539 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go @@ -1,10 +1,9 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package dynamiccertauthority import ( - "crypto/x509/pkix" "testing" "time" @@ -106,10 +105,9 @@ func TestCAIssuePEM(t *testing.T) { require.NotEmpty(t, keyPEM) caCrtPEM, _ := provider.CurrentCertKeyContent() - crtAssertions := testutil.ValidateCertificate(t, string(caCrtPEM), string(crtPEM)) - crtAssertions.RequireCommonName("some-common-name") - crtAssertions.RequireDNSName("some-dns-name") - crtAssertions.RequireDNSName("some-other-dns-name") + crtAssertions := testutil.ValidateClientCertificate(t, string(caCrtPEM), string(crtPEM)) + crtAssertions.RequireCommonName("some-username") + crtAssertions.RequireOrganizations([]string{"some-group1", "some-group2"}) crtAssertions.RequireLifetime(time.Now(), time.Now().Add(time.Hour*24), time.Minute*10) crtAssertions.RequireMatchesPrivateKey(string(keyPEM)) } @@ -126,11 +124,5 @@ func issuePEM(provider dynamiccert.Provider, ca *CA, caCrt, caKey []byte) ([]byt } // otherwise check to see if their is an issuing error - return ca.IssuePEM( - pkix.Name{ - CommonName: "some-common-name", - }, - []string{"some-dns-name", "some-other-dns-name"}, - time.Hour*24, - ) + return ca.IssueClientCertPEM("some-username", []string{"some-group1", "some-group2"}, time.Hour*24) } diff --git a/internal/concierge/apiserver/apiserver.go b/internal/concierge/apiserver/apiserver.go index f64e0104..f1bce95b 100644 --- a/internal/concierge/apiserver/apiserver.go +++ b/internal/concierge/apiserver/apiserver.go @@ -28,7 +28,7 @@ type Config struct { type ExtraConfig struct { Authenticator credentialrequest.TokenCredentialRequestAuthenticator - Issuer issuer.CertIssuer + Issuer issuer.ClientCertIssuer StartControllersPostStartHook func(ctx context.Context) Scheme *runtime.Scheme NegotiatedSerializer runtime.NegotiatedSerializer diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 9d1e231c..66985781 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -5,7 +5,6 @@ package impersonator import ( "context" - "crypto/x509/pkix" "net" "net/http" "net/http/httptest" @@ -36,7 +35,7 @@ import ( func TestImpersonator(t *testing.T) { const port = 9444 - ca, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) + ca, err := certauthority.New("ca", time.Hour) require.NoError(t, err) caKey, err := ca.PrivateKeyToPEM() require.NoError(t, err) @@ -44,13 +43,13 @@ func TestImpersonator(t *testing.T) { err = caContent.SetCertKeyContent(ca.Bundle(), caKey) require.NoError(t, err) - cert, key, err := ca.IssuePEM(pkix.Name{}, nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) + cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) require.NoError(t, err) certKeyContent := dynamiccert.New("cert-key") err = certKeyContent.SetCertKeyContent(cert, key) require.NoError(t, err) - unrelatedCA, err := certauthority.New(pkix.Name{CommonName: "ca"}, time.Hour) + unrelatedCA, err := certauthority.New("ca", time.Hour) require.NoError(t, err) // Punch out just enough stuff to make New actually run without error. @@ -486,9 +485,7 @@ type clientCert struct { } func newClientCert(t *testing.T, ca *certauthority.CA, username string, groups []string) *clientCert { - certPEM, keyPEM, err := ca.IssuePEM( - pkix.Name{CommonName: username, Organization: groups}, nil, nil, time.Hour, - ) + certPEM, keyPEM, err := ca.IssueClientCertPEM(username, groups, time.Hour) require.NoError(t, err) return &clientCert{ certPEM: certPEM, diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index 8602d07d..4a4415a5 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -150,7 +150,7 @@ func (a *App) runServer(ctx context.Context) error { return fmt.Errorf("could not prepare controllers: %w", err) } - certIssuer := issuer.CertIssuers{ + certIssuer := issuer.ClientCertIssuers{ dynamiccertauthority.New(dynamicSigningCertProvider), // attempt to use the real Kube CA if possible dynamiccertauthority.New(impersonationProxySigningCertProvider), // fallback to our internal CA if we need to } @@ -184,7 +184,7 @@ func (a *App) runServer(ctx context.Context) error { func getAggregatedAPIServerConfig( dynamicCertProvider dynamiccert.Provider, authenticator credentialrequest.TokenCredentialRequestAuthenticator, - issuer issuer.CertIssuer, + issuer issuer.ClientCertIssuer, startControllersPostStartHook func(context.Context), apiGroupSuffix string, scheme *runtime.Scheme, diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index f0c11d0d..c3d8bf36 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -4,7 +4,6 @@ package apicerts import ( - "crypto/x509/pkix" "fmt" "time" @@ -94,7 +93,7 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { } // Create a CA. - ca, err := certauthority.New(pkix.Name{CommonName: c.generatedCACommonName}, c.certDuration) + ca, err := certauthority.New(c.generatedCACommonName, c.certDuration) if err != nil { return fmt.Errorf("could not initialize CA: %w", err) } @@ -120,12 +119,7 @@ func (c *certsManagerController) Sync(ctx controllerlib.Context) error { // Using the CA from above, create a TLS server cert if we have service name. if len(c.serviceNameForGeneratedCertCommonName) != 0 { serviceEndpoint := c.serviceNameForGeneratedCertCommonName + "." + c.namespace + ".svc" - tlsCert, err := ca.Issue( - pkix.Name{CommonName: serviceEndpoint}, - []string{serviceEndpoint}, - nil, - c.certDuration, - ) + tlsCert, err := ca.IssueServerCert([]string{serviceEndpoint}, nil, c.certDuration) if err != nil { return fmt.Errorf("could not issue serving certificate: %w", err) } diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index b32b9228..092f504b 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -218,12 +218,12 @@ func TestManagerControllerSync(t *testing.T) { r.NotEmpty(actualCertChain) r.Len(actualSecret.StringData, 4) - validCACert := testutil.ValidateCertificate(t, actualCACert, actualCACert) + validCACert := testutil.ValidateServerCertificate(t, actualCACert, actualCACert) validCACert.RequireMatchesPrivateKey(actualCAPrivateKey) validCACert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) // Validate the created cert using the CA, and also validate the cert's hostname - validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain) + validCert := testutil.ValidateServerCertificate(t, actualCACert, actualCertChain) validCert.RequireDNSName("pinniped-api." + installedInNamespace + ".svc") validCert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) validCert.RequireMatchesPrivateKey(actualPrivateKey) @@ -252,7 +252,7 @@ func TestManagerControllerSync(t *testing.T) { r.NotEmpty(actualCAPrivateKey) r.Len(actualSecret.StringData, 2) - validCACert := testutil.ValidateCertificate(t, actualCACert, actualCACert) + validCACert := testutil.ValidateServerCertificate(t, actualCACert, actualCACert) validCACert.RequireMatchesPrivateKey(actualCAPrivateKey) validCACert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 6*time.Minute) }) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 6f81901d..1733d9da 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -7,7 +7,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/base64" "encoding/pem" "fmt" @@ -40,13 +39,13 @@ import ( ) const ( - impersonationProxyPort = 8444 - defaultHTTPSPort = 443 - oneHundredYears = 100 * 365 * 24 * time.Hour - caCommonName = "Pinniped Impersonation Proxy CA" - caCrtKey = "ca.crt" - caKeyKey = "ca.key" - appLabelKey = "app" + impersonationProxyPort = 8444 + defaultHTTPSPort = 443 + approximatelyOneHundredYears = 100 * 365 * 24 * time.Hour + caCommonName = "Pinniped Impersonation Proxy CA" + caCrtKey = "ca.crt" + caKeyKey = "ca.key" + appLabelKey = "app" ) type impersonatorConfigController struct { @@ -622,7 +621,7 @@ func (c *impersonatorConfigController) ensureCASecretIsCreated(ctx context.Conte } func (c *impersonatorConfigController) createCASecret(ctx context.Context) (*certauthority.CA, error) { - impersonationCA, err := certauthority.New(pkix.Name{CommonName: caCommonName}, oneHundredYears) + impersonationCA, err := certauthority.New(caCommonName, approximatelyOneHundredYears) if err != nil { return nil, fmt.Errorf("could not create impersonation CA: %w", err) } @@ -716,7 +715,7 @@ func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, c ips = []net.IP{ip} } - impersonationCert, err := ca.Issue(pkix.Name{}, hostnames, ips, oneHundredYears) + impersonationCert, err := ca.IssueServerCert(hostnames, ips, approximatelyOneHundredYears) if err != nil { return nil, fmt.Errorf("could not create impersonation cert: %w", err) } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index f926208b..f4996184 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -7,7 +7,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/base64" "encoding/pem" "errors" @@ -352,7 +351,10 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { require.NoError(t, err) roots := x509.NewCertPool() require.True(t, roots.AppendCertsFromPEM(currentClientCertCA)) - opts := x509.VerifyOptions{Roots: roots} + opts := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + } _, err = parsed.Verify(opts) require.NoError(t, err) return nil @@ -594,7 +596,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var newCA = func() *certauthority.CA { - ca, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) + ca, err := certauthority.New("test CA", 24*time.Hour) r.NoError(err) return ca } @@ -609,7 +611,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } var newTLSCertSecretData = func(ca *certauthority.CA, dnsNames []string, ip string) map[string][]byte { - impersonationCert, err := ca.Issue(pkix.Name{}, dnsNames, []net.IP{net.ParseIP(ip)}, 24*time.Hour) + impersonationCert, err := ca.IssueServerCert(dnsNames, []net.IP{net.ParseIP(ip)}, 24*time.Hour) r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) r.NoError(err) @@ -939,7 +941,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { createdKeyPEM := createdSecret.Data[corev1.TLSPrivateKeyKey] r.NotNil(createdKeyPEM) r.NotNil(createdCertPEM) - validCert := testutil.ValidateCertificate(t, string(caCert), string(createdCertPEM)) + validCert := testutil.ValidateServerCertificate(t, string(caCert), string(createdCertPEM)) validCert.RequireMatchesPrivateKey(string(createdKeyPEM)) validCert.RequireLifetime(time.Now().Add(-10*time.Second), time.Now().Add(100*time.Hour*24*365), 10*time.Second) } @@ -980,7 +982,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { signingCAKeyPEM, err = ca.PrivateKeyToPEM() r.NoError(err) signingCASecret = newSigningKeySecret(caSignerName, signingCACertPEM, signingCAKeyPEM) - validClientCert, err = ca.Issue(pkix.Name{}, nil, nil, time.Hour) + validClientCert, err = ca.IssueClientCert("username", nil, time.Hour) r.NoError(err) }) diff --git a/internal/issuer/issuer.go b/internal/issuer/issuer.go index 94e2f235..88ac1538 100644 --- a/internal/issuer/issuer.go +++ b/internal/issuer/issuer.go @@ -4,7 +4,6 @@ package issuer import ( - "crypto/x509/pkix" "time" "k8s.io/apimachinery/pkg/util/errors" @@ -14,19 +13,19 @@ import ( const defaultCertIssuerErr = constable.Error("failed to issue cert") -type CertIssuer interface { - IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) (certPEM, keyPEM []byte, err error) +type ClientCertIssuer interface { + IssueClientCertPEM(username string, groups []string, ttl time.Duration) (certPEM, keyPEM []byte, err error) } -var _ CertIssuer = CertIssuers{} +var _ ClientCertIssuer = ClientCertIssuers{} -type CertIssuers []CertIssuer +type ClientCertIssuers []ClientCertIssuer -func (c CertIssuers) IssuePEM(subject pkix.Name, dnsNames []string, ttl time.Duration) ([]byte, []byte, error) { +func (c ClientCertIssuers) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { var errs []error for _, issuer := range c { - certPEM, keyPEM, err := issuer.IssuePEM(subject, dnsNames, ttl) + certPEM, keyPEM, err := issuer.IssueClientCertPEM(username, groups, ttl) if err != nil { errs = append(errs, err) continue diff --git a/internal/mocks/credentialrequestmocks/credentialrequestmocks.go b/internal/mocks/credentialrequestmocks/credentialrequestmocks.go index 58bd134d..68ff7f2e 100644 --- a/internal/mocks/credentialrequestmocks/credentialrequestmocks.go +++ b/internal/mocks/credentialrequestmocks/credentialrequestmocks.go @@ -3,61 +3,20 @@ // // Code generated by MockGen. DO NOT EDIT. -// Source: go.pinniped.dev/internal/registry/credentialrequest (interfaces: CertIssuer,TokenCredentialRequestAuthenticator) +// Source: go.pinniped.dev/internal/registry/credentialrequest (interfaces: TokenCredentialRequestAuthenticator) // Package credentialrequestmocks is a generated GoMock package. package credentialrequestmocks import ( context "context" - pkix "crypto/x509/pkix" reflect "reflect" - time "time" gomock "github.com/golang/mock/gomock" login "go.pinniped.dev/generated/latest/apis/concierge/login" user "k8s.io/apiserver/pkg/authentication/user" ) -// MockCertIssuer is a mock of CertIssuer interface. -type MockCertIssuer struct { - ctrl *gomock.Controller - recorder *MockCertIssuerMockRecorder -} - -// MockCertIssuerMockRecorder is the mock recorder for MockCertIssuer. -type MockCertIssuerMockRecorder struct { - mock *MockCertIssuer -} - -// NewMockCertIssuer creates a new mock instance. -func NewMockCertIssuer(ctrl *gomock.Controller) *MockCertIssuer { - mock := &MockCertIssuer{ctrl: ctrl} - mock.recorder = &MockCertIssuerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockCertIssuer) EXPECT() *MockCertIssuerMockRecorder { - return m.recorder -} - -// IssuePEM mocks base method. -func (m *MockCertIssuer) IssuePEM(arg0 pkix.Name, arg1 []string, arg2 time.Duration) ([]byte, []byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IssuePEM", arg0, arg1, arg2) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].([]byte) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// IssuePEM indicates an expected call of IssuePEM. -func (mr *MockCertIssuerMockRecorder) IssuePEM(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssuePEM", reflect.TypeOf((*MockCertIssuer)(nil).IssuePEM), arg0, arg1, arg2) -} - // MockTokenCredentialRequestAuthenticator is a mock of TokenCredentialRequestAuthenticator interface. type MockTokenCredentialRequestAuthenticator struct { ctrl *gomock.Controller diff --git a/internal/mocks/credentialrequestmocks/generate.go b/internal/mocks/credentialrequestmocks/generate.go index c22ec978..ce19cf3f 100644 --- a/internal/mocks/credentialrequestmocks/generate.go +++ b/internal/mocks/credentialrequestmocks/generate.go @@ -1,6 +1,6 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package credentialrequestmocks -//go:generate go run -v github.com/golang/mock/mockgen -destination=credentialrequestmocks.go -package=credentialrequestmocks -copyright_file=../../../hack/header.txt go.pinniped.dev/internal/registry/credentialrequest CertIssuer,TokenCredentialRequestAuthenticator +//go:generate go run -v github.com/golang/mock/mockgen -destination=credentialrequestmocks.go -package=credentialrequestmocks -copyright_file=../../../hack/header.txt go.pinniped.dev/internal/registry/credentialrequest TokenCredentialRequestAuthenticator diff --git a/internal/mocks/issuermocks/generate.go b/internal/mocks/issuermocks/generate.go new file mode 100644 index 00000000..7d0c9937 --- /dev/null +++ b/internal/mocks/issuermocks/generate.go @@ -0,0 +1,6 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package issuermocks + +//go:generate go run -v github.com/golang/mock/mockgen -destination=issuermocks.go -package=issuermocks -copyright_file=../../../hack/header.txt go.pinniped.dev/internal/issuer ClientCertIssuer diff --git a/internal/mocks/issuermocks/issuermocks.go b/internal/mocks/issuermocks/issuermocks.go new file mode 100644 index 00000000..99dce2c5 --- /dev/null +++ b/internal/mocks/issuermocks/issuermocks.go @@ -0,0 +1,55 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: go.pinniped.dev/internal/issuer (interfaces: ClientCertIssuer) + +// Package issuermocks is a generated GoMock package. +package issuermocks + +import ( + reflect "reflect" + time "time" + + gomock "github.com/golang/mock/gomock" +) + +// MockClientCertIssuer is a mock of ClientCertIssuer interface. +type MockClientCertIssuer struct { + ctrl *gomock.Controller + recorder *MockClientCertIssuerMockRecorder +} + +// MockClientCertIssuerMockRecorder is the mock recorder for MockClientCertIssuer. +type MockClientCertIssuerMockRecorder struct { + mock *MockClientCertIssuer +} + +// NewMockClientCertIssuer creates a new mock instance. +func NewMockClientCertIssuer(ctrl *gomock.Controller) *MockClientCertIssuer { + mock := &MockClientCertIssuer{ctrl: ctrl} + mock.recorder = &MockClientCertIssuerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClientCertIssuer) EXPECT() *MockClientCertIssuerMockRecorder { + return m.recorder +} + +// IssueClientCertPEM mocks base method. +func (m *MockClientCertIssuer) IssueClientCertPEM(arg0 string, arg1 []string, arg2 time.Duration) ([]byte, []byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IssueClientCertPEM", arg0, arg1, arg2) + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].([]byte) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// IssueClientCertPEM indicates an expected call of IssueClientCertPEM. +func (mr *MockClientCertIssuerMockRecorder) IssueClientCertPEM(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssueClientCertPEM", reflect.TypeOf((*MockClientCertIssuer)(nil).IssueClientCertPEM), arg0, arg1, arg2) +} diff --git a/internal/registry/credentialrequest/rest.go b/internal/registry/credentialrequest/rest.go index a1846a40..9c9165d5 100644 --- a/internal/registry/credentialrequest/rest.go +++ b/internal/registry/credentialrequest/rest.go @@ -6,7 +6,6 @@ package credentialrequest import ( "context" - "crypto/x509/pkix" "fmt" "time" @@ -32,7 +31,7 @@ type TokenCredentialRequestAuthenticator interface { AuthenticateTokenCredentialRequest(ctx context.Context, req *loginapi.TokenCredentialRequest) (user.Info, error) } -func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer issuer.CertIssuer, resource schema.GroupResource) *REST { +func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer issuer.ClientCertIssuer, resource schema.GroupResource) *REST { return &REST{ authenticator: authenticator, issuer: issuer, @@ -42,7 +41,7 @@ func NewREST(authenticator TokenCredentialRequestAuthenticator, issuer issuer.Ce type REST struct { authenticator TokenCredentialRequestAuthenticator - issuer issuer.CertIssuer + issuer issuer.ClientCertIssuer tableConvertor rest.TableConvertor } @@ -97,30 +96,23 @@ func (r *REST) Create(ctx context.Context, obj runtime.Object, createValidation return nil, err } - user, err := r.authenticator.AuthenticateTokenCredentialRequest(ctx, credentialRequest) + userInfo, err := r.authenticator.AuthenticateTokenCredentialRequest(ctx, credentialRequest) if err != nil { traceFailureWithError(t, "token authentication", err) return failureResponse(), nil } - if user == nil || user.GetName() == "" { - traceSuccess(t, user, false) + if userInfo == nil || userInfo.GetName() == "" { + traceSuccess(t, userInfo, false) return failureResponse(), nil } - certPEM, keyPEM, err := r.issuer.IssuePEM( - pkix.Name{ - CommonName: user.GetName(), - Organization: user.GetGroups(), - }, - []string{}, - clientCertificateTTL, - ) + certPEM, keyPEM, err := r.issuer.IssueClientCertPEM(userInfo.GetName(), userInfo.GetGroups(), clientCertificateTTL) if err != nil { traceFailureWithError(t, "cert issuer", err) return failureResponse(), nil } - traceSuccess(t, user, true) + traceSuccess(t, userInfo, true) return &loginapi.TokenCredentialRequest{ Status: loginapi.TokenCredentialRequestStatus{ diff --git a/internal/registry/credentialrequest/rest_test.go b/internal/registry/credentialrequest/rest_test.go index 6b71ec20..9c05c2bb 100644 --- a/internal/registry/credentialrequest/rest_test.go +++ b/internal/registry/credentialrequest/rest_test.go @@ -5,7 +5,6 @@ package credentialrequest import ( "context" - "crypto/x509/pkix" "errors" "fmt" "testing" @@ -26,6 +25,7 @@ import ( loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/mocks/credentialrequestmocks" + "go.pinniped.dev/internal/mocks/issuermocks" "go.pinniped.dev/internal/testutil" ) @@ -89,16 +89,14 @@ func TestCreate(t *testing.T) { Groups: []string{"test-group-1", "test-group-2"}, }, nil) - issuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - issuer.EXPECT().IssuePEM( - pkix.Name{ - CommonName: "test-user", - Organization: []string{"test-group-1", "test-group-2"}}, - []string{}, + clientCertIssuer := issuermocks.NewMockClientCertIssuer(ctrl) + clientCertIssuer.EXPECT().IssueClientCertPEM( + "test-user", + []string{"test-group-1", "test-group-2"}, 5*time.Minute, ).Return([]byte("test-cert"), []byte("test-key"), nil) - storage := NewREST(requestAuthenticator, issuer, schema.GroupResource{}) + storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{}) response, err := callCreate(context.Background(), storage, req) @@ -132,12 +130,12 @@ func TestCreate(t *testing.T) { Groups: []string{"test-group-1", "test-group-2"}, }, nil) - issuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - issuer.EXPECT(). - IssuePEM(gomock.Any(), gomock.Any(), gomock.Any()). + clientCertIssuer := issuermocks.NewMockClientCertIssuer(ctrl) + clientCertIssuer.EXPECT(). + IssueClientCertPEM(gomock.Any(), gomock.Any(), gomock.Any()). Return(nil, nil, fmt.Errorf("some certificate authority error")) - storage := NewREST(requestAuthenticator, issuer, schema.GroupResource{}) + storage := NewREST(requestAuthenticator, clientCertIssuer, schema.GroupResource{}) response, err := callCreate(context.Background(), storage, req) requireSuccessfulResponseWithAuthenticationFailureMessage(t, err, response) @@ -354,12 +352,12 @@ func requireSuccessfulResponseWithAuthenticationFailureMessage(t *testing.T, err }) } -func successfulIssuer(ctrl *gomock.Controller) issuer.CertIssuer { - certIssuer := credentialrequestmocks.NewMockCertIssuer(ctrl) - certIssuer.EXPECT(). - IssuePEM(gomock.Any(), gomock.Any(), gomock.Any()). +func successfulIssuer(ctrl *gomock.Controller) issuer.ClientCertIssuer { + clientCertIssuer := issuermocks.NewMockClientCertIssuer(ctrl) + clientCertIssuer.EXPECT(). + IssueClientCertPEM(gomock.Any(), gomock.Any(), gomock.Any()). Return([]byte("test-cert"), []byte("test-key"), nil) - return certIssuer + return clientCertIssuer } func stringPtr(s string) *string { diff --git a/internal/testutil/certs.go b/internal/testutil/certs.go index 79792195..9d6b41d3 100644 --- a/internal/testutil/certs.go +++ b/internal/testutil/certs.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package testutil @@ -12,6 +12,7 @@ import ( "crypto/x509/pkix" "encoding/pem" "math/big" + "net" "testing" "time" @@ -25,10 +26,18 @@ type ValidCert struct { parsed *x509.Certificate } -// ValidateCertificate validates a certificate and provides an object for asserting properties of the certificate. -func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { +// ValidateServerCertificate validates a certificate and provides an object for asserting properties of the certificate. +func ValidateServerCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { t.Helper() + return validateCertificate(t, x509.ExtKeyUsageServerAuth, caPEM, certPEM) +} +func ValidateClientCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert { + t.Helper() + return validateCertificate(t, x509.ExtKeyUsageClientAuth, caPEM, certPEM) +} + +func validateCertificate(t *testing.T, extKeyUsage x509.ExtKeyUsage, caPEM string, certPEM string) *ValidCert { block, _ := pem.Decode([]byte(certPEM)) require.NotNil(t, block) parsed, err := x509.ParseCertificate(block.Bytes) @@ -37,7 +46,10 @@ func ValidateCertificate(t *testing.T, caPEM string, certPEM string) *ValidCert // Validate the created cert using the CA. roots := x509.NewCertPool() require.True(t, roots.AppendCertsFromPEM([]byte(caPEM))) - opts := x509.VerifyOptions{Roots: roots} + opts := x509.VerifyOptions{ + Roots: roots, + KeyUsages: []x509.ExtKeyUsage{extKeyUsage}, + } _, err = parsed.Verify(opts) require.NoError(t, err) @@ -61,6 +73,35 @@ func (v *ValidCert) RequireDNSName(expectDNSName string) { require.Contains(v.t, v.parsed.DNSNames, expectDNSName, "expected an explicit DNS SAN, not just Common Name") } +func (v *ValidCert) RequireDNSNames(names []string) { + v.t.Helper() + require.Equal(v.t, names, v.parsed.DNSNames) +} + +func (v *ValidCert) RequireEmptyDNSNames() { + v.t.Helper() + require.Empty(v.t, v.parsed.DNSNames) +} + +func (v *ValidCert) RequireIPs(ips []net.IP) { + v.t.Helper() + actualIPs := v.parsed.IPAddresses + actualIPsStrings := make([]string, len(actualIPs)) + for i := range actualIPs { + actualIPsStrings[i] = actualIPs[i].String() + } + expectedIPsStrings := make([]string, len(ips)) + for i := range ips { + expectedIPsStrings[i] = ips[i].String() + } + require.Equal(v.t, expectedIPsStrings, actualIPsStrings) +} + +func (v *ValidCert) RequireEmptyIPs() { + v.t.Helper() + require.Empty(v.t, v.parsed.IPAddresses) +} + // RequireLifetime asserts that the lifetime of the certificate matches the expected timestamps. func (v *ValidCert) RequireLifetime(expectNotBefore time.Time, expectNotAfter time.Time, delta time.Duration) { v.t.Helper() @@ -81,6 +122,11 @@ func (v *ValidCert) RequireCommonName(commonName string) { require.Equal(v.t, commonName, v.parsed.Subject.CommonName) } +func (v *ValidCert) RequireOrganizations(orgs []string) { + v.t.Helper() + require.Equal(v.t, orgs, v.parsed.Subject.Organization) +} + // CreateCertificate creates a certificate with the provided time bounds, and returns the PEM // representation of the certificate and its private key. The returned certificate is capable of // signing child certificates. diff --git a/pkg/conciergeclient/conciergeclient_test.go b/pkg/conciergeclient/conciergeclient_test.go index 162ecfee..6787cc92 100644 --- a/pkg/conciergeclient/conciergeclient_test.go +++ b/pkg/conciergeclient/conciergeclient_test.go @@ -5,7 +5,6 @@ package conciergeclient import ( "context" - "crypto/x509/pkix" "encoding/base64" "encoding/json" "fmt" @@ -26,7 +25,7 @@ import ( func TestNew(t *testing.T) { t.Parallel() - testCA, err := certauthority.New(pkix.Name{}, 1*time.Hour) + testCA, err := certauthority.New("Test CA", 1*time.Hour) require.NoError(t, err) tests := []struct { diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index dfa4e2a0..b382d85f 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -6,7 +6,6 @@ import ( "bufio" "bytes" "context" - "crypto/x509/pkix" "encoding/base64" "errors" "fmt" @@ -69,7 +68,7 @@ func TestE2EFullIntegration(t *testing.T) { // Generate a CA bundle with which to serve this provider. t.Logf("generating test CA") - ca, err := certauthority.New(pkix.Name{CommonName: "Downstream Test CA"}, 1*time.Hour) + ca, err := certauthority.New("Downstream Test CA", 1*time.Hour) require.NoError(t, err) // Save that bundle plus the one that signs the upstream issuer, for test purposes. @@ -80,12 +79,7 @@ func TestE2EFullIntegration(t *testing.T) { // Use the CA to issue a TLS server cert. t.Logf("issuing test certificate") - tlsCert, err := ca.Issue( - pkix.Name{CommonName: issuerURL.Hostname()}, - []string{issuerURL.Hostname()}, - nil, - 1*time.Hour, - ) + tlsCert, err := ca.IssueServerCert([]string{issuerURL.Hostname()}, nil, 1*time.Hour) require.NoError(t, err) certPEM, keyPEM, err := certauthority.ToPEM(tlsCert) require.NoError(t, err) diff --git a/test/integration/supervisor_discovery_test.go b/test/integration/supervisor_discovery_test.go index 8712a316..d076f828 100644 --- a/test/integration/supervisor_discovery_test.go +++ b/test/integration/supervisor_discovery_test.go @@ -7,7 +7,6 @@ import ( "context" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/json" "fmt" "io/ioutil" @@ -288,11 +287,11 @@ func defaultTLSCertSecretName(env *library.TestEnv) string { func createTLSCertificateSecret(ctx context.Context, t *testing.T, ns string, hostname string, ips []net.IP, secretName string, kubeClient kubernetes.Interface) *certauthority.CA { // Create a CA. - ca, err := certauthority.New(pkix.Name{CommonName: "Acme Corp"}, 1000*time.Hour) + ca, err := certauthority.New("Acme Corp", 1000*time.Hour) require.NoError(t, err) // Using the CA, create a TLS server cert. - tlsCert, err := ca.Issue(pkix.Name{CommonName: hostname}, []string{hostname}, ips, 1000*time.Hour) + tlsCert, err := ca.IssueServerCert([]string{hostname}, ips, 1000*time.Hour) require.NoError(t, err) // Write the serving cert to the SNI secret. diff --git a/test/integration/supervisor_login_test.go b/test/integration/supervisor_login_test.go index 9d6b6787..fe2060ea 100644 --- a/test/integration/supervisor_login_test.go +++ b/test/integration/supervisor_login_test.go @@ -6,7 +6,6 @@ package integration import ( "context" "crypto/tls" - "crypto/x509/pkix" "encoding/base64" "encoding/json" "fmt" @@ -58,7 +57,7 @@ func TestSupervisorLogin(t *testing.T) { // Generate a CA bundle with which to serve this provider. t.Logf("generating test CA") - ca, err := certauthority.New(pkix.Name{CommonName: "Downstream Test CA"}, 1*time.Hour) + ca, err := certauthority.New("Downstream Test CA", 1*time.Hour) require.NoError(t, err) // Create an HTTP client that can reach the downstream discovery endpoint using the CA certs. @@ -85,12 +84,7 @@ func TestSupervisorLogin(t *testing.T) { // Use the CA to issue a TLS server cert. t.Logf("issuing test certificate") - tlsCert, err := ca.Issue( - pkix.Name{CommonName: issuerURL.Hostname()}, - []string{issuerURL.Hostname()}, - nil, - 1*time.Hour, - ) + tlsCert, err := ca.IssueServerCert([]string{issuerURL.Hostname()}, nil, 1*time.Hour) require.NoError(t, err) certPEM, keyPEM, err := certauthority.ToPEM(tlsCert) require.NoError(t, err) From b530cef3b1b95c2d815b107f7237d7a2aa8d59aa Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Sat, 13 Mar 2021 20:25:23 -0500 Subject: [PATCH 156/203] impersonator: encode proper API status on failure Signed-off-by: Monis Khan --- .../concierge/impersonator/impersonator.go | 27 ++++++++++++--- .../impersonator/impersonator_test.go | 33 ++++++++++++++----- .../concierge_impersonation_proxy_test.go | 21 +++++++++--- 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 87f2d1b4..aae77bee 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -12,14 +12,18 @@ import ( "strings" "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" + genericapirequest "k8s.io/apiserver/pkg/endpoints/request" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/apiserver/pkg/server/filters" @@ -227,7 +231,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi "url", r.URL.String(), "method", r.Method, ) - http.Error(w, "invalid authorization header", http.StatusInternalServerError) + newInternalErrResponse(w, r, c.Serializer, "invalid authorization header") return } @@ -237,7 +241,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi "url", r.URL.String(), "method", r.Method, ) - http.Error(w, "invalid impersonation", http.StatusInternalServerError) + newInternalErrResponse(w, r, c.Serializer, "invalid impersonation") return } @@ -247,7 +251,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi "url", r.URL.String(), "method", r.Method, ) - http.Error(w, "invalid user", http.StatusInternalServerError) + newInternalErrResponse(w, r, c.Serializer, "invalid user") return } @@ -257,7 +261,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi "url", r.URL.String(), "method", r.Method, ) - http.Error(w, "unable to act as user", http.StatusUnprocessableEntity) + newInternalErrResponse(w, r, c.Serializer, "unimplemented functionality - unable to act as current user") return } @@ -309,3 +313,18 @@ func getTransportForUser(userInfo user.Info, delegate http.RoundTripper) (http.R // 8. the above would be safe even if in the future Kube started supporting UIDs asserted by client certs return nil, constable.Error("unexpected uid") } + +func newInternalErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, msg string) { + newStatusErrResponse(w, r, s, apierrors.NewInternalError(constable.Error(msg))) +} + +func newStatusErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, err *apierrors.StatusError) { + requestInfo, ok := genericapirequest.RequestInfoFrom(r.Context()) + if !ok { + responsewriters.InternalError(w, r, constable.Error("no RequestInfo found in the context")) + return + } + + gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} + responsewriters.ErrorNegotiated(err, s, gv, w, r) +} diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 66985781..d08cbbca 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -16,9 +16,12 @@ import ( "github.com/stretchr/testify/require" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" + genericapiserver "k8s.io/apiserver/pkg/server" genericoptions "k8s.io/apiserver/pkg/server/options" utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/client-go/rest" @@ -284,6 +287,12 @@ func TestImpersonatorHTTPHandler(t *testing.T) { r, err := http.NewRequestWithContext(ctx, http.MethodGet, validURL.String(), nil) require.NoError(t, err) r.Header = h + reqInfo := &request.RequestInfo{ + IsResourceRequest: false, + Path: validURL.Path, + Verb: "get", + } + r = r.WithContext(request.WithRequestInfo(ctx, reqInfo)) return r } @@ -324,44 +333,44 @@ func TestImpersonatorHTTPHandler(t *testing.T) { { name: "Impersonate-User header already in request", request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}, nil), - wantHTTPBody: "invalid impersonation\n", + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-Group header already in request", request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}, nil), - wantHTTPBody: "invalid impersonation\n", + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-Extra header already in request", request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}, nil), - wantHTTPBody: "invalid impersonation\n", + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-* header already in request", request: newRequest(map[string][]string{"Impersonate-Something": {"some-newfangled-impersonate-header"}}, nil), - wantHTTPBody: "invalid impersonation\n", + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "unexpected authorization header", request: newRequest(map[string][]string{"Authorization": {"panda"}}, nil), - wantHTTPBody: "invalid authorization header\n", + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid authorization header","reason":"InternalError","details":{"causes":[{"message":"invalid authorization header"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "missing user", request: newRequest(map[string][]string{}, nil), - wantHTTPBody: "invalid user\n", + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid user","reason":"InternalError","details":{"causes":[{"message":"invalid user"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "unexpected UID", request: newRequest(map[string][]string{}, &user.DefaultInfo{UID: "007"}), - wantHTTPBody: "unable to act as user\n", - wantHTTPStatus: http.StatusUnprocessableEntity, + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: unimplemented functionality - unable to act as current user","reason":"InternalError","details":{"causes":[{"message":"unimplemented functionality - unable to act as current user"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, }, // happy path { @@ -458,9 +467,15 @@ func TestImpersonatorHTTPHandler(t *testing.T) { require.NoError(t, err) require.NotNil(t, impersonatorHTTPHandlerFunc) + // this is not a valid way to get a server config, but it is good enough for a unit test + scheme := runtime.NewScheme() + metav1.AddToGroupVersion(scheme, metav1.Unversioned) + codecs := serializer.NewCodecFactory(scheme) + serverConfig := genericapiserver.NewRecommendedConfig(codecs) + w := httptest.NewRecorder() requestBeforeServe := tt.request.Clone(tt.request.Context()) - impersonatorHTTPHandlerFunc(nil).ServeHTTP(w, tt.request) + impersonatorHTTPHandlerFunc(&serverConfig.Config).ServeHTTP(w, tt.request) require.Equal(t, requestBeforeServe, tt.request, "ServeHTTP() mutated the request, and it should not per http.Handler docs") if tt.wantHTTPStatus != 0 { diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 6132ffaf..8d9cfa73 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -475,10 +475,23 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge _, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) - require.Error(t, err) - // The server checks that we have a UID in the request and rejects it with a 422 Unprocessable Entity. - // The API machinery turns 422's into this error text... - require.Contains(t, err.Error(), "the server rejected our request due to an error in our request") + require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") + require.True(t, k8serrors.IsInternalError(err), err) + require.Equal(t, &k8serrors.StatusError{ + ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusInternalServerError, + Reason: metav1.StatusReasonInternalError, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + { + Message: "unimplemented functionality - unable to act as current user", + }, + }, + }, + Message: "Internal error occurred: unimplemented functionality - unable to act as current user", + }, + }, err) }) t.Run("kubectl as a client", func(t *testing.T) { From 4c162be8bf3d22f8ea24e643d0054d17b5bd1136 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Mon, 15 Mar 2021 09:43:06 -0400 Subject: [PATCH 157/203] impersonator: add comment about long running func Signed-off-by: Monis Khan --- internal/concierge/impersonator/impersonator.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index aae77bee..117909a4 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -129,6 +129,10 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. serverConfig.LoopbackClientConfig.BearerToken = "" // match KAS exactly since our long running operations are just a proxy to it + // this must be kept in sync with github.com/kubernetes/kubernetes/cmd/kube-apiserver/app/server.go + // this is nothing to stress about - it has not changed since the beginning of Kube: + // v1.6 no-op move away from regex to request info https://github.com/kubernetes/kubernetes/pull/38119 + // v1.1 added pods/attach to the list https://github.com/kubernetes/kubernetes/pull/13705 serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck( sets.NewString("watch", "proxy"), sets.NewString("attach", "exec", "proxy", "log", "portforward"), From 00694c9cb662b50340974352e146f2da183bec4a Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Mon, 15 Mar 2021 12:24:07 -0400 Subject: [PATCH 158/203] dynamiccert: split into serving cert and CA providers Signed-off-by: Monis Khan --- cmd/local-user-authenticator/main.go | 10 ++-- cmd/local-user-authenticator/main_test.go | 4 +- internal/certauthority/certauthority.go | 11 ++++- .../dynamiccertauthority.go | 22 ++++++--- .../dynamiccertauthority_test.go | 5 +- .../impersonator/impersonator_test.go | 4 +- internal/concierge/server/server.go | 8 ++-- .../controller/apicerts/certs_observer.go | 4 +- .../apicerts/certs_observer_test.go | 21 +++++++-- .../impersonatorconfig/impersonator_config.go | 4 +- .../impersonator_config_test.go | 2 +- internal/controller/kubecertagent/execer.go | 4 +- .../controller/kubecertagent/execer_test.go | 2 +- .../controllermanager/prepare_controllers.go | 4 +- internal/dynamiccert/provider.go | 46 +++++++++++++++++-- internal/issuer/issuer.go | 18 +++++++- internal/mocks/issuermocks/issuermocks.go | 14 ++++++ 17 files changed, 141 insertions(+), 42 deletions(-) diff --git a/cmd/local-user-authenticator/main.go b/cmd/local-user-authenticator/main.go index f4f2249d..7c1833ff 100644 --- a/cmd/local-user-authenticator/main.go +++ b/cmd/local-user-authenticator/main.go @@ -54,12 +54,12 @@ const ( ) type webhook struct { - certProvider dynamiccert.Provider + certProvider dynamiccert.Private secretInformer corev1informers.SecretInformer } func newWebhook( - certProvider dynamiccert.Provider, + certProvider dynamiccert.Private, secretInformer corev1informers.SecretInformer, ) *webhook { return &webhook{ @@ -281,7 +281,7 @@ func respondWithAuthenticated( func startControllers( ctx context.Context, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, kubeClient kubernetes.Interface, kubeInformers kubeinformers.SharedInformerFactory, ) { @@ -328,7 +328,7 @@ func startControllers( func startWebhook( ctx context.Context, l net.Listener, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, secretInformer corev1informers.SecretInformer, ) error { return newWebhook(dynamicCertProvider, secretInformer).start(ctx, l) @@ -355,7 +355,7 @@ func run() error { kubeinformers.WithNamespace(namespace), ) - dynamicCertProvider := dynamiccert.New("local-user-authenticator-tls-serving-certificate") + dynamicCertProvider := dynamiccert.NewServingCert("local-user-authenticator-tls-serving-certificate") startControllers(ctx, dynamicCertProvider, client.Kubernetes, kubeInformers) plog.Debug("controllers are ready") diff --git a/cmd/local-user-authenticator/main_test.go b/cmd/local-user-authenticator/main_test.go index 5a6fb0bd..cee265ee 100644 --- a/cmd/local-user-authenticator/main_test.go +++ b/cmd/local-user-authenticator/main_test.go @@ -458,7 +458,7 @@ func createSecretInformer(ctx context.Context, t *testing.T, kubeClient kubernet // newClientProvider returns a dynamiccert.Provider configured // with valid serving cert, the CA bundle that can be used to verify the serving // cert, and the server name that can be used to verify the TLS peer. -func newCertProvider(t *testing.T) (dynamiccert.Provider, []byte, string) { +func newCertProvider(t *testing.T) (dynamiccert.Private, []byte, string) { t.Helper() serverName := "local-user-authenticator" @@ -472,7 +472,7 @@ func newCertProvider(t *testing.T) (dynamiccert.Provider, []byte, string) { certPEM, keyPEM, err := certauthority.ToPEM(cert) require.NoError(t, err) - certProvider := dynamiccert.New(t.Name()) + certProvider := dynamiccert.NewServingCert(t.Name()) err = certProvider.SetCertKeyContent(certPEM, keyPEM) require.NoError(t, err) diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index b64bf0fa..60aed89f 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -19,6 +19,8 @@ import ( "math/big" "net" "time" + + "go.pinniped.dev/internal/constable" ) // certBackdate is the amount of time before time.Now() that will be used to set @@ -71,7 +73,7 @@ func secureEnv() env { } // ErrInvalidCACertificate is returned when the contents of the loaded CA certificate do not meet our assumptions. -var ErrInvalidCACertificate = fmt.Errorf("invalid CA certificate") +const ErrInvalidCACertificate = constable.Error("invalid CA certificate") // Load a certificate authority from an existing certificate and private key (in PEM format). func Load(certPEM string, keyPEM string) (*CA, error) { @@ -82,6 +84,13 @@ func Load(certPEM string, keyPEM string) (*CA, error) { if certCount := len(cert.Certificate); certCount != 1 { return nil, fmt.Errorf("%w: expected a single certificate, found %d certificates", ErrInvalidCACertificate, certCount) } + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return nil, fmt.Errorf("failed to parse key pair as x509 cert: %w", err) + } + if !x509Cert.IsCA { + return nil, fmt.Errorf("%w: passed in key pair is not a CA", ErrInvalidCACertificate) + } return &CA{ caCertBytes: cert.Certificate[0], signer: cert.PrivateKey.(crypto.Signer), diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go index 78eb97c6..42dbf7d5 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority.go @@ -11,25 +11,33 @@ import ( "k8s.io/apiserver/pkg/server/dynamiccertificates" "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/issuer" ) -// CA is a type capable of issuing certificates. -type CA struct { +// ca is a type capable of issuing certificates. +type ca struct { provider dynamiccertificates.CertKeyContentProvider } -// New creates a new CA, ready to issue certs whenever the provided provider has a keypair to -// provide. -func New(provider dynamiccertificates.CertKeyContentProvider) *CA { - return &CA{ +// New creates a ClientCertIssuer, ready to issue certs whenever +// the given CertKeyContentProvider has a keypair to provide. +func New(provider dynamiccertificates.CertKeyContentProvider) issuer.ClientCertIssuer { + return &ca{ provider: provider, } } +func (c *ca) Name() string { + return c.provider.Name() +} + // IssueClientCertPEM issues a new client certificate for the given identity and duration, returning it as a // pair of PEM-formatted byte slices for the certificate and private key. -func (c *CA) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { +func (c *ca) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { caCrtPEM, caKeyPEM := c.provider.CurrentCertKeyContent() + // in the future we could split dynamiccert.Private into two interfaces (Private and PrivateRead) + // and have this code take PrivateRead as input. We would then add ourselves as a listener to + // the PrivateRead. This would allow us to only reload the CA contents when they actually change. ca, err := certauthority.Load(string(caCrtPEM), string(caKeyPEM)) if err != nil { return nil, nil, err diff --git a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go index 47bc6539..b33cb9dd 100644 --- a/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go +++ b/internal/certauthority/dynamiccertauthority/dynamiccertauthority_test.go @@ -10,13 +10,14 @@ import ( "github.com/stretchr/testify/require" "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/issuer" "go.pinniped.dev/internal/testutil" ) func TestCAIssuePEM(t *testing.T) { t.Parallel() - provider := dynamiccert.New(t.Name()) + provider := dynamiccert.NewCA(t.Name()) ca := New(provider) goodCACrtPEM0, goodCAKeyPEM0, err := testutil.CreateCertificate( @@ -115,7 +116,7 @@ func TestCAIssuePEM(t *testing.T) { } } -func issuePEM(provider dynamiccert.Provider, ca *CA, caCrt, caKey []byte) ([]byte, []byte, error) { +func issuePEM(provider dynamiccert.Provider, ca issuer.ClientCertIssuer, caCrt, caKey []byte) ([]byte, []byte, error) { // if setting fails, look at that error if caCrt != nil || caKey != nil { if err := provider.SetCertKeyContent(caCrt, caKey); err != nil { diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index d08cbbca..610bad3f 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -42,13 +42,13 @@ func TestImpersonator(t *testing.T) { require.NoError(t, err) caKey, err := ca.PrivateKeyToPEM() require.NoError(t, err) - caContent := dynamiccert.New("ca") + caContent := dynamiccert.NewCA("ca") err = caContent.SetCertKeyContent(ca.Bundle(), caKey) require.NoError(t, err) cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) require.NoError(t, err) - certKeyContent := dynamiccert.New("cert-key") + certKeyContent := dynamiccert.NewServingCert("cert-key") err = certKeyContent.SetCertKeyContent(cert, key) require.NoError(t, err) diff --git a/internal/concierge/server/server.go b/internal/concierge/server/server.go index 4a4415a5..d5e51379 100644 --- a/internal/concierge/server/server.go +++ b/internal/concierge/server/server.go @@ -114,15 +114,15 @@ func (a *App) runServer(ctx context.Context) error { // is stored in a k8s Secret. Therefore it also effectively acting as // an in-memory cache of what is stored in the k8s Secret, helping to // keep incoming requests fast. - dynamicServingCertProvider := dynamiccert.New("concierge-serving-cert") + dynamicServingCertProvider := dynamiccert.NewServingCert("concierge-serving-cert") // This cert provider will be used to provide the Kube signing key to the // cert issuer used to issue certs to Pinniped clients wishing to login. - dynamicSigningCertProvider := dynamiccert.New("concierge-kube-signing-cert") + dynamicSigningCertProvider := dynamiccert.NewCA("concierge-kube-signing-cert") // This cert provider will be used to provide the impersonation proxy signing key to the // cert issuer used to issue certs to Pinniped clients wishing to login. - impersonationProxySigningCertProvider := dynamiccert.New("impersonation-proxy-signing-cert") + impersonationProxySigningCertProvider := dynamiccert.NewCA("impersonation-proxy-signing-cert") // Get the "real" name of the login concierge API group (i.e., the API group name with the // injected suffix). @@ -182,7 +182,7 @@ func (a *App) runServer(ctx context.Context) error { // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, authenticator credentialrequest.TokenCredentialRequestAuthenticator, issuer issuer.ClientCertIssuer, startControllersPostStartHook func(context.Context), diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go index c8b6f2b8..f01312d5 100644 --- a/internal/controller/apicerts/certs_observer.go +++ b/internal/controller/apicerts/certs_observer.go @@ -18,14 +18,14 @@ import ( type certsObserverController struct { namespace string certsSecretResourceName string - dynamicCertProvider dynamiccert.Provider + dynamicCertProvider dynamiccert.Private secretInformer corev1informers.SecretInformer } func NewCertsObserverController( namespace string, certsSecretResourceName string, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, secretInformer corev1informers.SecretInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, ) controllerlib.Controller { diff --git a/internal/controller/apicerts/certs_observer_test.go b/internal/controller/apicerts/certs_observer_test.go index 553e66f3..93de9801 100644 --- a/internal/controller/apicerts/certs_observer_test.go +++ b/internal/controller/apicerts/certs_observer_test.go @@ -17,6 +17,7 @@ import ( kubeinformers "k8s.io/client-go/informers" kubernetesfake "k8s.io/client-go/kubernetes/fake" + "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/controllerlib" "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/testutil" @@ -109,7 +110,7 @@ func TestObserverControllerSync(t *testing.T) { var cancelContext context.Context var cancelContextCancelFunc context.CancelFunc var syncContext *controllerlib.Context - var dynamicCertProvider dynamiccert.Provider + var dynamicCertProvider dynamiccert.Private // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. @@ -145,7 +146,7 @@ func TestObserverControllerSync(t *testing.T) { kubeInformerClient = kubernetesfake.NewSimpleClientset() kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) - dynamicCertProvider = dynamiccert.New(name) + dynamicCertProvider = dynamiccert.NewServingCert(name) }) it.After(func() { @@ -163,12 +164,18 @@ func TestObserverControllerSync(t *testing.T) { err := kubeInformerClient.Tracker().Add(unrelatedSecret) r.NoError(err) - crt, key, err := testutil.CreateCertificate( + caCrt, caKey, err := testutil.CreateCertificate( time.Now().Add(-time.Hour), time.Now().Add(time.Hour), ) require.NoError(t, err) + ca, err := certauthority.Load(string(caCrt), string(caKey)) + require.NoError(t, err) + + crt, key, err := ca.IssueServerCertPEM(nil, nil, time.Hour) + require.NoError(t, err) + err = dynamicCertProvider.SetCertKeyContent(crt, key) r.NoError(err) }) @@ -186,12 +193,18 @@ func TestObserverControllerSync(t *testing.T) { when("there is a serving cert Secret with the expected keys already in the installation namespace", func() { it.Before(func() { - crt, key, err := testutil.CreateCertificate( + caCrt, caKey, err := testutil.CreateCertificate( time.Now().Add(-time.Hour), time.Now().Add(time.Hour), ) require.NoError(t, err) + ca, err := certauthority.Load(string(caCrt), string(caKey)) + require.NoError(t, err) + + crt, key, err := ca.IssueServerCertPEM(nil, nil, time.Hour) + require.NoError(t, err) + apiServingCertSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: certsSecretResourceName, diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 1733d9da..4fc140d8 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -72,7 +72,7 @@ type impersonatorConfigController struct { hasControlPlaneNodes *bool serverStopCh chan struct{} errorCh chan error - tlsServingCertDynamicCertProvider dynamiccert.Provider + tlsServingCertDynamicCertProvider dynamiccert.Private } func NewImpersonatorConfigController( @@ -116,7 +116,7 @@ func NewImpersonatorConfigController( clock: clock, impersonationSigningCertProvider: impersonationSigningCertProvider, impersonatorFunc: impersonatorFunc, - tlsServingCertDynamicCertProvider: dynamiccert.New("impersonation-proxy-serving-cert"), + tlsServingCertDynamicCertProvider: dynamiccert.NewServingCert("impersonation-proxy-serving-cert"), }, }, withInformer( diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index f4996184..d785d67f 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -974,7 +974,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { kubeAPIClient = kubernetesfake.NewSimpleClientset() pinnipedAPIClient = pinnipedfake.NewSimpleClientset() frozenNow = time.Date(2021, time.March, 2, 7, 42, 0, 0, time.Local) - signingCertProvider = dynamiccert.New(name) + signingCertProvider = dynamiccert.NewCA(name) ca := newCA() signingCACertPEM = ca.Bundle() diff --git a/internal/controller/kubecertagent/execer.go b/internal/controller/kubecertagent/execer.go index 691af1c4..e021ff26 100644 --- a/internal/controller/kubecertagent/execer.go +++ b/internal/controller/kubecertagent/execer.go @@ -33,7 +33,7 @@ type execerController struct { credentialIssuerLocationConfig *CredentialIssuerLocationConfig credentialIssuerLabels map[string]string discoveryURLOverride *string - dynamicCertProvider dynamiccert.Provider + dynamicCertProvider dynamiccert.Private podCommandExecutor PodCommandExecutor clock clock.Clock pinnipedAPIClient pinnipedclientset.Interface @@ -51,7 +51,7 @@ func NewExecerController( credentialIssuerLocationConfig *CredentialIssuerLocationConfig, credentialIssuerLabels map[string]string, discoveryURLOverride *string, - dynamicCertProvider dynamiccert.Provider, + dynamicCertProvider dynamiccert.Private, podCommandExecutor PodCommandExecutor, pinnipedAPIClient pinnipedclientset.Interface, clock clock.Clock, diff --git a/internal/controller/kubecertagent/execer_test.go b/internal/controller/kubecertagent/execer_test.go index fe19d198..c9441010 100644 --- a/internal/controller/kubecertagent/execer_test.go +++ b/internal/controller/kubecertagent/execer_test.go @@ -243,7 +243,7 @@ func TestManagerControllerSync(t *testing.T) { kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClientset, 0) fakeExecutor = &fakePodExecutor{r: r} frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - dynamicCertProvider = dynamiccert.New(name) + dynamicCertProvider = dynamiccert.NewCA(name) err = dynamicCertProvider.SetCertKeyContent([]byte(defaultDynamicCertProviderCert), []byte(defaultDynamicCertProviderKey)) r.NoError(err) diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index aef64f28..6fe0b212 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -63,12 +63,12 @@ type Config struct { DiscoveryURLOverride *string // DynamicServingCertProvider provides a setter and a getter to the Pinniped API's serving cert. - DynamicServingCertProvider dynamiccert.Provider + DynamicServingCertProvider dynamiccert.Private // DynamicSigningCertProvider provides a setter and a getter to the Pinniped API's // signing cert, i.e., the cert that it uses to sign certs for Pinniped clients wishing to login. // This is filled with the Kube API server's signing cert by a controller, if it can be found. - DynamicSigningCertProvider dynamiccert.Provider + DynamicSigningCertProvider dynamiccert.Private // ImpersonationSigningCertProvider provides a setter and a getter to the CA cert that should be // used to sign client certs for authentication to the impersonation proxy. This CA is used by diff --git a/internal/dynamiccert/provider.go b/internal/dynamiccert/provider.go index 8cef78bd..4e74d118 100644 --- a/internal/dynamiccert/provider.go +++ b/internal/dynamiccert/provider.go @@ -10,6 +10,8 @@ import ( "sync" "k8s.io/apiserver/pkg/server/dynamiccertificates" + + "go.pinniped.dev/internal/plog" ) type Provider interface { @@ -36,8 +38,12 @@ type notifier interface { dynamiccertificates.ControllerRunner // we do not need this today, but it could grow and change in the future } +var _ Provider = &provider{} + type provider struct { + // these fields are constant after struct initialization and thus do not need locking name string + isCA bool // mutex guards all the fields below it mutex sync.RWMutex @@ -46,13 +52,20 @@ type provider struct { listeners []dynamiccertificates.Listener } -// New returns an empty Provider. The returned Provider is thread-safe. -func New(name string) Provider { +// NewServingCert returns a Private that is go routine safe. +// It can only hold key pairs that have IsCA=false. +func NewServingCert(name string) Private { return &provider{name: name} } +// NewCA returns a Provider that is go routine safe. +// It can only hold key pairs that have IsCA=true. +func NewCA(name string) Provider { + return &provider{name: name, isCA: true} +} + func (p *provider) Name() string { - return p.name // constant after struct initialization and thus does not need locking + return p.name } func (p *provider) CurrentCertKeyContent() (cert []byte, key []byte) { @@ -65,10 +78,25 @@ func (p *provider) CurrentCertKeyContent() (cert []byte, key []byte) { func (p *provider) SetCertKeyContent(certPEM, keyPEM []byte) error { // always make sure that we have valid PEM data, otherwise // dynamiccertificates.NewUnionCAContentProvider.VerifyOptions will panic - if _, err := tls.X509KeyPair(certPEM, keyPEM); err != nil { + cert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { return fmt.Errorf("%s: attempt to set invalid key pair: %w", p.name, err) } + // these checks should always pass if tls.X509KeyPair did not error + if len(cert.Certificate) == 0 { + return fmt.Errorf("%s: key pair has empty cert slice", p.name) + } + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return fmt.Errorf("%s: failed to parse key pair as x509 cert: %w", p.name, err) + } + + // confirm that we are not trying to use a CA as a serving cert and vice versa + if p.isCA != x509Cert.IsCA { + return fmt.Errorf("%s: attempt to set x509 cert with unexpected IsCA=%v", p.name, x509Cert.IsCA) + } + p.setCertKeyContent(certPEM, keyPEM) return nil @@ -85,17 +113,27 @@ func (p *provider) setCertKeyContent(certPEM, keyPEM []byte) { p.certPEM = certPEM p.keyPEM = keyPEM + // technically this only reads a read lock but we already have the write lock for _, listener := range p.listeners { listener.Enqueue() } } func (p *provider) CurrentCABundleContent() []byte { + if !p.isCA { + panic("*provider from NewServingCert was cast into wrong CA interface") + } + ca, _ := p.CurrentCertKeyContent() return ca } func (p *provider) VerifyOptions() (x509.VerifyOptions, bool) { + if !p.isCA { + panic("*provider from NewServingCert was cast into wrong CA interface") + } + + plog.Warning("unexpected call to *provider.VerifyOptions; CA union logic is broken") return x509.VerifyOptions{}, false // assume we are unioned via dynamiccertificates.NewUnionCAContentProvider } diff --git a/internal/issuer/issuer.go b/internal/issuer/issuer.go index 88ac1538..ef563e99 100644 --- a/internal/issuer/issuer.go +++ b/internal/issuer/issuer.go @@ -4,6 +4,8 @@ package issuer import ( + "fmt" + "strings" "time" "k8s.io/apimachinery/pkg/util/errors" @@ -14,6 +16,7 @@ import ( const defaultCertIssuerErr = constable.Error("failed to issue cert") type ClientCertIssuer interface { + Name() string IssueClientCertPEM(username string, groups []string, ttl time.Duration) (certPEM, keyPEM []byte, err error) } @@ -21,13 +24,26 @@ var _ ClientCertIssuer = ClientCertIssuers{} type ClientCertIssuers []ClientCertIssuer +func (c ClientCertIssuers) Name() string { + if len(c) == 0 { + return "empty-client-cert-issuers" + } + + names := make([]string, 0, len(c)) + for _, issuer := range c { + names = append(names, issuer.Name()) + } + + return strings.Join(names, ",") +} + func (c ClientCertIssuers) IssueClientCertPEM(username string, groups []string, ttl time.Duration) ([]byte, []byte, error) { var errs []error for _, issuer := range c { certPEM, keyPEM, err := issuer.IssueClientCertPEM(username, groups, ttl) if err != nil { - errs = append(errs, err) + errs = append(errs, fmt.Errorf("%s failed to issue client cert: %w", issuer.Name(), err)) continue } return certPEM, keyPEM, nil diff --git a/internal/mocks/issuermocks/issuermocks.go b/internal/mocks/issuermocks/issuermocks.go index 99dce2c5..045fbed3 100644 --- a/internal/mocks/issuermocks/issuermocks.go +++ b/internal/mocks/issuermocks/issuermocks.go @@ -53,3 +53,17 @@ func (mr *MockClientCertIssuerMockRecorder) IssueClientCertPEM(arg0, arg1, arg2 mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IssueClientCertPEM", reflect.TypeOf((*MockClientCertIssuer)(nil).IssueClientCertPEM), arg0, arg1, arg2) } + +// Name mocks base method. +func (m *MockClientCertIssuer) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockClientCertIssuerMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockClientCertIssuer)(nil).Name)) +} From e22ad6171a8827d29c3a3a5996fec791cc818a2f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 15 Mar 2021 09:37:19 -0700 Subject: [PATCH 159/203] Fix a race detector warning by re-declaring `err` in a t.Cleanup() --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 8d9cfa73..433e6045 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -163,7 +163,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Delete any version that was created by this test. t.Logf("cleaning up configmap at end of test %s", impersonationProxyConfigMapName(env)) - err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{}) + err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Delete(ctx, impersonationProxyConfigMapName(env), metav1.DeleteOptions{}) if !k8serrors.IsNotFound(err) { require.NoError(t, err) // only not found errors are acceptable } From 8065a8d2e61fc931504e8c92044ba7b6162b4d68 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 15 Mar 2021 11:42:57 -0700 Subject: [PATCH 160/203] TestKubeCertAgent waits for CredentialIssuer strategy to be successful At the end of the test, wait for the KubeClusterSigningCertificate strategy on the CredentialIssuer to go back to being healthy, to avoid polluting other integration tests which follow this one. --- .../concierge_impersonation_proxy_test.go | 18 ++++++------- .../concierge_kubecertagent_test.go | 25 +++++++++++++++++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 433e6045..14b38fc1 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -39,7 +39,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/yaml" - "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" @@ -1008,7 +1008,7 @@ func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *librar } for _, strategy := range credentialIssuer.Status.Strategies { // There will be other strategy types in the list, so ignore those. - if strategy.Type == v1alpha1.ImpersonationProxyStrategyType && strategy.Status == v1alpha1.SuccessStrategyStatus { //nolint:nestif + if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif if strategy.Frontend == nil { return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test } @@ -1021,10 +1021,10 @@ func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *librar return false, err // unexpected, fail the test } return true, nil // found it, continue the test! - } else if strategy.Type == v1alpha1.ImpersonationProxyStrategyType { + } else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType { t.Logf("Waiting for successful impersonation proxy strategy on %s: found status %s with reason %s and message: %s", credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) - if strategy.Reason == v1alpha1.ErrorDuringSetupStrategyReason { + if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason { // The server encountered an unexpected error while starting the impersonator, so fail the test fast. return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) } @@ -1049,14 +1049,14 @@ func requireDisabledByConfigurationStrategy(ctx context.Context, t *testing.T, e } for _, strategy := range credentialIssuer.Status.Strategies { // There will be other strategy types in the list, so ignore those. - if strategy.Type == v1alpha1.ImpersonationProxyStrategyType && - strategy.Status == v1alpha1.ErrorStrategyStatus && - strategy.Reason == v1alpha1.DisabledStrategyReason { //nolint:nestif + if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && + strategy.Status == conciergev1alpha.ErrorStrategyStatus && + strategy.Reason == conciergev1alpha.DisabledStrategyReason { //nolint:nestif return true, nil // found it, continue the test! - } else if strategy.Type == v1alpha1.ImpersonationProxyStrategyType { + } else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType { t.Logf("Waiting for disabled impersonation proxy strategy on %s: found status %s with reason %s and message: %s", credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) - if strategy.Reason == v1alpha1.ErrorDuringSetupStrategyReason { + if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason { // The server encountered an unexpected error while stopping the impersonator, so fail the test fast. return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) } diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 73959bf8..78cfad98 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/wait" + conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" "go.pinniped.dev/test/library" ) @@ -127,6 +128,30 @@ func TestKubeCertAgent(t *testing.T) { assert.Eventually(t, agentPodsReconciled, 10*time.Second, 250*time.Millisecond) require.NoError(t, err) }) + + // Because the above tests have purposefully put the kube cert issuer strategy into a broken + // state, wait for it to become healthy again before moving on to other integration tests, + // otherwise those tests would be polluted by this test and would have to wait for the + // strategy to become successful again. + library.RequireEventuallyWithoutError(t, func() (bool, error) { + adminConciergeClient := library.NewConciergeClientset(t) + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil || credentialIssuer.Status.Strategies == nil { + t.Log("Did not find any CredentialIssuer with any strategies") + return false, nil // didn't find it, but keep trying + } + for _, strategy := range credentialIssuer.Status.Strategies { + // There will be other strategy types in the list, so ignore those. + if strategy.Type == conciergev1alpha.KubeClusterSigningCertificateStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif + if strategy.Frontend == nil { + return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test + } + return true, nil // found it, continue the test! + } + } + t.Log("Did not find any successful KubeClusterSigningCertificate strategy on CredentialIssuer") + return false, nil // didn't find it, but keep trying + }, 3*time.Minute, 3*time.Second) } func ensureKubeCertAgentSteadyState(t *testing.T, agentPodsReconciled func() bool) { From 4f671f5dca40d54382c748429991b29080f14a38 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Mon, 15 Mar 2021 16:59:51 -0400 Subject: [PATCH 161/203] dynamiccert: unit test with DynamicServingCertificateController Signed-off-by: Monis Khan --- internal/dynamiccert/provider_test.go | 226 ++++++++++++++++++ .../concierge_api_serving_certs_test.go | 8 +- 2 files changed, 230 insertions(+), 4 deletions(-) create mode 100644 internal/dynamiccert/provider_test.go diff --git a/internal/dynamiccert/provider_test.go b/internal/dynamiccert/provider_test.go new file mode 100644 index 00000000..140efe63 --- /dev/null +++ b/internal/dynamiccert/provider_test.go @@ -0,0 +1,226 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package dynamiccert + +import ( + "crypto/tls" + "crypto/x509" + "net" + "reflect" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/server/dynamiccertificates" + "k8s.io/apiserver/pkg/storage/names" + + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/test/library" +) + +func TestProviderWithDynamicServingCertificateController(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + f func(t *testing.T, ca Provider, certKey Private) (wantClientCASubjects [][]byte, wantCerts []tls.Certificate) + }{ + { + name: "no-op leave everything alone", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(ca.CurrentCABundleContent()) + require.True(t, ok, "should have valid non-empty CA bundle") + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return pool.Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "unset the CA", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + ca.UnsetCertKeyContent() + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return nil, []tls.Certificate{cert} + }, + }, + { + name: "unset the serving cert - still serves the old content", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(ca.CurrentCABundleContent()) + require.True(t, ok, "should have valid non-empty CA bundle") + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + certKey.UnsetCertKeyContent() + + return pool.Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "change to a new CA", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + // use unique names for all CAs to make sure the pool subjects are different + newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour) + require.NoError(t, err) + caKey, err := newCA.PrivateKeyToPEM() + require.NoError(t, err) + err = ca.SetCertKeyContent(newCA.Bundle(), caKey) + require.NoError(t, err) + + certPEM, keyPEM := certKey.CurrentCertKeyContent() + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + return newCA.Pool().Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "change to new serving cert", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + // use unique names for all CAs to make sure the pool subjects are different + newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour) + require.NoError(t, err) + + certPEM, keyPEM, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.2")}, time.Hour) + require.NoError(t, err) + + err = certKey.SetCertKeyContent(certPEM, keyPEM) + require.NoError(t, err) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + pool := x509.NewCertPool() + ok := pool.AppendCertsFromPEM(ca.CurrentCABundleContent()) + require.True(t, ok, "should have valid non-empty CA bundle") + + return pool.Subjects(), []tls.Certificate{cert} + }, + }, + { + name: "change both CA and serving cert", + f: func(t *testing.T, ca Provider, certKey Private) ([][]byte, []tls.Certificate) { + // use unique names for all CAs to make sure the pool subjects are different + newCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-ca"), time.Hour) + require.NoError(t, err) + + certPEM, keyPEM, err := newCA.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.3")}, time.Hour) + require.NoError(t, err) + + err = certKey.SetCertKeyContent(certPEM, keyPEM) + require.NoError(t, err) + + cert, err := tls.X509KeyPair(certPEM, keyPEM) + require.NoError(t, err) + + // use unique names for all CAs to make sure the pool subjects are different + newOtherCA, err := certauthority.New(names.SimpleNameGenerator.GenerateName("new-other-ca"), time.Hour) + require.NoError(t, err) + caKey, err := newOtherCA.PrivateKeyToPEM() + require.NoError(t, err) + err = ca.SetCertKeyContent(newOtherCA.Bundle(), caKey) + require.NoError(t, err) + + return newOtherCA.Pool().Subjects(), []tls.Certificate{cert} + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // use unique names for all CAs to make sure the pool subjects are different + ca, err := certauthority.New(names.SimpleNameGenerator.GenerateName("ca"), time.Hour) + require.NoError(t, err) + caKey, err := ca.PrivateKeyToPEM() + require.NoError(t, err) + caContent := NewCA("ca") + err = caContent.SetCertKeyContent(ca.Bundle(), caKey) + require.NoError(t, err) + + cert, key, err := ca.IssueServerCertPEM(nil, []net.IP{net.ParseIP("127.0.0.1")}, time.Hour) + require.NoError(t, err) + certKeyContent := NewServingCert("cert-key") + err = certKeyContent.SetCertKeyContent(cert, key) + require.NoError(t, err) + + tlsConfig := &tls.Config{ + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + ClientAuth: tls.RequestClientCert, + } + + dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController( + tlsConfig, + caContent, + certKeyContent, + nil, // we do not care about SNI + nil, // we do not care about events + ) + + caContent.AddListener(dynamicCertificateController) + certKeyContent.AddListener(dynamicCertificateController) + + err = dynamicCertificateController.RunOnce() + require.NoError(t, err) + + stopCh := make(chan struct{}) + defer close(stopCh) + go dynamicCertificateController.Run(1, stopCh) + + tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient + + wantClientCASubjects, wantCerts := tt.f(t, caContent, certKeyContent) + + var lastTLSConfig *tls.Config + + // it will take some time for the controller to catch up + err = wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) { + actualTLSConfig, err := tlsConfig.GetConfigForClient(&tls.ClientHelloInfo{ServerName: "force-standard-sni"}) + if err != nil { + return false, err + } + + lastTLSConfig = actualTLSConfig + + return reflect.DeepEqual(wantClientCASubjects, poolSubjects(actualTLSConfig.ClientCAs)) && + reflect.DeepEqual(wantCerts, actualTLSConfig.Certificates), nil + }) + + if err != nil && lastTLSConfig != nil { + // for debugging failures + t.Log("diff between client CAs:\n", cmp.Diff( + library.Sdump(wantClientCASubjects), + library.Sdump(poolSubjects(lastTLSConfig.ClientCAs)), + )) + t.Log("diff between serving certs:\n", cmp.Diff( + library.Sdump(wantCerts), + library.Sdump(lastTLSConfig.Certificates), + )) + } + require.NoError(t, err) + }) + } +} + +func poolSubjects(pool *x509.CertPool) [][]byte { + if pool == nil { + return nil + } + return pool.Subjects() +} diff --git a/test/integration/concierge_api_serving_certs_test.go b/test/integration/concierge_api_serving_certs_test.go index 04ee11af..bbdbde33 100644 --- a/test/integration/concierge_api_serving_certs_test.go +++ b/test/integration/concierge_api_serving_certs_test.go @@ -159,10 +159,10 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) { return err == nil } - // Unfortunately, although our code changes all the certs immediately, it seems to take ~1 minute for - // the API machinery to notice that we updated our serving cert, causing 1 minute of downtime for our endpoint. - assert.Eventually(t, aggregatedAPIWorking, 2*time.Minute, 250*time.Millisecond) - require.NoError(t, err) // prints out the error and stops the test in case of failure + // our code changes all the certs immediately thus this should be healthy fairly quickly + // if this starts flaking, check for bugs in our dynamiccertificates.Notifier implementation + assert.Eventually(t, aggregatedAPIWorking, 30*time.Second, 250*time.Millisecond) + require.NoError(t, err, "dynamiccertificates.Notifier broken?") // prints out the error and stops the test in case of failure }) } } From efd973fa17e4ffdd7ac6236a9220d8121b7e61f4 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Mon, 15 Mar 2021 12:28:53 -0700 Subject: [PATCH 162/203] Test waiting for a minute and keeping connection open Signed-off-by: Margo Crawford --- .../concierge_impersonation_proxy_test.go | 240 ++++++++++-------- 1 file changed, 140 insertions(+), 100 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 14b38fc1..cfd3996c 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -241,6 +241,68 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl ) } + t.Run("watching for a full minute", func(t *testing.T) { + kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) + + namespaceName := createTestNamespace(t, adminClient) + + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) + + // Get pods in concierge namespace and pick one. + // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. + pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Greater(t, len(pods.Items), 0) + var conciergePod *corev1.Pod + for _, pod := range pods.Items { + pod := pod + if !strings.Contains(pod.Name, "kube-cert-agent") { + conciergePod = &pod + } + } + require.NotNil(t, conciergePod, "could not find a concierge pod") + + // run the kubectl port-forward command + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd.Env = envVarsWithProxy + + // start, but don't wait for the command to finish + err = portForwardCmd.Start() + require.NoError(t, err, `"kubectl port-forward" failed`) + go func() { + assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) + }() + time.Sleep(1 * time.Minute) + + require.Eventually(t, func() bool { + // then run curl something against it + timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") + var curlStdOut, curlStdErr bytes.Buffer + curlCmd.Stdout = &curlStdOut + curlCmd.Stderr = &curlStdErr + err = curlCmd.Run() + if err != nil { + t.Log("curl error: " + err.Error()) + t.Log("curlStdErr: " + curlStdErr.String()) + t.Log("stdout: " + curlStdOut.String()) + } + // we expect this to 403, but all we care is that it gets through + return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + }, 5*time.Minute, 500*time.Millisecond) + }) + t.Run("using and watching all the basic verbs", func(t *testing.T) { // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := createTestNamespace(t, adminClient) @@ -505,69 +567,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl Verb: "get", Group: "", Version: "v1", Resource: "namespaces", }) - pinnipedExe := library.PinnipedCLIPath(t) - tempDir := testutil.TempDir(t) - - var envVarsWithProxy []string - if !env.HasCapability(library.HasExternalLoadBalancerProvider) { - // Only if you don't have a load balancer, use the squid proxy when it's available. - envVarsWithProxy = append(os.Environ(), env.ProxyEnv()...) - } - - // Get the kubeconfig. - getKubeConfigCmd := []string{"get", "kubeconfig", - "--concierge-api-group-suffix", env.APIGroupSuffix, - "--oidc-skip-browser", - "--static-token", env.TestUser.Token, - // Force the use of impersonation proxy strategy, but let it auto-discover the endpoint and CA. - "--concierge-mode", "ImpersonationProxy"} - t.Log("Running:", pinnipedExe, getKubeConfigCmd) - kubeconfigYAML, getKubeConfigStderr := runPinnipedCLI(t, envVarsWithProxy, pinnipedExe, getKubeConfigCmd...) - // "pinniped get kubectl" prints some status messages to stderr - t.Log(getKubeConfigStderr) - // Make sure that the "pinniped get kubeconfig" auto-discovered the impersonation proxy and we're going to - // make our kubectl requests through the impersonation proxy. Avoid using require.Contains because the error - // message would contain credentials. - require.True(t, - strings.Contains(kubeconfigYAML, "server: "+impersonationProxyURL+"\n"), - "the generated kubeconfig did not include the expected impersonation server address: %s", - impersonationProxyURL, - ) - require.True(t, - strings.Contains(kubeconfigYAML, "- --concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(impersonationProxyCACertPEM)+"\n"), - "the generated kubeconfig did not include the base64 encoded version of this expected impersonation CA cert: %s", - impersonationProxyCACertPEM, - ) - - // Write the kubeconfig to a temp file. - kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") - require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) - - // func to create kubectl commands with a kubeconfig - kubectlCommand := func(timeout context.Context, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) { - allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) - //nolint:gosec // we are not performing malicious argument injection against ourselves - kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) - var stdout, stderr syncBuffer - kubectlCmd.Stdout = &stdout - kubectlCmd.Stderr = &stderr - kubectlCmd.Env = envVarsWithProxy - - t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " ")) - return kubectlCmd, &stdout, &stderr - } - // Func to run kubeconfig commands. - runKubectl := func(args ...string) (string, string, error) { - timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() - - kubectlCmd, stdout, stderr := kubectlCommand(timeout, args...) - - err := kubectlCmd.Run() - t.Logf("kubectl stdout output: %s", stdout.String()) - t.Logf("kubectl stderr output: %s", stderr.String()) - return stdout.String(), stderr.String(), err - } + kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) // Get pods in concierge namespace and pick one. // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. @@ -586,58 +586,27 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) - stdout, _, err := runKubectl("exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) + stdout, err := runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) require.NoError(t, err, `"kubectl exec" failed`) require.Equal(t, echoString+"\n", stdout) // run the kubectl cp command localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) - _, _, err = runKubectl("cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, conciergePod.Name, remoteEchoFile), localEchoFile) + _, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, conciergePod.Name, remoteEchoFile), localEchoFile) require.NoError(t, err, `"kubectl cp" failed`) localEchoFileData, err := ioutil.ReadFile(localEchoFile) require.NoError(t, err) require.Equal(t, echoString+"\n", string(localEchoFileData)) defer func() { - _, _, _ = runKubectl("exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "rm", remoteEchoFile) // cleanup remote echo file + _, _ = runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "rm", remoteEchoFile) // cleanup remote echo file }() // run the kubectl logs command logLinesCount := 10 - stdout, _, err = runKubectl("logs", "--namespace", env.ConciergeNamespace, conciergePod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) + stdout, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "logs", "--namespace", env.ConciergeNamespace, conciergePod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) require.NoError(t, err, `"kubectl logs" failed`) require.Equalf(t, logLinesCount, strings.Count(stdout, "\n"), "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) - // run the kubectl port-forward command - timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() - portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") - portForwardCmd.Env = envVarsWithProxy - - // start, but don't wait for the command to finish - err = portForwardCmd.Start() - require.NoError(t, err, `"kubectl port-forward" failed`) - go func() { - assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) - }() - - require.Eventually(t, func() bool { - // then run curl something against it - timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") - var curlStdOut, curlStdErr bytes.Buffer - curlCmd.Stdout = &curlStdOut - curlCmd.Stderr = &curlStdErr - err = curlCmd.Run() - if err != nil { - t.Log("curl error: " + err.Error()) - t.Log("curlStdErr: " + curlStdErr.String()) - t.Log("stdout: " + curlStdOut.String()) - } - // we expect this to 403, but all we care is that it gets through - return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") - }, 5*time.Minute, 500*time.Millisecond) - // run the kubectl attach command namespaceName := createTestNamespace(t, adminClient) attachPod := library.CreatePod(ctx, t, "impersonation-proxy-attach", namespaceName, corev1.PodSpec{ @@ -651,9 +620,9 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, }, }) - timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) + attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) attachCmd.Env = envVarsWithProxy attachStdin, err := attachCmd.StdinPipe() require.NoError(t, err) @@ -1160,6 +1129,77 @@ func credentialIssuerName(env *library.TestEnv) string { return env.ConciergeAppName + "-config" } +func getImpersonationKubeconfig(t *testing.T, env *library.TestEnv, impersonationProxyURL string, impersonationProxyCACertPEM []byte) (string, []string, string) { + t.Helper() + + pinnipedExe := library.PinnipedCLIPath(t) + tempDir := testutil.TempDir(t) + + var envVarsWithProxy []string + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + // Only if you don't have a load balancer, use the squid proxy when it's available. + envVarsWithProxy = append(os.Environ(), env.ProxyEnv()...) + } + + // Get the kubeconfig. + getKubeConfigCmd := []string{"get", "kubeconfig", + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--oidc-skip-browser", + "--static-token", env.TestUser.Token, + // Force the use of impersonation proxy strategy, but let it auto-discover the endpoint and CA. + "--concierge-mode", "ImpersonationProxy"} + t.Log("Running:", pinnipedExe, getKubeConfigCmd) + kubeconfigYAML, getKubeConfigStderr := runPinnipedCLI(t, envVarsWithProxy, pinnipedExe, getKubeConfigCmd...) + // "pinniped get kubectl" prints some status messages to stderr + t.Log(getKubeConfigStderr) + // Make sure that the "pinniped get kubeconfig" auto-discovered the impersonation proxy and we're going to + // make our kubectl requests through the impersonation proxy. Avoid using require.Contains because the error + // message would contain credentials. + require.True(t, + strings.Contains(kubeconfigYAML, "server: "+impersonationProxyURL+"\n"), + "the generated kubeconfig did not include the expected impersonation server address: %s", + impersonationProxyURL, + ) + require.True(t, + strings.Contains(kubeconfigYAML, "- --concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(impersonationProxyCACertPEM)+"\n"), + "the generated kubeconfig did not include the base64 encoded version of this expected impersonation CA cert: %s", + impersonationProxyCACertPEM, + ) + + // Write the kubeconfig to a temp file. + kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") + require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) + + return kubeconfigPath, envVarsWithProxy, tempDir +} + +// func to create kubectl commands with a kubeconfig. +func kubectlCommand(timeout context.Context, t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) { + allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) + //nolint:gosec // we are not performing malicious argument injection against ourselves + kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) + var stdout, stderr syncBuffer + kubectlCmd.Stdout = &stdout + kubectlCmd.Stderr = &stderr + kubectlCmd.Env = envVarsWithProxy + + t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " ")) + return kubectlCmd, &stdout, &stderr +} + +// Func to run kubeconfig commands. +func runKubectl(t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (string, error) { + timeout, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancelFunc() + + kubectlCmd, stdout, stderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, args...) + + err := kubectlCmd.Run() + t.Logf("kubectl stdout output: %s", stdout.String()) + t.Logf("kubectl stderr output: %s", stderr.String()) + return stdout.String(), err +} + // watchJSON defines the expected JSON wire equivalent of watch.Event. type watchJSON struct { Type watch.EventType `json:"type,omitempty"` From 939ea30030cd1ca735cbb2883c49e4a1bf0198c8 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Mon, 15 Mar 2021 14:34:09 -0700 Subject: [PATCH 163/203] Make all tests but disable test parallelized Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 1107 +++++++++-------- 1 file changed, 562 insertions(+), 545 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index cfd3996c..ec767aa3 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -140,7 +140,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } newImpersonationProxyClient := func(impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { - refreshedCredentials := refreshCredential() + refreshedCredentials := refreshCredential().DeepCopy() refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests return newImpersonationProxyClientWithCredentials(refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) } @@ -226,599 +226,616 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl impersonationProxyKubeClient := func() kubernetes.Interface { return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "").Kubernetes } - - // Test that the user can perform basic actions through the client with their username and group membership - // influencing RBAC checks correctly. - t.Run( - "access as user", - library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyKubeClient()), - ) - for _, group := range env.TestUser.ExpectedGroups { - group := group + t.Run("positive tests", func(t *testing.T) { + // Test that the user can perform basic actions through the client with their username and group membership + // influencing RBAC checks correctly. t.Run( - "access as group "+group, - library.AccessAsGroupTest(ctx, group, impersonationProxyKubeClient()), + "access as user", + library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyKubeClient()), ) - } - - t.Run("watching for a full minute", func(t *testing.T) { - kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) - - namespaceName := createTestNamespace(t, adminClient) - - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) - - // Get pods in concierge namespace and pick one. - // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. - pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - require.Greater(t, len(pods.Items), 0) - var conciergePod *corev1.Pod - for _, pod := range pods.Items { - pod := pod - if !strings.Contains(pod.Name, "kube-cert-agent") { - conciergePod = &pod - } + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run( + "access as group "+group, + library.AccessAsGroupTest(ctx, group, impersonationProxyKubeClient()), + ) } - require.NotNil(t, conciergePod, "could not find a concierge pod") - // run the kubectl port-forward command - timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() - portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") - portForwardCmd.Env = envVarsWithProxy + t.Run("watching for a full minute", func(t *testing.T) { + t.Parallel() - // start, but don't wait for the command to finish - err = portForwardCmd.Start() - require.NoError(t, err, `"kubectl port-forward" failed`) - go func() { - assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) - }() - time.Sleep(1 * time.Minute) + kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) - require.Eventually(t, func() bool { - // then run curl something against it - timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + namespaceName := createTestNamespace(t, adminClient) + + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) + + // Get pods in concierge namespace and pick one. + // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. + pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Greater(t, len(pods.Items), 0) + var conciergePod *corev1.Pod + for _, pod := range pods.Items { + pod := pod + if !strings.Contains(pod.Name, "kube-cert-agent") { + conciergePod = &pod + } + } + require.NotNil(t, conciergePod, "could not find a concierge pod") + + // run the kubectl port-forward command + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") - var curlStdOut, curlStdErr bytes.Buffer - curlCmd.Stdout = &curlStdOut - curlCmd.Stderr = &curlStdErr - err = curlCmd.Run() - if err != nil { - t.Log("curl error: " + err.Error()) - t.Log("curlStdErr: " + curlStdErr.String()) - t.Log("stdout: " + curlStdOut.String()) + portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd.Env = envVarsWithProxy + + // start, but don't wait for the command to finish + err = portForwardCmd.Start() + require.NoError(t, err, `"kubectl port-forward" failed`) + go func() { + assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) + }() + time.Sleep(1 * time.Minute) + + require.Eventually(t, func() bool { + // then run curl something against it + timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") + var curlStdOut, curlStdErr bytes.Buffer + curlCmd.Stdout = &curlStdOut + curlCmd.Stderr = &curlStdErr + err = curlCmd.Run() + if err != nil { + t.Log("curl error: " + err.Error()) + t.Log("curlStdErr: " + curlStdErr.String()) + t.Log("stdout: " + curlStdOut.String()) + } + // we expect this to 403, but all we care is that it gets through + return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + }, 5*time.Minute, 500*time.Millisecond) + }) + + t.Run("using and watching all the basic verbs", func(t *testing.T) { + t.Parallel() + + // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. + namespaceName := createTestNamespace(t, adminClient) + + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) + + // Create and start informer to exercise the "watch" verb for us. + informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( + impersonationProxyKubeClient(), + 0, + k8sinformers.WithNamespace(namespaceName)) + informer := informerFactory.Core().V1().ConfigMaps() + informer.Informer() // makes sure that the informer will cache + stopChannel := make(chan struct{}) + informerFactory.Start(stopChannel) + t.Cleanup(func() { + // Shut down the informer. + close(stopChannel) + }) + informerFactory.WaitForCacheSync(ctx.Done()) + + // Use labels on our created ConfigMaps to avoid accidentally listing other ConfigMaps that might + // exist in the namespace. In Kube 1.20+ there is a default ConfigMap in every namespace. + configMapLabels := labels.Set{ + "pinniped.dev/testConfigMap": library.RandHex(t, 8), } - // we expect this to 403, but all we care is that it gets through - return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") - }, 5*time.Minute, 500*time.Millisecond) - }) - t.Run("using and watching all the basic verbs", func(t *testing.T) { - // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. - namespaceName := createTestNamespace(t, adminClient) + // Test "create" verb through the impersonation proxy. + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, + metav1.CreateOptions{}, + ) + require.NoError(t, err) - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + // Make sure that all of the created ConfigMaps show up in the informer's cache to + // demonstrate that the informer's "watch" verb is working through the impersonation proxy. + require.Eventually(t, func() bool { + _, err1 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-1") + _, err2 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-2") + _, err3 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + return err1 == nil && err2 == nil && err3 == nil + }, 10*time.Second, 50*time.Millisecond) + + // Test "get" verb through the impersonation proxy. + configMap3, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Get(ctx, "configmap-3", metav1.GetOptions{}) + require.NoError(t, err) + + // Test "list" verb through the impersonation proxy. + listResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ + LabelSelector: configMapLabels.String(), + }) + require.NoError(t, err) + require.Len(t, listResult.Items, 3) + + // Test "update" verb through the impersonation proxy. + configMap3.Data = map[string]string{"foo": "bar"} + updateResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Update(ctx, configMap3, metav1.UpdateOptions{}) + require.NoError(t, err) + require.Equal(t, "bar", updateResult.Data["foo"]) + + // Make sure that the updated ConfigMap shows up in the informer's cache. + require.Eventually(t, func() bool { + configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + return err == nil && configMap.Data["foo"] == "bar" + }, 10*time.Second, 50*time.Millisecond) + + // Test "patch" verb through the impersonation proxy. + patchResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Patch(ctx, + "configmap-3", + types.MergePatchType, + []byte(`{"data":{"baz":"42"}}`), + metav1.PatchOptions{}, + ) + require.NoError(t, err) + require.Equal(t, "bar", patchResult.Data["foo"]) + require.Equal(t, "42", patchResult.Data["baz"]) + + // Make sure that the patched ConfigMap shows up in the informer's cache. + require.Eventually(t, func() bool { + configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + return err == nil && configMap.Data["foo"] == "bar" && configMap.Data["baz"] == "42" + }, 10*time.Second, 50*time.Millisecond) + + // Test "delete" verb through the impersonation proxy. + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) + require.NoError(t, err) + + // Make sure that the deleted ConfigMap shows up in the informer's cache. + require.Eventually(t, func() bool { + _, getErr := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") + list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) + return k8serrors.IsNotFound(getErr) && listErr == nil && len(list) == 2 + }, 10*time.Second, 50*time.Millisecond) + + // Test "deletecollection" verb through the impersonation proxy. + err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + require.NoError(t, err) + + // Make sure that the deleted ConfigMaps shows up in the informer's cache. + require.Eventually(t, func() bool { + list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) + return listErr == nil && len(list) == 0 + }, 10*time.Second, 50*time.Millisecond) + + // There should be no ConfigMaps left. + listResult, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ + LabelSelector: configMapLabels.String(), + }) + require.NoError(t, err) + require.Len(t, listResult.Items, 0) }) - // Create and start informer to exercise the "watch" verb for us. - informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( - impersonationProxyKubeClient(), - 0, - k8sinformers.WithNamespace(namespaceName)) - informer := informerFactory.Core().V1().ConfigMaps() - informer.Informer() // makes sure that the informer will cache - stopChannel := make(chan struct{}) - informerFactory.Start(stopChannel) - t.Cleanup(func() { - // Shut down the informer. - close(stopChannel) - }) - informerFactory.WaitForCacheSync(ctx.Done()) + t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { + t.Parallel() - // Use labels on our created ConfigMaps to avoid accidentally listing other ConfigMaps that might - // exist in the namespace. In Kube 1.20+ there is a default ConfigMap in every namespace. - configMapLabels := labels.Set{ - "pinniped.dev/testConfigMap": library.RandHex(t, 8), - } + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: env.ConciergeNamespace, Verb: "get", Group: "", Version: "v1", Resource: "secrets", + }) - // Test "create" verb through the impersonation proxy. - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, - metav1.CreateOptions{}, - ) - require.NoError(t, err) - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, - metav1.CreateOptions{}, - ) - require.NoError(t, err) - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, - &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, - metav1.CreateOptions{}, - ) - require.NoError(t, err) + // Make a client which will send requests through the impersonation proxy and will also add + // impersonate headers to the request. + doubleImpersonationKubeClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes - // Make sure that all of the created ConfigMaps show up in the informer's cache to - // demonstrate that the informer's "watch" verb is working through the impersonation proxy. - require.Eventually(t, func() bool { - _, err1 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-1") - _, err2 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-2") - _, err3 := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") - return err1 == nil && err2 == nil && err3 == nil - }, 10*time.Second, 50*time.Millisecond) + // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. + // We could use any resource for this, but we happen to know that this one should exist. + _, err := impersonationProxyKubeClient().CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + require.NoError(t, err) - // Test "get" verb through the impersonation proxy. - configMap3, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Get(ctx, "configmap-3", metav1.GetOptions{}) - require.NoError(t, err) - - // Test "list" verb through the impersonation proxy. - listResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ - LabelSelector: configMapLabels.String(), - }) - require.NoError(t, err) - require.Len(t, listResult.Items, 3) - - // Test "update" verb through the impersonation proxy. - configMap3.Data = map[string]string{"foo": "bar"} - updateResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Update(ctx, configMap3, metav1.UpdateOptions{}) - require.NoError(t, err) - require.Equal(t, "bar", updateResult.Data["foo"]) - - // Make sure that the updated ConfigMap shows up in the informer's cache. - require.Eventually(t, func() bool { - configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") - return err == nil && configMap.Data["foo"] == "bar" - }, 10*time.Second, 50*time.Millisecond) - - // Test "patch" verb through the impersonation proxy. - patchResult, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Patch(ctx, - "configmap-3", - types.MergePatchType, - []byte(`{"data":{"baz":"42"}}`), - metav1.PatchOptions{}, - ) - require.NoError(t, err) - require.Equal(t, "bar", patchResult.Data["foo"]) - require.Equal(t, "42", patchResult.Data["baz"]) - - // Make sure that the patched ConfigMap shows up in the informer's cache. - require.Eventually(t, func() bool { - configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") - return err == nil && configMap.Data["foo"] == "bar" && configMap.Data["baz"] == "42" - }, 10*time.Second, 50*time.Millisecond) - - // Test "delete" verb through the impersonation proxy. - err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) - require.NoError(t, err) - - // Make sure that the deleted ConfigMap shows up in the informer's cache. - require.Eventually(t, func() bool { - _, getErr := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") - list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) - return k8serrors.IsNotFound(getErr) && listErr == nil && len(list) == 2 - }, 10*time.Second, 50*time.Millisecond) - - // Test "deletecollection" verb through the impersonation proxy. - err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) - require.NoError(t, err) - - // Make sure that the deleted ConfigMaps shows up in the informer's cache. - require.Eventually(t, func() bool { - list, listErr := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) - return listErr == nil && len(list) == 0 - }, 10*time.Second, 50*time.Millisecond) - - // There should be no ConfigMaps left. - listResult, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ - LabelSelector: configMapLabels.String(), - }) - require.NoError(t, err) - require.Len(t, listResult.Items, 0) - }) - - t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, - ) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: env.ConciergeNamespace, Verb: "get", Group: "", Version: "v1", Resource: "secrets", + // Now we'll see what happens when we add an impersonation header to the request. This should generate a + // request similar to the one above, except that it will also have an impersonation header. + _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // Double impersonation is not supported yet, so we should get an error. + require.EqualError(t, err, fmt.Sprintf( + `users "other-user-to-impersonate" is forbidden: `+ + `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `impersonation is not allowed or invalid verb`, + env.TestUser.ExpectedUsername)) }) - // Make a client which will send requests through the impersonation proxy and will also add - // impersonate headers to the request. - doubleImpersonationKubeClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes + // This is a separate test from the above double impersonation test because the cluster admin user gets special + // authorization treatment from the Kube API server code that we are using, and we want to ensure that we are blocking + // double impersonation even for the cluster admin. + t.Run("double impersonation as a cluster admin user is blocked", func(t *testing.T) { + t.Parallel() - // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. - // We could use any resource for this, but we happen to know that this one should exist. - _, err = impersonationProxyKubeClient().CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - require.NoError(t, err) + // Copy the admin credentials from the admin kubeconfig. + adminClientRestConfig := library.NewClientConfig(t) - // Now we'll see what happens when we add an impersonation header to the request. This should generate a - // request similar to the one above, except that it will also have an impersonation header. - _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - // Double impersonation is not supported yet, so we should get an error. - require.EqualError(t, err, fmt.Sprintf( - `users "other-user-to-impersonate" is forbidden: `+ - `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ - `impersonation is not allowed or invalid verb`, - env.TestUser.ExpectedUsername)) - }) + if adminClientRestConfig.BearerToken == "" && adminClientRestConfig.CertData == nil && adminClientRestConfig.KeyData == nil { + t.Skip("The admin kubeconfig does not include credentials, so skipping this test.") + } - // This is a separate test from the above double impersonation test because the cluster admin user gets special - // authorization treatment from the Kube API server code that we are using, and we want to ensure that we are blocking - // double impersonation even for the cluster admin. - t.Run("double impersonation as a cluster admin user is blocked", func(t *testing.T) { - // Copy the admin credentials from the admin kubeconfig. - adminClientRestConfig := library.NewClientConfig(t) + clusterAdminCredentials := &loginv1alpha1.ClusterCredential{ + Token: adminClientRestConfig.BearerToken, + ClientCertificateData: string(adminClientRestConfig.CertData), + ClientKeyData: string(adminClientRestConfig.KeyData), + } - if adminClientRestConfig.BearerToken == "" && adminClientRestConfig.CertData == nil && adminClientRestConfig.KeyData == nil { - t.Skip("The admin kubeconfig does not include credentials, so skipping this test.") - } + // Make a client using the admin credentials which will send requests through the impersonation proxy + // and will also add impersonate headers to the request. + doubleImpersonationKubeClient := newImpersonationProxyClientWithCredentials( + clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate", + ).Kubernetes - clusterAdminCredentials := &loginv1alpha1.ClusterCredential{ - Token: adminClientRestConfig.BearerToken, - ClientCertificateData: string(adminClientRestConfig.CertData), - ClientKeyData: string(adminClientRestConfig.KeyData), - } + _, err := doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // Double impersonation is not supported yet, so we should get an error. + require.Error(t, err) + require.Regexp(t, + `users "other-user-to-impersonate" is forbidden: `+ + `User ".*" cannot impersonate resource "users" in API group "" at the cluster scope: `+ + `impersonation is not allowed or invalid verb`, + err.Error(), + ) + }) - // Make a client using the admin credentials which will send requests through the impersonation proxy - // and will also add impersonate headers to the request. - doubleImpersonationKubeClient := newImpersonationProxyClientWithCredentials( - clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate", - ).Kubernetes + t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { + t.Parallel() - _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - // Double impersonation is not supported yet, so we should get an error. - require.Error(t, err) - require.Regexp(t, - `users "other-user-to-impersonate" is forbidden: `+ - `User ".*" cannot impersonate resource "users" in API group "" at the cluster scope: `+ - `impersonation is not allowed or invalid verb`, - err.Error(), - ) - }) + // Test using the TokenCredentialRequest for authentication. + impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( + impersonationProxyURL, impersonationProxyCACertPEM, "", + ).PinnipedConcierge + whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + env.TestUser.ExpectedUsername, + append(env.TestUser.ExpectedGroups, "system:authenticated"), + ), + whoAmI, + ) - t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { - // Test using the TokenCredentialRequest for authentication. - impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( - impersonationProxyURL, impersonationProxyCACertPEM, "", - ).PinnipedConcierge - whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). - Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) - require.NoError(t, err) - require.Equal(t, - expectedWhoAmIRequestResponse( - env.TestUser.ExpectedUsername, - append(env.TestUser.ExpectedGroups, "system:authenticated"), - ), - whoAmI, - ) + // Test an unauthenticated request which does not include any credentials. + impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient( + impersonationProxyURL, impersonationProxyCACertPEM, "", + ).PinnipedConcierge + whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + "system:anonymous", + []string{"system:unauthenticated"}, + ), + whoAmI, + ) - // Test an unauthenticated request which does not include any credentials. - impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient( - impersonationProxyURL, impersonationProxyCACertPEM, "", - ).PinnipedConcierge - whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). - Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) - require.NoError(t, err) - require.Equal(t, - expectedWhoAmIRequestResponse( - "system:anonymous", - []string{"system:unauthenticated"}, - ), - whoAmI, - ) - - // Test using a service account token. Authenticating as Service Accounts through the impersonation - // proxy is not supported, so it should fail. - namespaceName := createTestNamespace(t, adminClient) - impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials( - &loginv1alpha1.ClusterCredential{Token: createServiceAccountToken(ctx, t, adminClient, namespaceName)}, - impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge - _, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). - Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) - require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") - require.True(t, k8serrors.IsInternalError(err), err) - require.Equal(t, &k8serrors.StatusError{ - ErrStatus: metav1.Status{ - Status: metav1.StatusFailure, - Code: http.StatusInternalServerError, - Reason: metav1.StatusReasonInternalError, - Details: &metav1.StatusDetails{ - Causes: []metav1.StatusCause{ - { - Message: "unimplemented functionality - unable to act as current user", + // Test using a service account token. Authenticating as Service Accounts through the impersonation + // proxy is not supported, so it should fail. + namespaceName := createTestNamespace(t, adminClient) + impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials( + &loginv1alpha1.ClusterCredential{Token: createServiceAccountToken(ctx, t, adminClient, namespaceName)}, + impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge + _, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") + require.True(t, k8serrors.IsInternalError(err), err) + require.Equal(t, &k8serrors.StatusError{ + ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusInternalServerError, + Reason: metav1.StatusReasonInternalError, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + { + Message: "unimplemented functionality - unable to act as current user", + }, }, }, + Message: "Internal error occurred: unimplemented functionality - unable to act as current user", }, - Message: "Internal error occurred: unimplemented functionality - unable to act as current user", - }, - }, err) - }) - - t.Run("kubectl as a client", func(t *testing.T) { - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, - ) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Verb: "get", Group: "", Version: "v1", Resource: "namespaces", + }, err) }) - kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) + t.Run("kubectl as a client", func(t *testing.T) { + t.Parallel() - // Get pods in concierge namespace and pick one. - // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. - pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - require.Greater(t, len(pods.Items), 0) - var conciergePod *corev1.Pod - for _, pod := range pods.Items { - pod := pod - if !strings.Contains(pod.Name, "kube-cert-agent") { - conciergePod = &pod + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Verb: "get", Group: "", Version: "v1", Resource: "namespaces", + }) + + kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) + + // Get pods in concierge namespace and pick one. + // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. + pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Greater(t, len(pods.Items), 0) + var conciergePod *corev1.Pod + for _, pod := range pods.Items { + pod := pod + if !strings.Contains(pod.Name, "kube-cert-agent") { + conciergePod = &pod + } } - } - require.NotNil(t, conciergePod, "could not find a concierge pod") + require.NotNil(t, conciergePod, "could not find a concierge pod") - // Try "kubectl exec" through the impersonation proxy. - echoString := "hello world" - remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) - stdout, err := runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) - require.NoError(t, err, `"kubectl exec" failed`) - require.Equal(t, echoString+"\n", stdout) + // Try "kubectl exec" through the impersonation proxy. + echoString := "hello world" + remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) + stdout, err := runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) + require.NoError(t, err, `"kubectl exec" failed`) + require.Equal(t, echoString+"\n", stdout) - // run the kubectl cp command - localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) - _, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, conciergePod.Name, remoteEchoFile), localEchoFile) - require.NoError(t, err, `"kubectl cp" failed`) - localEchoFileData, err := ioutil.ReadFile(localEchoFile) - require.NoError(t, err) - require.Equal(t, echoString+"\n", string(localEchoFileData)) - defer func() { - _, _ = runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "rm", remoteEchoFile) // cleanup remote echo file - }() - - // run the kubectl logs command - logLinesCount := 10 - stdout, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "logs", "--namespace", env.ConciergeNamespace, conciergePod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) - require.NoError(t, err, `"kubectl logs" failed`) - require.Equalf(t, logLinesCount, strings.Count(stdout, "\n"), "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) - - // run the kubectl attach command - namespaceName := createTestNamespace(t, adminClient) - attachPod := library.CreatePod(ctx, t, "impersonation-proxy-attach", namespaceName, corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "impersonation-proxy-attach", - Image: conciergePod.Spec.Containers[0].Image, - Command: []string{"bash"}, - Args: []string{"-c", `while true; do read VAR; echo "VAR: $VAR"; done`}, - Stdin: true, - }, - }, - }) - timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) - defer cancelFunc() - attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) - attachCmd.Env = envVarsWithProxy - attachStdin, err := attachCmd.StdinPipe() - require.NoError(t, err) - - // start but don't wait for the attach command - err = attachCmd.Start() - require.NoError(t, err) - - // write to stdin on the attach process - _, err = attachStdin.Write([]byte(echoString + "\n")) - require.NoError(t, err) - - // see that we can read stdout and it spits out stdin output back to us - wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) - require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*30, time.Second, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) - - // close stdin and attach process should exit - err = attachStdin.Close() - require.NoError(t, err) - err = attachCmd.Wait() - require.NoError(t, err) - }) - - t.Run("websocket client", func(t *testing.T) { - namespaceName := createTestNamespace(t, adminClient) - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) - - impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") - tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) - require.NoError(t, err) - - wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" - dest, _ := url.Parse(impersonationProxyURL) - dest.Scheme = "wss" - dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" - dest.RawQuery = url.Values{ - "watch": {"1"}, - "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, - "resourceVersion": {"0"}, - }.Encode() - dialer := websocket.Dialer{ - TLSClientConfig: tlsConfig, - } - if !env.HasCapability(library.HasExternalLoadBalancerProvider) { - dialer.Proxy = func(req *http.Request) (*url.URL, error) { - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil - } - } - c, r, err := dialer.Dial(dest.String(), http.Header{"Origin": {dest.String()}}) - if r != nil { + // run the kubectl cp command + localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) + _, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "cp", fmt.Sprintf("%s/%s:%s", env.ConciergeNamespace, conciergePod.Name, remoteEchoFile), localEchoFile) + require.NoError(t, err, `"kubectl cp" failed`) + localEchoFileData, err := ioutil.ReadFile(localEchoFile) + require.NoError(t, err) + require.Equal(t, echoString+"\n", string(localEchoFileData)) defer func() { - require.NoError(t, r.Body.Close()) + _, _ = runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", env.ConciergeNamespace, conciergePod.Name, "--", "rm", remoteEchoFile) // cleanup remote echo file }() - } - if err != nil && r != nil { - body, _ := ioutil.ReadAll(r.Body) - t.Logf("websocket dial failed: %d:%s", r.StatusCode, body) - } - require.NoError(t, err) - // perform a create through the admin client - wantConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, - } - wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, - wantConfigMap, - metav1.CreateOptions{}, - ) - require.NoError(t, err) - t.Cleanup(func() { - require.NoError(t, adminClient.CoreV1().ConfigMaps(namespaceName). - DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})) + // run the kubectl logs command + logLinesCount := 10 + stdout, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "logs", "--namespace", env.ConciergeNamespace, conciergePod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) + require.NoError(t, err, `"kubectl logs" failed`) + require.Equalf(t, logLinesCount, strings.Count(stdout, "\n"), "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) + + // run the kubectl attach command + namespaceName := createTestNamespace(t, adminClient) + attachPod := library.CreatePod(ctx, t, "impersonation-proxy-attach", namespaceName, corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "impersonation-proxy-attach", + Image: conciergePod.Spec.Containers[0].Image, + Command: []string{"bash"}, + Args: []string{"-c", `while true; do read VAR; echo "VAR: $VAR"; done`}, + Stdin: true, + }, + }, + }) + timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) + defer cancelFunc() + attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) + attachCmd.Env = envVarsWithProxy + attachStdin, err := attachCmd.StdinPipe() + require.NoError(t, err) + + // start but don't wait for the attach command + err = attachCmd.Start() + require.NoError(t, err) + + // write to stdin on the attach process + _, err = attachStdin.Write([]byte(echoString + "\n")) + require.NoError(t, err) + + // see that we can read stdout and it spits out stdin output back to us + wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) + require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*30, time.Second, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) + + // close stdin and attach process should exit + err = attachStdin.Close() + require.NoError(t, err) + err = attachCmd.Wait() + require.NoError(t, err) }) - // see if the websocket client received an event for the create - _, message, err := c.ReadMessage() - if err != nil { - t.Fatalf("Unexpected error: %v", err) - } - var got watchJSON - err = json.Unmarshal(message, &got) - require.NoError(t, err) - if got.Type != watch.Added { - t.Errorf("Unexpected type: %v", got.Type) - } - var actualConfigMap corev1.ConfigMap - require.NoError(t, json.Unmarshal(got.Object, &actualConfigMap)) - actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. - require.Equal(t, *wantConfigMap, actualConfigMap) - }) + t.Run("websocket client", func(t *testing.T) { + t.Parallel() - t.Run("http2 client", func(t *testing.T) { - namespaceName := createTestNamespace(t, adminClient) - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) + namespaceName := createTestNamespace(t, adminClient) + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) - wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" - wantConfigMap := &corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, - } - wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, - wantConfigMap, - metav1.CreateOptions{}, - ) - require.NoError(t, err) - t.Cleanup(func() { - _ = adminClient.CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) - }) + impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") + tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) + require.NoError(t, err) - // create rest client - restConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") - - tlsConfig, err := rest.TLSConfigFor(restConfig) - require.NoError(t, err) - httpTransport := http.Transport{ - TLSClientConfig: tlsConfig, - } - if !env.HasCapability(library.HasExternalLoadBalancerProvider) { - httpTransport.Proxy = func(req *http.Request) (*url.URL, error) { - proxyURL, err := url.Parse(env.Proxy) - require.NoError(t, err) - t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) - return proxyURL, nil + wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" + dest, _ := url.Parse(impersonationProxyURL) + dest.Scheme = "wss" + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" + dest.RawQuery = url.Values{ + "watch": {"1"}, + "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, + "resourceVersion": {"0"}, + }.Encode() + dialer := websocket.Dialer{ + TLSClientConfig: tlsConfig, } - } - err = http2.ConfigureTransport(&httpTransport) - require.NoError(t, err) + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + dialer.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + } + } + c, r, err := dialer.Dial(dest.String(), http.Header{"Origin": {dest.String()}}) + if r != nil { + defer func() { + require.NoError(t, r.Body.Close()) + }() + } + if err != nil && r != nil { + body, _ := ioutil.ReadAll(r.Body) + t.Logf("websocket dial failed: %d:%s", r.StatusCode, body) + } + require.NoError(t, err) - httpClient := http.Client{ - Transport: &httpTransport, - } + // perform a create through the admin client + wantConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, + } + wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, + wantConfigMap, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, adminClient.CoreV1().ConfigMaps(namespaceName). + DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})) + }) - dest, _ := url.Parse(impersonationProxyURL) - dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1" - getConfigmapRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) - require.NoError(t, err) - response, err := httpClient.Do(getConfigmapRequest) - require.NoError(t, err) - body, _ := ioutil.ReadAll(response.Body) - t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body) - require.Equal(t, "HTTP/2.0", response.Proto) - require.Equal(t, http.StatusOK, response.StatusCode) - defer func() { - require.NoError(t, response.Body.Close()) - }() - var actualConfigMap corev1.ConfigMap - require.NoError(t, json.Unmarshal(body, &actualConfigMap)) - actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. - require.Equal(t, *wantConfigMap, actualConfigMap) + // see if the websocket client received an event for the create + _, message, err := c.ReadMessage() + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + var got watchJSON + err = json.Unmarshal(message, &got) + require.NoError(t, err) + if got.Type != watch.Added { + t.Errorf("Unexpected type: %v", got.Type) + } + var actualConfigMap corev1.ConfigMap + require.NoError(t, json.Unmarshal(got.Object, &actualConfigMap)) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + }) - // watch configmaps - dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" - dest.RawQuery = url.Values{ - "watch": {"1"}, - "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, - "resourceVersion": {"0"}, - }.Encode() - watchConfigmapsRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) - require.NoError(t, err) - response, err = httpClient.Do(watchConfigmapsRequest) - require.NoError(t, err) - require.Equal(t, "HTTP/2.0", response.Proto) - require.Equal(t, http.StatusOK, response.StatusCode) - defer func() { - require.NoError(t, response.Body.Close()) - }() + t.Run("http2 client", func(t *testing.T) { + t.Parallel() - // decode - decoder := json.NewDecoder(response.Body) - var got watchJSON - err = decoder.Decode(&got) - require.NoError(t, err) - if got.Type != watch.Added { - t.Errorf("Unexpected type: %v", got.Type) - } - err = json.Unmarshal(got.Object, &actualConfigMap) - require.NoError(t, err) - require.Equal(t, "configmap-1", actualConfigMap.Name) - actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. - require.Equal(t, *wantConfigMap, actualConfigMap) + namespaceName := createTestNamespace(t, adminClient) + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, + ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", + }) + + wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" + wantConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, + } + wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, + wantConfigMap, + metav1.CreateOptions{}, + ) + require.NoError(t, err) + t.Cleanup(func() { + _ = adminClient.CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) + }) + + // create rest client + restConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") + + tlsConfig, err := rest.TLSConfigFor(restConfig) + require.NoError(t, err) + httpTransport := http.Transport{ + TLSClientConfig: tlsConfig, + } + if !env.HasCapability(library.HasExternalLoadBalancerProvider) { + httpTransport.Proxy = func(req *http.Request) (*url.URL, error) { + proxyURL, err := url.Parse(env.Proxy) + require.NoError(t, err) + t.Logf("passing request for %s through proxy %s", req.URL, proxyURL.String()) + return proxyURL, nil + } + } + err = http2.ConfigureTransport(&httpTransport) + require.NoError(t, err) + + httpClient := http.Client{ + Transport: &httpTransport, + } + + dest, _ := url.Parse(impersonationProxyURL) + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1" + getConfigmapRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) + require.NoError(t, err) + response, err := httpClient.Do(getConfigmapRequest) + require.NoError(t, err) + body, _ := ioutil.ReadAll(response.Body) + t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body) + require.Equal(t, "HTTP/2.0", response.Proto) + require.Equal(t, http.StatusOK, response.StatusCode) + defer func() { + require.NoError(t, response.Body.Close()) + }() + var actualConfigMap corev1.ConfigMap + require.NoError(t, json.Unmarshal(body, &actualConfigMap)) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + + // watch configmaps + dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" + dest.RawQuery = url.Values{ + "watch": {"1"}, + "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, + "resourceVersion": {"0"}, + }.Encode() + watchConfigmapsRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) + require.NoError(t, err) + response, err = httpClient.Do(watchConfigmapsRequest) + require.NoError(t, err) + require.Equal(t, "HTTP/2.0", response.Proto) + require.Equal(t, http.StatusOK, response.StatusCode) + defer func() { + require.NoError(t, response.Body.Close()) + }() + + // decode + decoder := json.NewDecoder(response.Body) + var got watchJSON + err = decoder.Decode(&got) + require.NoError(t, err) + if got.Type != watch.Added { + t.Errorf("Unexpected type: %v", got.Type) + } + err = json.Unmarshal(got.Object, &actualConfigMap) + require.NoError(t, err) + require.Equal(t, "configmap-1", actualConfigMap.Name) + actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. + require.Equal(t, *wantConfigMap, actualConfigMap) + }) }) t.Run("manually disabling the impersonation proxy feature", func(t *testing.T) { From 1b314893478af2a959c054a12b043701bbc0e135 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 15 Mar 2021 16:08:42 -0700 Subject: [PATCH 164/203] Add prepare-impersonator-on-kind.sh for manually starting impersonator It takes a lot of manual steps to get ready to manually test the impersonation proxy on a kind cluster, which makes it error prone, so encapsulate them into a script to make it easier. --- hack/prepare-impersonator-on-kind.sh | 114 +++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100755 hack/prepare-impersonator-on-kind.sh diff --git a/hack/prepare-impersonator-on-kind.sh b/hack/prepare-impersonator-on-kind.sh new file mode 100755 index 00000000..aae4f459 --- /dev/null +++ b/hack/prepare-impersonator-on-kind.sh @@ -0,0 +1,114 @@ +#!/usr/bin/env bash + +# A script to perform the setup required to manually test using the impersonation proxy on a kind cluster. +# Assumes that you installed the apps already using hack/prepare-for-integration-tests.sh. + +set -euo pipefail + +# The name of the namespace in which the concierge is installed. +CONCIERGE_NAMESPACE=concierge +# The name of the concierge app's Deployment. +CONCIERGE_DEPLOYMENT=pinniped-concierge +# The namespace in which the local-user-authenticator app is installed. +LOCAL_USER_AUTHENTICATOR_NAMESPACE=local-user-authenticator +# The port on which the impersonation proxy runs in the concierge pods. +IMPERSONATION_PROXY_PORT=8444 +# The port that we will use to access the impersonator from outside the cluster via `kubectl port-forward`. +LOCAL_PORT=8777 +LOCAL_HOST="127.0.0.1:${LOCAL_PORT}" + +# Change working directory to the top of the repo. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# Build the CLI for use later in the script. +go build ./cmd/pinniped + +# Create a test user and password. +if ! kubectl get secret pinny-the-seal --namespace $LOCAL_USER_AUTHENTICATOR_NAMESPACE; then + kubectl create secret generic pinny-the-seal --namespace $LOCAL_USER_AUTHENTICATOR_NAMESPACE \ + --from-literal=groups=group1,group2 \ + --from-literal=passwordHash="$(htpasswd -nbBC 10 x password123 | sed -e "s/^x://")" +fi + +# Get the CA of the local-user-authenticator. +LOCAL_USER_AUTHENTICATOR_CA=$(kubectl get secret local-user-authenticator-tls-serving-certificate \ + --namespace $LOCAL_USER_AUTHENTICATOR_NAMESPACE \ + -o jsonpath=\{.data.caCertificate\}) + +# Create a WebhookAuthenticator which points at the local-user-authenticator. +cat < /dev/null || true +} +trap cleanup EXIT + +# Get a working kubeconfig that will send requests through the impersonation proxy. +./pinniped get kubeconfig \ + --static-token "pinny-the-seal:password123" \ + --concierge-mode ImpersonationProxy >/tmp/kubeconfig + +echo +echo 'Ready. In another tab, use "kubectl --kubeconfig /tmp/kubeconfig " to make requests through the impersonation proxy.' +echo "When done, cancel with ctrl-C to clean up." +wait $port_forward_pid From 2460568be3abfe9817428b28752ddc304edf1636 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 15 Mar 2021 16:11:45 -0700 Subject: [PATCH 165/203] Add some debug logging --- internal/concierge/impersonator/impersonator.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 117909a4..6df28a76 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -269,9 +269,8 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi return } - plog.Trace("proxying authenticated request", - "url", r.URL.String(), - "method", r.Method, + plog.Debug("impersonation proxy servicing request", "method", r.Method, "url", r.URL.String()) + plog.Trace("impersonation proxy servicing request was for user", "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) @@ -279,6 +278,8 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi reverseProxy.Transport = rt reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line reverseProxy.ServeHTTP(w, r) + + plog.Debug("impersonation proxy finished servicing request", "method", r.Method, "url", r.URL.String()) }) }, nil } From 64e0dbb481bcb989922aa2a1a992e7cc6e42fa54 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Mon, 15 Mar 2021 16:31:54 -0700 Subject: [PATCH 166/203] Sleep for 1 minute 10 seconds instead of a minute in timeout test --- test/integration/concierge_impersonation_proxy_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index ec767aa3..c6b1561a 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -284,7 +284,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl go func() { assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() - time.Sleep(1 * time.Minute) + + time.Sleep(70 * time.Second) require.Eventually(t, func() bool { // then run curl something against it From 6887d0aca2f3dc4f3401b546ec169c9fbcbe10ce Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 15 Mar 2021 17:10:55 -0700 Subject: [PATCH 167/203] Repeat the method and url in the log line for the userinfo username --- internal/concierge/impersonator/impersonator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 6df28a76..6e5ef6cc 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -270,7 +270,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi } plog.Debug("impersonation proxy servicing request", "method", r.Method, "url", r.URL.String()) - plog.Trace("impersonation proxy servicing request was for user", + plog.Trace("impersonation proxy servicing request was for user", "method", r.Method, "url", r.URL.String(), "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) From 236dbdb2c49e5d2e935eb26b6bc7fcd9a7e43680 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Tue, 16 Mar 2021 12:59:07 -0400 Subject: [PATCH 168/203] impersonator: test UID impersonation and header canonicalization Signed-off-by: Monis Khan --- .../impersonator/impersonator_test.go | 69 +++++++++++++++---- .../httputil/roundtripper/roundtripper.go | 14 ++++ 2 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 internal/httputil/roundtripper/roundtripper.go diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 610bad3f..53efe3a7 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -14,7 +14,7 @@ import ( "time" "github.com/stretchr/testify/require" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -31,6 +31,7 @@ import ( "go.pinniped.dev/internal/certauthority" "go.pinniped.dev/internal/dynamiccert" "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/httputil/roundtripper" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" ) @@ -68,6 +69,7 @@ func TestImpersonator(t *testing.T) { name string clientCert *clientCert clientImpersonateUser rest.ImpersonationConfig + clientMutateHeaders func(http.Header) kubeAPIServerClientBearerTokenFile string kubeAPIServerStatusCode int wantKubeAPIServerRequestHeaders http.Header @@ -144,11 +146,42 @@ func TestImpersonator(t *testing.T) { name: "no bearer token file in Kube API server client config", wantConstructionError: "invalid impersonator loopback rest config has wrong bearer token semantics", }, + { + name: "header canonicalization user header", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientMutateHeaders: func(header http.Header) { + header.Set("imPerSonaTE-USer", "PANDA") + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: `users "PANDA" is forbidden: User "test-username" ` + + `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + }, + { + name: "header canonicalization future UID header", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientMutateHeaders: func(header http.Header) { + header.Set("imPerSonaTE-uid", "007") + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: invalid impersonation", + }, + { + name: "future UID header", + clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), + clientMutateHeaders: func(header http.Header) { + header.Set("Impersonate-Uid", "008") + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: invalid impersonation", + }, } for _, tt := range tests { tt := tt // This is a serial test because the production code binds to the port. t.Run(tt.name, func(t *testing.T) { + // After failing to start and after shutdown, the impersonator port should be available again. + defer requireCanBindToPort(t, port) + if tt.kubeAPIServerStatusCode == 0 { tt.kubeAPIServerStatusCode = http.StatusOK } @@ -156,7 +189,7 @@ func TestImpersonator(t *testing.T) { // Set up a fake Kube API server which will stand in for the real one. The impersonator // will proxy incoming calls to this fake server. testKubeAPIServerWasCalled := false - testKubeAPIServerSawHeaders := http.Header{} + var testKubeAPIServerSawHeaders http.Header testKubeAPIServerCA, testKubeAPIServerURL := testutil.TLSTestServer(t, func(w http.ResponseWriter, r *http.Request) { require.Equal(t, http.MethodGet, r.Method) switch r.URL.Path { @@ -203,8 +236,6 @@ func TestImpersonator(t *testing.T) { if len(tt.wantConstructionError) > 0 { require.EqualError(t, constructionErr, tt.wantConstructionError) require.Nil(t, runner) - // After failing to start, the impersonator port should be available again. - requireCanBindToPort(t, port) // The rest of the test doesn't make sense when you expect a construction error, so stop here. return } @@ -232,6 +263,17 @@ func TestImpersonator(t *testing.T) { // and it should not passed into the impersonator handler func as an authorization header. BearerToken: "must-be-ignored", Impersonate: tt.clientImpersonateUser, + WrapTransport: func(rt http.RoundTripper) http.RoundTripper { + if tt.clientMutateHeaders == nil { + return rt + } + + return roundtripper.Func(func(req *http.Request) (*http.Response, error) { + req = req.Clone(req.Context()) + tt.clientMutateHeaders(req.Header) + return rt.RoundTrip(req) + }) + }, } // Create a real Kube client to make API requests to the impersonator. @@ -243,28 +285,27 @@ func TestImpersonator(t *testing.T) { listResponse, err := client.Kubernetes.CoreV1().Namespaces().List(context.Background(), metav1.ListOptions{}) if len(tt.wantError) > 0 { require.EqualError(t, err, tt.wantError) + require.Equal(t, &corev1.NamespaceList{}, listResponse) } else { require.NoError(t, err) - require.Equal(t, &v1.NamespaceList{ - Items: []v1.Namespace{ + require.Equal(t, &corev1.NamespaceList{ + Items: []corev1.Namespace{ {ObjectMeta: metav1.ObjectMeta{Name: "namespace1"}}, {ObjectMeta: metav1.ObjectMeta{Name: "namespace2"}}, }, }, listResponse) - - // The impersonator should have proxied the request to the fake Kube API server, which should have seen - // the headers of the original request mutated by the impersonator. - require.True(t, testKubeAPIServerWasCalled) - require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) } + // If we expect to see some headers, then the fake KAS should have been called. + require.Equal(t, len(tt.wantKubeAPIServerRequestHeaders) != 0, testKubeAPIServerWasCalled) + // If the impersonator proxied the request to the fake Kube API server, we should see the headers + // of the original request mutated by the impersonator. Otherwise the headers should be nil. + require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) + // Stop the impersonator server. close(stopCh) exitErr := <-errCh require.NoError(t, exitErr) - - // After shutdown, the impersonator port should be available again. - requireCanBindToPort(t, port) }) } } diff --git a/internal/httputil/roundtripper/roundtripper.go b/internal/httputil/roundtripper/roundtripper.go new file mode 100644 index 00000000..d0094b8a --- /dev/null +++ b/internal/httputil/roundtripper/roundtripper.go @@ -0,0 +1,14 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package roundtripper + +import "net/http" + +var _ http.RoundTripper = Func(nil) + +type Func func(*http.Request) (*http.Response, error) + +func (f Func) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} From 897340860b853b185d20885fcd80c1585915a949 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 16 Mar 2021 16:57:28 -0700 Subject: [PATCH 169/203] Small refactor to impersonation proxy integration test --- .../concierge_impersonation_proxy_test.go | 80 ++++++------------- 1 file changed, 24 insertions(+), 56 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index c6b1561a..bb760fe0 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -227,6 +227,27 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "").Kubernetes } t.Run("positive tests", func(t *testing.T) { + // Create an RBAC rule to allow this user to read/write everything. + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + + // Get pods in concierge namespace and pick one. + // this is for tests that require performing actions against a running pod. We use the concierge pod because we already have it handy. + // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. + pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + require.Greater(t, len(pods.Items), 0) + var conciergePod *corev1.Pod + for _, pod := range pods.Items { + pod := pod + if !strings.Contains(pod.Name, "kube-cert-agent") { + conciergePod = &pod + } + } + require.NotNil(t, conciergePod, "could not find a concierge pod") + // Test that the user can perform basic actions through the client with their username and group membership // influencing RBAC checks correctly. t.Run( @@ -248,30 +269,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl namespaceName := createTestNamespace(t, adminClient) - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", }) - // Get pods in concierge namespace and pick one. - // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. - pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - require.Greater(t, len(pods.Items), 0) - var conciergePod *corev1.Pod - for _, pod := range pods.Items { - pod := pod - if !strings.Contains(pod.Name, "kube-cert-agent") { - conciergePod = &pod - } - } - require.NotNil(t, conciergePod, "could not find a concierge pod") - // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() @@ -285,6 +287,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() + // wait to see if we time out. time.Sleep(70 * time.Second) require.Eventually(t, func() bool { @@ -312,11 +315,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := createTestNamespace(t, adminClient) - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", @@ -441,11 +439,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { t.Parallel() - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, - ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Namespace: env.ConciergeNamespace, Verb: "get", Group: "", Version: "v1", Resource: "secrets", @@ -570,11 +563,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Run("kubectl as a client", func(t *testing.T) { t.Parallel() - // Create an RBAC rule to allow this user to read/write everything. - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, - ) // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Verb: "get", Group: "", Version: "v1", Resource: "namespaces", @@ -582,20 +570,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) - // Get pods in concierge namespace and pick one. - // We want to make sure it's a concierge pod (not cert agent), because we need to be able to "exec echo" and port-forward a running port. - pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{}) - require.NoError(t, err) - require.Greater(t, len(pods.Items), 0) - var conciergePod *corev1.Pod - for _, pod := range pods.Items { - pod := pod - if !strings.Contains(pod.Name, "kube-cert-agent") { - conciergePod = &pod - } - } - require.NotNil(t, conciergePod, "could not find a concierge pod") - // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) @@ -663,10 +637,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Parallel() namespaceName := createTestNamespace(t, adminClient) - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) + // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", @@ -743,10 +714,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Parallel() namespaceName := createTestNamespace(t, adminClient) - library.CreateTestClusterRoleBinding(t, - rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, - rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "cluster-admin"}, - ) + // Wait for the above RBAC rule to take effect. library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", From aa79bc76098a44b7ca4c004cb251a9106fde2835 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 10:14:11 -0400 Subject: [PATCH 170/203] internal/concierge/impersonator: ensure log statement is printed When the frontend connection to our proxy is closed, the proxy falls through to a panic(), which means the HTTP handler goroutine is killed, so we were not seeing this log statement. Signed-off-by: Andrew Keesler --- internal/concierge/impersonator/impersonator.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 6e5ef6cc..a95dc770 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -270,6 +270,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi } plog.Debug("impersonation proxy servicing request", "method", r.Method, "url", r.URL.String()) + defer plog.Debug("impersonation proxy finished servicing request", "method", r.Method, "url", r.URL.String()) plog.Trace("impersonation proxy servicing request was for user", "method", r.Method, "url", r.URL.String(), "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) @@ -278,8 +279,6 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi reverseProxy.Transport = rt reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line reverseProxy.ServeHTTP(w, r) - - plog.Debug("impersonation proxy finished servicing request", "method", r.Method, "url", r.URL.String()) }) }, nil } From 205c22ddbec918542a24881d106acefc7de9ba2a Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Thu, 18 Mar 2021 10:28:16 -0400 Subject: [PATCH 171/203] impersonator config: catch panics when running impersonator Signed-off-by: Monis Khan --- internal/controller/impersonatorconfig/impersonator_config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 4fc140d8..72cd4d68 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/util/clock" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/intstr" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/client-go/kubernetes" @@ -383,6 +384,8 @@ func (c *impersonatorConfigController) ensureImpersonatorIsStarted(syncCtx contr // startImpersonatorFunc will block until the server shuts down (or fails to start), so run it in the background. go func() { + defer utilruntime.HandleCrash() + // The server has stopped, so enqueue ourselves for another sync, // so we can try to start the server again as quickly as possible. defer syncCtx.Queue.AddRateLimited(syncCtx.Key) From 257d69045db5731605c1867f3abd223fade9e902 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 10:40:59 -0400 Subject: [PATCH 172/203] Reuse internal/concierge/scheme Signed-off-by: Andrew Keesler --- cmd/pinniped/cmd/whoami.go | 116 +--------------------------------- test/integration/cli_test.go | 118 +---------------------------------- 2 files changed, 4 insertions(+), 230 deletions(-) diff --git a/cmd/pinniped/cmd/whoami.go b/cmd/pinniped/cmd/whoami.go index bf973981..de7399f9 100644 --- a/cmd/pinniped/cmd/whoami.go +++ b/cmd/pinniped/cmd/whoami.go @@ -15,18 +15,13 @@ import ( "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/tools/clientcmd" - identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/groupsuffix" "go.pinniped.dev/internal/here" - "go.pinniped.dev/internal/plog" ) //nolint: gochecknoinits @@ -158,7 +153,7 @@ func writeWhoamiOutputYAML(output io.Writer, apiGroupSuffix string, whoAmI *iden } func serialize(output io.Writer, apiGroupSuffix string, whoAmI *identityv1alpha1.WhoAmIRequest, contentType string) error { - scheme, _, identityGV := conciergeschemeNew(apiGroupSuffix) + scheme, _, identityGV := conciergescheme.New(apiGroupSuffix) codecs := serializer.NewCodecFactory(scheme) respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), contentType) if !ok { @@ -189,110 +184,3 @@ func prettyStrings(ss []string) string { } return b.String() } - -// conciergeschemeNew is a temporary private function to stand in place for -// "go.pinniped.dev/internal/concierge/scheme".New until the later function is merged to main. -func conciergeschemeNew(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) { - // standard set up of the server side scheme - scheme := runtime.NewScheme() - - // add the options to empty v1 - metav1.AddToGroupVersion(scheme, metav1.Unversioned) - - // nothing fancy is required if using the standard group suffix - if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix { - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.AddToScheme, - loginapi.AddToScheme, - identityv1alpha1.AddToScheme, - identityapi.AddToScheme, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion - } - - loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix) - - addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme) - addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme) - - // manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.RegisterConversions, - loginv1alpha1.RegisterDefaults, - identityv1alpha1.RegisterConversions, - identityv1alpha1.RegisterDefaults, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - - // we do not want to return errors from the scheme and instead would prefer to defer - // to the REST storage layer for consistency. The simplest way to do this is to force - // a cache miss from the authenticator cache. Kube API groups are validated via the - // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never - // to be in the authenticator cache. Add a timestamp just to be extra sure. - const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" - authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() - - // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest - // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites - // any previously registered defaulting function. Thus to make sure that we catch - // a situation where we add a defaulting func, we attempt to call it here with a nil - // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no - // defaulting func registered, but it will almost certainly panic if one is added. - scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) - - // on incoming requests, restore the authenticator API group to the standard group - // note that we are responsible for duplicating this logic for every external API version - scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { - credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) - - if credentialRequest.Spec.Authenticator.APIGroup == nil { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - if !ok { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup - }) - - return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData) -} - -func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) { - // we need a temporary place to register our types to avoid double registering them - tmpScheme := runtime.NewScheme() - schemeBuilder := runtime.NewSchemeBuilder(funcs...) - utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme)) - - for gvk := range tmpScheme.AllKnownTypes() { - if gvk.GroupVersion() == metav1.Unversioned { - continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore - } - - if gvk.Group != oldGroup { - panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error - } - - obj, err := tmpScheme.New(gvk) - if err != nil { - panic(err) // programmer error, scheme internal code is broken - } - newGVK := schema.GroupVersionKind{ - Group: newGroup, - Version: gvk.Version, - Kind: gvk.Kind, - } - - // register the existing type but with the new group in the correct scheme - scheme.AddKnownTypeWithName(newGVK, obj) - } -} diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index 2934b55c..5f93ed40 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -24,19 +24,12 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "gopkg.in/square/go-jose.v2" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" - utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientauthenticationv1beta1 "k8s.io/client-go/pkg/apis/clientauthentication/v1beta1" - identityapi "go.pinniped.dev/generated/latest/apis/concierge/identity" identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" - loginapi "go.pinniped.dev/generated/latest/apis/concierge/login" - loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" - "go.pinniped.dev/internal/groupsuffix" - "go.pinniped.dev/internal/plog" + conciergescheme "go.pinniped.dev/internal/concierge/scheme" "go.pinniped.dev/internal/testutil" "go.pinniped.dev/pkg/oidcclient" "go.pinniped.dev/pkg/oidcclient/filesession" @@ -173,7 +166,7 @@ func assertWhoami(ctx context.Context, t *testing.T, useProxy bool, pinnipedExe, func deserializeWhoAmIRequest(t *testing.T, data string, apiGroupSuffix string) *identityv1alpha1.WhoAmIRequest { t.Helper() - scheme, _, _ := conciergeschemeNew(apiGroupSuffix) + scheme, _, _ := conciergescheme.New(apiGroupSuffix) codecs := serializer.NewCodecFactory(scheme) respInfo, ok := runtime.SerializerInfoForMediaType(codecs.SupportedMediaTypes(), runtime.ContentTypeYAML) require.True(t, ok) @@ -420,110 +413,3 @@ func oidcLoginCommand(ctx context.Context, t *testing.T, pinnipedExe string, ses cmd.Env = append(os.Environ(), env.ProxyEnv()...) return cmd } - -// conciergeschemeNew is a temporary private function to stand in place for -// "go.pinniped.dev/internal/concierge/scheme".New until the later function is merged to main. -func conciergeschemeNew(apiGroupSuffix string) (_ *runtime.Scheme, login, identity schema.GroupVersion) { - // standard set up of the server side scheme - scheme := runtime.NewScheme() - - // add the options to empty v1 - metav1.AddToGroupVersion(scheme, metav1.Unversioned) - - // nothing fancy is required if using the standard group suffix - if apiGroupSuffix == groupsuffix.PinnipedDefaultSuffix { - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.AddToScheme, - loginapi.AddToScheme, - identityv1alpha1.AddToScheme, - identityapi.AddToScheme, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - return scheme, loginv1alpha1.SchemeGroupVersion, identityv1alpha1.SchemeGroupVersion - } - - loginConciergeGroupData, identityConciergeGroupData := groupsuffix.ConciergeAggregatedGroups(apiGroupSuffix) - - addToSchemeAtNewGroup(scheme, loginv1alpha1.GroupName, loginConciergeGroupData.Group, loginv1alpha1.AddToScheme, loginapi.AddToScheme) - addToSchemeAtNewGroup(scheme, identityv1alpha1.GroupName, identityConciergeGroupData.Group, identityv1alpha1.AddToScheme, identityapi.AddToScheme) - - // manually register conversions and defaulting into the correct scheme since we cannot directly call AddToScheme - schemeBuilder := runtime.NewSchemeBuilder( - loginv1alpha1.RegisterConversions, - loginv1alpha1.RegisterDefaults, - identityv1alpha1.RegisterConversions, - identityv1alpha1.RegisterDefaults, - ) - utilruntime.Must(schemeBuilder.AddToScheme(scheme)) - - // we do not want to return errors from the scheme and instead would prefer to defer - // to the REST storage layer for consistency. The simplest way to do this is to force - // a cache miss from the authenticator cache. Kube API groups are validated via the - // IsDNS1123Subdomain func thus we can easily create a group that is guaranteed never - // to be in the authenticator cache. Add a timestamp just to be extra sure. - const authenticatorCacheMissPrefix = "_INVALID_API_GROUP_" - authenticatorCacheMiss := authenticatorCacheMissPrefix + time.Now().UTC().String() - - // we do not have any defaulting functions for *loginv1alpha1.TokenCredentialRequest - // today, but we may have some in the future. Calling AddTypeDefaultingFunc overwrites - // any previously registered defaulting function. Thus to make sure that we catch - // a situation where we add a defaulting func, we attempt to call it here with a nil - // *loginv1alpha1.TokenCredentialRequest. This will do nothing when there is no - // defaulting func registered, but it will almost certainly panic if one is added. - scheme.Default((*loginv1alpha1.TokenCredentialRequest)(nil)) - - // on incoming requests, restore the authenticator API group to the standard group - // note that we are responsible for duplicating this logic for every external API version - scheme.AddTypeDefaultingFunc(&loginv1alpha1.TokenCredentialRequest{}, func(obj interface{}) { - credentialRequest := obj.(*loginv1alpha1.TokenCredentialRequest) - - if credentialRequest.Spec.Authenticator.APIGroup == nil { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, nil group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - restoredGroup, ok := groupsuffix.Unreplace(*credentialRequest.Spec.Authenticator.APIGroup, apiGroupSuffix) - if !ok { - // force a cache miss because this is an invalid request - plog.Debug("invalid token credential request, wrong group", "authenticator", credentialRequest.Spec.Authenticator) - credentialRequest.Spec.Authenticator.APIGroup = &authenticatorCacheMiss - return - } - - credentialRequest.Spec.Authenticator.APIGroup = &restoredGroup - }) - - return scheme, schema.GroupVersion(loginConciergeGroupData), schema.GroupVersion(identityConciergeGroupData) -} - -func addToSchemeAtNewGroup(scheme *runtime.Scheme, oldGroup, newGroup string, funcs ...func(*runtime.Scheme) error) { - // we need a temporary place to register our types to avoid double registering them - tmpScheme := runtime.NewScheme() - schemeBuilder := runtime.NewSchemeBuilder(funcs...) - utilruntime.Must(schemeBuilder.AddToScheme(tmpScheme)) - - for gvk := range tmpScheme.AllKnownTypes() { - if gvk.GroupVersion() == metav1.Unversioned { - continue // metav1.AddToGroupVersion registers types outside of our aggregated API group that we need to ignore - } - - if gvk.Group != oldGroup { - panic(fmt.Errorf("tmp scheme has type not in the old aggregated API group %s: %s", oldGroup, gvk)) // programmer error - } - - obj, err := tmpScheme.New(gvk) - if err != nil { - panic(err) // programmer error, scheme internal code is broken - } - newGVK := schema.GroupVersionKind{ - Group: newGroup, - Version: gvk.Version, - Kind: gvk.Kind, - } - - // register the existing type but with the new group in the correct scheme - scheme.AddKnownTypeWithName(newGVK, obj) - } -} From 120e46b5f7b52d099acd6c4c72b623e00a183099 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Thu, 18 Mar 2021 11:24:02 -0400 Subject: [PATCH 173/203] test/integration: fix race condition Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index bb760fe0..ffede110 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -281,7 +281,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl portForwardCmd.Env = envVarsWithProxy // start, but don't wait for the command to finish - err = portForwardCmd.Start() + err := portForwardCmd.Start() require.NoError(t, err, `"kubectl port-forward" failed`) go func() { assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) @@ -342,7 +342,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // Test "create" verb through the impersonation proxy. - _, err = impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, + _, err := impersonationProxyKubeClient().CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, ) @@ -812,11 +812,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl configMap := configMapForConfig(t, env, impersonator.Config{Mode: impersonator.ModeDisabled}) if env.HasCapability(library.HasExternalLoadBalancerProvider) { t.Logf("creating configmap %s", configMap.Name) - _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) + _, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, &configMap, metav1.CreateOptions{}) require.NoError(t, err) } else { t.Logf("updating configmap %s", configMap.Name) - _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) + _, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Update(ctx, &configMap, metav1.UpdateOptions{}) require.NoError(t, err) } @@ -838,7 +838,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Eventually(t, func() bool { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. - _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err := impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) return err.Error() == serviceUnavailableViaSquidError }, 20*time.Second, 500*time.Millisecond) } @@ -846,14 +846,14 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up // when we disable the impersonator. require.Eventually(t, func() bool { - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + _, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) return k8serrors.IsNotFound(err) }, 10*time.Second, 250*time.Millisecond) // Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this // around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when // possible because they make their way into kubeconfigs on client machines. - _, err = adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) + _, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) require.NoError(t, err) // At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to From e4bf6e068fd756986ae035c897609e4e4d582c1b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 18 Mar 2021 10:00:06 -0700 Subject: [PATCH 174/203] Add a comment to impersonator.go --- internal/concierge/impersonator/impersonator.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index a95dc770..3d7ad324 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -270,11 +270,14 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi } plog.Debug("impersonation proxy servicing request", "method", r.Method, "url", r.URL.String()) - defer plog.Debug("impersonation proxy finished servicing request", "method", r.Method, "url", r.URL.String()) plog.Trace("impersonation proxy servicing request was for user", "method", r.Method, "url", r.URL.String(), "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) + // The proxy library used below will panic when the client disconnects abruptly, so in order to + // assure that this log message is always printed at the end of this func, it must be deferred. + defer plog.Debug("impersonation proxy finished servicing request", "method", r.Method, "url", r.URL.String()) + reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) reverseProxy.Transport = rt reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line From bd8c243636ca636257fce70ac02b0517f42ce0f3 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Thu, 18 Mar 2021 10:44:37 -0700 Subject: [PATCH 175/203] concierge_impersonation_proxy_test.go: small refactor --- .../concierge_impersonation_proxy_test.go | 41 ++++--------------- 1 file changed, 7 insertions(+), 34 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index ffede110..7ad6c7c2 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -226,12 +226,17 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl impersonationProxyKubeClient := func() kubernetes.Interface { return newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "").Kubernetes } + t.Run("positive tests", func(t *testing.T) { // Create an RBAC rule to allow this user to read/write everything. library.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, ) + // Wait for the above RBAC rule to take effect. + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + Verb: "get", Group: "", Version: "v1", Resource: "namespaces", + }) // Get pods in concierge namespace and pick one. // this is for tests that require performing actions against a running pod. We use the concierge pod because we already have it handy. @@ -267,13 +272,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) - namespaceName := createTestNamespace(t, adminClient) - - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) - // run the kubectl port-forward command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() @@ -294,7 +292,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // then run curl something against it timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "https://127.0.0.1:8443") + curlCmd := exec.CommandContext(timeout, "curl", "-k", "-s", "https://127.0.0.1:8443") var curlStdOut, curlStdErr bytes.Buffer curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr @@ -306,7 +304,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // we expect this to 403, but all we care is that it gets through return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") - }, 5*time.Minute, 500*time.Millisecond) + }, 1*time.Minute, 500*time.Millisecond) }) t.Run("using and watching all the basic verbs", func(t *testing.T) { @@ -315,11 +313,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := createTestNamespace(t, adminClient) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) - // Create and start informer to exercise the "watch" verb for us. informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( impersonationProxyKubeClient(), @@ -439,11 +432,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { t.Parallel() - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: env.ConciergeNamespace, Verb: "get", Group: "", Version: "v1", Resource: "secrets", - }) - // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. doubleImpersonationKubeClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes @@ -563,11 +551,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Run("kubectl as a client", func(t *testing.T) { t.Parallel() - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Verb: "get", Group: "", Version: "v1", Resource: "namespaces", - }) - kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) // Try "kubectl exec" through the impersonation proxy. @@ -638,11 +621,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl namespaceName := createTestNamespace(t, adminClient) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) - impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) require.NoError(t, err) @@ -715,11 +693,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl namespaceName := createTestNamespace(t, adminClient) - // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ - Namespace: namespaceName, Verb: "create", Group: "", Version: "v1", Resource: "configmaps", - }) - wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" wantConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, From c22ac17dfed914f5ed2aeaef3e697a1e9fda1033 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 15:32:33 -0400 Subject: [PATCH 176/203] internal/concierge/impersonator: use http/2.0 as much as we can Signed-off-by: Monis Khan --- .../concierge/impersonator/impersonator.go | 40 ++++++++++++++++--- .../impersonator/impersonator_test.go | 9 ++++- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 3d7ad324..311b2fc4 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -217,15 +217,14 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err) } - kubeTransportConfig, err := restConfig.TransportConfig() + http1RoundTripper, err := getTransportForProtocol(restConfig, "http/1.1") if err != nil { - return nil, fmt.Errorf("could not get in-cluster transport config: %w", err) + return nil, fmt.Errorf("could not get http/1.1 round tripper: %w", err) } - kubeTransportConfig.TLS.NextProtos = []string{"http/1.1"} // TODO huh? - kubeRoundTripper, err := transport.New(kubeTransportConfig) + http2RoundTripper, err := getTransportForProtocol(restConfig, "h2") if err != nil { - return nil, fmt.Errorf("could not get in-cluster transport: %w", err) + return nil, fmt.Errorf("could not get http/2.0 round tripper: %w", err) } return func(c *genericapiserver.Config) http.Handler { @@ -259,7 +258,26 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi return } - rt, err := getTransportForUser(userInfo, kubeRoundTripper) + reqInfo, ok := request.RequestInfoFrom(r.Context()) + if !ok { + plog.Warning("aggregated API server logic did not set request info but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + newInternalErrResponse(w, r, c.Serializer, "invalid request info") + return + } + + // when we are running regular requests (e.g., CRUD) we should always be able to use HTTP/2.0 + // since KAS always supports that and it goes through proxies just fine. for long running + // requests (e.g., proxy, watch), we know they use http/1.1 with an upgrade to + // websockets/SPDY (this upgrade is NEVER to HTTP/2.0 as the KAS does not support that). + baseRT := http2RoundTripper + if c.LongRunningFunc(r, reqInfo) { + baseRT = http1RoundTripper + } + + rt, err := getTransportForUser(userInfo, baseRT) if err != nil { plog.WarningErr("rejecting request as we cannot act as the current user", err, "url", r.URL.String(), @@ -335,3 +353,13 @@ func newStatusErrResponse(w http.ResponseWriter, r *http.Request, s runtime.Nego gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} responsewriters.ErrorNegotiated(err, s, gv, w, r) } + +func getTransportForProtocol(restConfig *rest.Config, protocol string) (http.RoundTripper, error) { + transportConfig, err := restConfig.TransportConfig() + if err != nil { + return nil, fmt.Errorf("could not get in-cluster transport config: %w", err) + } + transportConfig.TLS.NextProtos = []string{protocol} + + return transport.New(transportConfig) +} diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 53efe3a7..9fccbe9e 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -5,6 +5,7 @@ package impersonator import ( "context" + "math/rand" "net" "net/http" "net/http/httptest" @@ -359,7 +360,7 @@ func TestImpersonatorHTTPHandler(t *testing.T) { ExecProvider: &api.ExecConfig{}, AuthProvider: &api.AuthProviderConfig{}, }, - wantCreationErr: "could not get in-cluster transport config: execProvider and authProvider cannot be used in combination", + wantCreationErr: "could not get http/1.1 round tripper: could not get in-cluster transport config: execProvider and authProvider cannot be used in combination", }, { name: "fail to get transport from config", @@ -369,7 +370,7 @@ func TestImpersonatorHTTPHandler(t *testing.T) { Transport: http.DefaultTransport, TLSClientConfig: rest.TLSClientConfig{Insecure: true}, }, - wantCreationErr: "could not get in-cluster transport: using a custom transport with TLS certificate options or the insecure flag is not allowed", + wantCreationErr: "could not get http/1.1 round tripper: using a custom transport with TLS certificate options or the insecure flag is not allowed", }, { name: "Impersonate-User header already in request", @@ -513,6 +514,10 @@ func TestImpersonatorHTTPHandler(t *testing.T) { metav1.AddToGroupVersion(scheme, metav1.Unversioned) codecs := serializer.NewCodecFactory(scheme) serverConfig := genericapiserver.NewRecommendedConfig(codecs) + serverConfig.Config.LongRunningFunc = func(_ *http.Request, _ *request.RequestInfo) bool { + // take the HTTP/2.0 vs HTTP/1.1 branch randomly to make sure we exercise both branches + return rand.Int()%2 == 0 //nolint:gosec // we don't care whether this is cryptographically secure or not + } w := httptest.NewRecorder() requestBeforeServe := tt.request.Clone(tt.request.Context()) From dae62929e0f9c89b8c4ff4eae29e23419cdb99ee Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 15:33:47 -0400 Subject: [PATCH 177/203] test/integration: error assertions pass w/ and w/o middleware In the case where we are using middleware (e.g., when the api group is different) in our kubeclient, these error messages have a "...middleware request for..." bit in the middle. Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 7ad6c7c2..87a35ea5 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -196,7 +196,9 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Check that we can't use the impersonation proxy to execute kubectl commands yet. _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - require.EqualError(t, err, serviceUnavailableViaSquidError) + require.Error(t, err) + require.Contains(t, err.Error(), proxyServiceEndpoint) + require.Contains(t, err.Error(), ": Service Unavailable") // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). configMap := configMapForConfig(t, env, impersonator.Config{ From 14a28bec245271b28d2d737d1fc582d448bc7bed Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 16:34:30 -0400 Subject: [PATCH 178/203] test/integration: fix second assertion from dae62929 Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 87a35ea5..8c5754ff 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -95,8 +95,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) - // The error message that will be returned by squid when the impersonation proxy port inside the cluster is not listening. - serviceUnavailableViaSquidError := fmt.Sprintf(`Get "https://%s/api/v1/namespaces": Service Unavailable`, proxyServiceEndpoint) var mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest refreshCredential := func() *loginv1alpha1.ClusterCredential { @@ -196,9 +194,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Check that we can't use the impersonation proxy to execute kubectl commands yet. _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - require.Error(t, err) - require.Contains(t, err.Error(), proxyServiceEndpoint) - require.Contains(t, err.Error(), ": Service Unavailable") + isErr, message := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) + require.Truef(t, isErr, "wanted error %q to be service unavailable via squid error, but: %s", err, message) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). configMap := configMapForConfig(t, env, impersonator.Config{ @@ -814,7 +811,8 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. _, err := impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) - return err.Error() == serviceUnavailableViaSquidError + isErr, _ := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) + return isErr }, 20*time.Second, 500*time.Millisecond) } @@ -1166,3 +1164,22 @@ type watchJSON struct { Type watch.EventType `json:"type,omitempty"` Object json.RawMessage `json:"object,omitempty"` } + +// requireServiceUnavailableViaSquidError returns whether the provided err is the error that is +// returned by squid when the impersonation proxy port inside the cluster is not listening. +func isServiceUnavailableViaSquidError(err error, proxyServiceEndpoint string) (bool, string) { + if err == nil { + return false, "error is nil" + } + + for _, wantContains := range []string{ + fmt.Sprintf(`Get "https://%s/api/v1/namespaces"`, proxyServiceEndpoint), + ": Service Unavailable", + } { + if !strings.Contains(err.Error(), wantContains) { + return false, fmt.Sprintf("error does not contain %q", wantContains) + } + } + + return true, "" +} From f2a48aee2b1e5acc0272fad2f36f5b5358e3f64a Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 17:48:00 -0400 Subject: [PATCH 179/203] test/integration: increase timeout to a minute to see if it helps Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 8c5754ff..0cd750d3 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -606,7 +606,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // see that we can read stdout and it spits out stdin output back to us wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) - require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*30, time.Second, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) + require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*60, time.Second*5, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) // close stdin and attach process should exit err = attachStdin.Close() From 1a9922d050721e0b71bb8ac5ab753903e66bfc0f Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Thu, 18 Mar 2021 17:53:14 -0400 Subject: [PATCH 180/203] test/integration: poll more quickly in f2a48aee Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 0cd750d3..f2b6bb12 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -606,7 +606,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // see that we can read stdout and it spits out stdin output back to us wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) - require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*60, time.Second*5, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) + require.Eventuallyf(t, func() bool { return attachStdout.String() == wantAttachStdout }, time.Second*60, time.Millisecond*250, `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String()) // close stdin and attach process should exit err = attachStdin.Close() From ebe01a5aef6e9f5aa9e35e0c843bb4de5d107cdc Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 19 Mar 2021 09:59:24 -0400 Subject: [PATCH 181/203] test/integration: catch early 'kubectl attach' return Signed-off-by: Andrew Keesler --- .../concierge_impersonation_proxy_test.go | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index f2b6bb12..adda98f0 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -599,6 +599,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // start but don't wait for the attach command err = attachCmd.Start() require.NoError(t, err) + attachExitCh := make(chan struct{}) + go func() { + assert.NoError(t, attachCmd.Wait()) + close(attachExitCh) + }() // write to stdin on the attach process _, err = attachStdin.Write([]byte(echoString + "\n")) @@ -611,8 +616,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // close stdin and attach process should exit err = attachStdin.Close() require.NoError(t, err) - err = attachCmd.Wait() - require.NoError(t, err) + requireClose(t, attachExitCh, time.Second*20) }) t.Run("websocket client", func(t *testing.T) { @@ -1183,3 +1187,16 @@ func isServiceUnavailableViaSquidError(err error, proxyServiceEndpoint string) ( return true, "" } + +func requireClose(t *testing.T, c chan struct{}, timeout time.Duration) { + t.Helper() + timer := time.NewTimer(timeout) + select { + case <-c: + if !timer.Stop() { + <-timer.C + } + case <-timer.C: + require.FailNow(t, "failed to receive from channel within "+timeout.String()) + } +} From 61548838559c7ff94728491b284cf3cf3e6bb68a Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 19 Mar 2021 10:42:11 -0400 Subject: [PATCH 182/203] test/integration: add temporary debug 'kubectl attach' logging Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index adda98f0..6593080c 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -591,7 +591,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name) + attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", namespaceName, attachPod.Name, "-v=10") attachCmd.Env = envVarsWithProxy attachStdin, err := attachCmd.StdinPipe() require.NoError(t, err) From ebd5e45fa60a8b7ce73bad03cd955c380686693b Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 19 Mar 2021 12:54:37 -0400 Subject: [PATCH 183/203] test/integration: wait for convergence at end of impersonation test Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 6593080c..3e0002a3 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -156,7 +156,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // At the end of the test, clean up the ConfigMap. t.Cleanup(func() { - ctx, cancel = context.WithTimeout(context.Background(), time.Minute) + ctx, cancel = context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() // Delete any version that was created by this test. @@ -175,6 +175,14 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl _, err = adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Create(ctx, oldConfigMap, metav1.CreateOptions{}) require.NoError(t, err) } + + // If we are running on an environment that has a load balancer, expect that the + // CredentialIssuer will be updated eventually with a successful impersonation proxy frontend. + // We do this to ensure that future tests that use the impersonation proxy (e.g., + // TestE2EFullIntegration) will start with a known-good state. + if env.HasCapability(library.HasExternalLoadBalancerProvider) { + performImpersonatorDiscovery(ctx, t, env, adminConciergeClient) + } }) if env.HasCapability(library.HasExternalLoadBalancerProvider) { //nolint:nestif // come on... it's just a test From f73c70d8f93a142106afbc0ee1d78662ab129f0b Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 19 Mar 2021 13:18:10 -0400 Subject: [PATCH 184/203] test/integration: use Ryan's 20x rule to harden simple access tests Signed-off-by: Andrew Keesler --- test/library/access.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/library/access.go b/test/library/access.go index e953cff8..46c31bd3 100644 --- a/test/library/access.go +++ b/test/library/access.go @@ -22,7 +22,7 @@ import ( const ( accessRetryInterval = 250 * time.Millisecond - accessRetryTimeout = 10 * time.Second + accessRetryTimeout = 60 * time.Second ) // AccessAsUserTest runs a generic test in which a clientUnderTest operating with username From 27490446251867f19a6b0f787094e1d2e1b07a3b Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Fri, 19 Mar 2021 13:31:28 -0400 Subject: [PATCH 185/203] test/integration: unparallelize impersonation kubectl test Maybe this will cut down on flakes we see in CI? Signed-off-by: Andrew Keesler --- test/integration/concierge_impersonation_proxy_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 3e0002a3..a0a945d6 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -556,8 +556,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("kubectl as a client", func(t *testing.T) { - t.Parallel() - kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) // Try "kubectl exec" through the impersonation proxy. From c03fe2d1fe09c046c23d229b952d0aacf03ec046 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Fri, 19 Mar 2021 13:39:55 -0400 Subject: [PATCH 186/203] Use http2 for all non-upgrade requests Instead of using the LongRunningFunc to determine if we can safely use http2, follow the same logic as the aggregation proxy and only use http2 when the request is not an upgrade. Signed-off-by: Monis Khan --- .../concierge/impersonator/impersonator.go | 57 ++++++++++------ .../impersonator/impersonator_test.go | 65 +++++++++++++++---- 2 files changed, 92 insertions(+), 30 deletions(-) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 311b2fc4..615e6ea8 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -18,9 +18,11 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/endpoints/filterlatency" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/apiserver/pkg/endpoints/request" genericapirequest "k8s.io/apiserver/pkg/endpoints/request" @@ -148,9 +150,25 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler { // We ignore the passed in handler because we never have any REST APIs to delegate to. - handler := impersonationProxyFunc(c) + // This means we are ignoring the admission, discovery, REST storage, etc layers. + doNotDelegate := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {}) + + // Impersonation proxy business logic with timing information. + impersonationProxyCompleted := filterlatency.TrackCompleted(doNotDelegate) + impersonationProxy := impersonationProxyFunc(c) + handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer impersonationProxyCompleted.ServeHTTP(w, r) + impersonationProxy.ServeHTTP(w, r) + })) + handler = filterlatency.TrackStarted(handler, "impersonationproxy") + + // The standard Kube handler chain (authn, authz, impersonation, audit, etc). + // See the genericapiserver.DefaultBuildHandlerChain func for details. handler = defaultBuildHandlerChainFunc(handler, c) + + // Always set security headers so browsers do the right thing. handler = securityheader.Wrap(handler) + return handler } @@ -258,22 +276,11 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi return } - reqInfo, ok := request.RequestInfoFrom(r.Context()) - if !ok { - plog.Warning("aggregated API server logic did not set request info but it is always supposed to do so", - "url", r.URL.String(), - "method", r.Method, - ) - newInternalErrResponse(w, r, c.Serializer, "invalid request info") - return - } - - // when we are running regular requests (e.g., CRUD) we should always be able to use HTTP/2.0 - // since KAS always supports that and it goes through proxies just fine. for long running - // requests (e.g., proxy, watch), we know they use http/1.1 with an upgrade to - // websockets/SPDY (this upgrade is NEVER to HTTP/2.0 as the KAS does not support that). + // KAS only supports upgrades via http/1.1 to websockets/SPDY (upgrades never use http/2.0) + // Thus we default to using http/2.0 when the request is not an upgrade, otherwise we use http/1.1 baseRT := http2RoundTripper - if c.LongRunningFunc(r, reqInfo) { + isUpgradeRequest := httpstream.IsUpgradeRequest(r) + if isUpgradeRequest { baseRT = http1RoundTripper } @@ -282,19 +289,31 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi plog.WarningErr("rejecting request as we cannot act as the current user", err, "url", r.URL.String(), "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, ) newInternalErrResponse(w, r, c.Serializer, "unimplemented functionality - unable to act as current user") return } - plog.Debug("impersonation proxy servicing request", "method", r.Method, "url", r.URL.String()) - plog.Trace("impersonation proxy servicing request was for user", "method", r.Method, "url", r.URL.String(), + plog.Debug("impersonation proxy servicing request", + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, + ) + plog.Trace("impersonation proxy servicing request was for user", + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, "username", userInfo.GetName(), // this info leak seems fine for trace level logs ) // The proxy library used below will panic when the client disconnects abruptly, so in order to // assure that this log message is always printed at the end of this func, it must be deferred. - defer plog.Debug("impersonation proxy finished servicing request", "method", r.Method, "url", r.URL.String()) + defer plog.Debug("impersonation proxy finished servicing request", + "url", r.URL.String(), + "method", r.Method, + "isUpgradeRequest", isUpgradeRequest, + ) reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) reverseProxy.Transport = rt diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 9fccbe9e..2999236b 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -19,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/util/httpstream" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" @@ -71,6 +72,7 @@ func TestImpersonator(t *testing.T) { clientCert *clientCert clientImpersonateUser rest.ImpersonationConfig clientMutateHeaders func(http.Header) + clientNextProtos []string kubeAPIServerClientBearerTokenFile string kubeAPIServerStatusCode int wantKubeAPIServerRequestHeaders http.Header @@ -91,6 +93,31 @@ func TestImpersonator(t *testing.T) { "X-Forwarded-For": {"127.0.0.1"}, }, }, + { + name: "happy path with upgrade", + clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + clientMutateHeaders: func(header http.Header) { + header.Add("Connection", "Upgrade") + header.Add("Upgrade", "spdy/3.1") + + if ok := httpstream.IsUpgradeRequest(&http.Request{Header: header}); !ok { + panic("request must be upgrade in this test") + } + }, + clientNextProtos: []string{"http/1.1"}, // we need to use http1 as http2 does not support upgrades, see http2checkConnHeaders + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username2"}, + "Impersonate-Group": {"test-group3", "test-group4", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + "Connection": {"Upgrade"}, + "Upgrade": {"spdy/3.1"}, + }, + }, { name: "user is authenticated but the kube API request returns an error", kubeAPIServerStatusCode: http.StatusNotFound, @@ -255,9 +282,10 @@ func TestImpersonator(t *testing.T) { clientKubeconfig := &rest.Config{ Host: "https://127.0.0.1:" + strconv.Itoa(port), TLSClientConfig: rest.TLSClientConfig{ - CAData: ca.Bundle(), - CertData: tt.clientCert.certPEM, - KeyData: tt.clientCert.keyPEM, + CAData: ca.Bundle(), + CertData: tt.clientCert.certPEM, + KeyData: tt.clientCert.keyPEM, + NextProtos: tt.clientNextProtos, }, UserAgent: "test-agent", // BearerToken should be ignored during auth when there are valid client certs, @@ -514,16 +542,31 @@ func TestImpersonatorHTTPHandler(t *testing.T) { metav1.AddToGroupVersion(scheme, metav1.Unversioned) codecs := serializer.NewCodecFactory(scheme) serverConfig := genericapiserver.NewRecommendedConfig(codecs) - serverConfig.Config.LongRunningFunc = func(_ *http.Request, _ *request.RequestInfo) bool { - // take the HTTP/2.0 vs HTTP/1.1 branch randomly to make sure we exercise both branches - return rand.Int()%2 == 0 //nolint:gosec // we don't care whether this is cryptographically secure or not - } w := httptest.NewRecorder() - requestBeforeServe := tt.request.Clone(tt.request.Context()) - impersonatorHTTPHandlerFunc(&serverConfig.Config).ServeHTTP(w, tt.request) - require.Equal(t, requestBeforeServe, tt.request, "ServeHTTP() mutated the request, and it should not per http.Handler docs") + r := tt.request + wantKubeAPIServerRequestHeaders := tt.wantKubeAPIServerRequestHeaders + + // take the isUpgradeRequest branch randomly to make sure we exercise both branches + forceUpgradeRequest := rand.Int()%2 == 0 //nolint:gosec // we do not care if this is cryptographically secure + if forceUpgradeRequest && len(r.Header.Get("Upgrade")) == 0 { + r = r.Clone(r.Context()) + r.Header.Add("Connection", "Upgrade") + r.Header.Add("Upgrade", "spdy/3.1") + + wantKubeAPIServerRequestHeaders = wantKubeAPIServerRequestHeaders.Clone() + if wantKubeAPIServerRequestHeaders == nil { + wantKubeAPIServerRequestHeaders = http.Header{} + } + wantKubeAPIServerRequestHeaders.Add("Connection", "Upgrade") + wantKubeAPIServerRequestHeaders.Add("Upgrade", "spdy/3.1") + } + + requestBeforeServe := r.Clone(r.Context()) + impersonatorHTTPHandlerFunc(&serverConfig.Config).ServeHTTP(w, r) + + require.Equal(t, requestBeforeServe, r, "ServeHTTP() mutated the request, and it should not per http.Handler docs") if tt.wantHTTPStatus != 0 { require.Equalf(t, tt.wantHTTPStatus, w.Code, "fyi, response body was %q", w.Body.String()) } @@ -533,7 +576,7 @@ func TestImpersonatorHTTPHandler(t *testing.T) { if tt.wantHTTPStatus == http.StatusOK || tt.kubeAPIServerStatusCode != http.StatusOK { require.True(t, testKubeAPIServerWasCalled, "Should have proxied the request to the Kube API server, but didn't") - require.Equal(t, tt.wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) + require.Equal(t, wantKubeAPIServerRequestHeaders, testKubeAPIServerSawHeaders) } else { require.False(t, testKubeAPIServerWasCalled, "Should not have proxied the request to the Kube API server, but did") } From f519f0cb098b1b8715ece825d78d60cb7e091ed3 Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Fri, 19 Mar 2021 15:35:06 -0400 Subject: [PATCH 187/203] impersonator: disallow clients from setting the X-Forwarded-For header Signed-off-by: Monis Khan --- .../concierge/impersonator/impersonator.go | 7 ++++ .../impersonator/impersonator_test.go | 34 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index 615e6ea8..f3a7372d 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -19,6 +19,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/httpstream" + utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" @@ -315,6 +316,12 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi "isUpgradeRequest", isUpgradeRequest, ) + // do not allow the client to cause log confusion by spoofing this header + if len(r.Header.Values("X-Forwarded-For")) > 0 { + r = utilnet.CloneRequest(r) + r.Header.Del("X-Forwarded-For") + } + reverseProxy := httputil.NewSingleHostReverseProxy(serverURL) reverseProxy.Transport = rt reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 2999236b..7633c9df 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -118,6 +118,40 @@ func TestImpersonator(t *testing.T) { "Upgrade": {"spdy/3.1"}, }, }, + { + name: "happy path ignores forwarded header", + clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + clientMutateHeaders: func(header http.Header) { + header.Add("X-Forwarded-For", "example.com") + }, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username2"}, + "Impersonate-Group": {"test-group3", "test-group4", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "happy path ignores forwarded header canonicalization", + clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + clientMutateHeaders: func(header http.Header) { + header.Add("x-FORWARDED-for", "example.com") + }, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"test-username2"}, + "Impersonate-Group": {"test-group3", "test-group4", "system:authenticated"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, { name: "user is authenticated but the kube API request returns an error", kubeAPIServerStatusCode: http.StatusNotFound, From d856221f567e9cf3039a722fff2a67cd3183e48c Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 19 Mar 2021 09:36:05 -0700 Subject: [PATCH 188/203] Edit some comments in concierge_impersonation_proxy_test.go --- .../concierge_impersonation_proxy_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index a0a945d6..3b1eec07 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -274,29 +274,29 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl ) } - t.Run("watching for a full minute", func(t *testing.T) { + t.Run("kubectl port-forward and keeping the connection open for over a minute", func(t *testing.T) { t.Parallel() kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) - // run the kubectl port-forward command + // Run the kubectl port-forward command. timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") portForwardCmd.Env = envVarsWithProxy - // start, but don't wait for the command to finish + // Start, but don't wait for the command to finish. err := portForwardCmd.Start() require.NoError(t, err, `"kubectl port-forward" failed`) go func() { assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() - // wait to see if we time out. + // Wait to see if we time out. The default timeout is 60 seconds, but the server should recognize this this + // is going to be a long-running command and keep the connection open as long as the client stays connected. time.Sleep(70 * time.Second) require.Eventually(t, func() bool { - // then run curl something against it timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() curlCmd := exec.CommandContext(timeout, "curl", "-k", "-s", "https://127.0.0.1:8443") @@ -309,7 +309,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl t.Log("curlStdErr: " + curlStdErr.String()) t.Log("stdout: " + curlStdOut.String()) } - // we expect this to 403, but all we care is that it gets through + // We expect this to 403, but all we care is that it gets through. return err == nil && strings.Contains(curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") }, 1*time.Minute, 500*time.Millisecond) }) From 3e50b4e129edd422bbd239f644c538eeabbb3238 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 19 Mar 2021 13:23:28 -0700 Subject: [PATCH 189/203] Add -sS to the curl command in concierge_impersonation_proxy_test.go --- test/integration/concierge_impersonation_proxy_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 3b1eec07..544e2252 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -299,7 +299,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Eventually(t, func() bool { timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "-s", "https://127.0.0.1:8443") + curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:8443") // -sS turns off the progressbar but still prints errors var curlStdOut, curlStdErr bytes.Buffer curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr From 6ff3e4260248917f2612bd9a3247c1d8bb9f1fee Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 16 Mar 2021 10:24:01 -0700 Subject: [PATCH 190/203] Add description of impersonation proxy strategy to docs --- site/content/docs/background/architecture.md | 9 +++++---- .../docs/reference/supported-clusters.md | 17 +++++++++++------ 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/site/content/docs/background/architecture.md b/site/content/docs/background/architecture.md index 5e6469a2..bf258bac 100644 --- a/site/content/docs/background/architecture.md +++ b/site/content/docs/background/architecture.md @@ -1,3 +1,4 @@ + --- title: Architecture description: Dive into the overall design and implementation details of Pinniped. @@ -90,14 +91,14 @@ cleanly enable this integration. Pinniped supports the following cluster integration strategies. -* Pinniped hosts a credential exchange API endpoint via a Kubernetes aggregated API server. +* Kube Cluster Signing Certificate: Pinniped hosts a credential exchange API endpoint via a Kubernetes aggregated API server. This API returns a new cluster-specific credential using the cluster's signing keypair to issue short-lived cluster certificates. (In the future, when the Kubernetes CSR API provides a way to issue short-lived certificates, then the Pinniped credential exchange API will use that instead of using the cluster's signing keypair.) - -More cluster integration strategies are coming soon, which will allow Pinniped to -support more Kubernetes cluster types. +* Impersonation Proxy: Pinniped hosts an [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation) +proxy that performs actions on behalf of the end user. The impersonation proxy accepts and modifies user requests before passing them through to the +Kubernetes API server. ## kubectl Integration diff --git a/site/content/docs/reference/supported-clusters.md b/site/content/docs/reference/supported-clusters.md index 62ff07fe..278e51d4 100644 --- a/site/content/docs/reference/supported-clusters.md +++ b/site/content/docs/reference/supported-clusters.md @@ -15,15 +15,20 @@ menu: | [VMware Tanzu Kubernetes Grid (TKG) clusters](https://tanzu.vmware.com/kubernetes-grid) | Yes | | [Kind clusters](https://kind.sigs.k8s.io/) | Yes | | [Kubeadm-based clusters](https://kubernetes.io/docs/reference/setup-tools/kubeadm/) | Yes | -| [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/) | No | -| [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) | No | -| [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/overview/kubernetes-on-azure) | No | +| [Amazon Elastic Kubernetes Service (EKS)](https://aws.amazon.com/eks/) | Yes | +| [Google Kubernetes Engine (GKE)](https://cloud.google.com/kubernetes-engine) | Yes | +| [Azure Kubernetes Service (AKS)](https://azure.microsoft.com/en-us/overview/kubernetes-on-azure) | Yes | ## Background -The Pinniped Concierge currently supports clusters where a custom pod can be executed on the same node running `kube-controller-manager`. +The Pinniped Concierge has two strategies available to support clusters, under the following conditions: + +1. Kube Cluster Signing Certificate: Can be run on any Kubernetes cluster where a custom pod can be executed on the same node running `kube-controller-manager`. This type of cluster is typically called "self-hosted" because the cluster's control plane is running on nodes that are part of the cluster itself. +Most managed Kubernetes services do not support this. -In practice, this means that many Kubernetes distributions are supported, but not most managed Kubernetes services +2. Impersonation Proxy: Can be run on any Kubernetes cluster where a `LoadBalancer` service can be created. Most cloud-hosted Kubernetes environments have this +capability. The Impersonation Proxy automatically provisions a `LoadBalancer` for ingress to the impersonation endpoint. -Support for more cluster types, including managed Kubernetes environments, is planned. +If a cluster is capable of supporting both strategies, the Pinniped Concierge will use the +kube cluster signing certificate strategy. From 698bffc2ad43395e48cefe20da2782a2dd0c8f48 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 16 Mar 2021 10:46:06 -0700 Subject: [PATCH 191/203] Naming changes --- site/content/docs/background/architecture.md | 2 +- site/content/docs/reference/supported-clusters.md | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/site/content/docs/background/architecture.md b/site/content/docs/background/architecture.md index bf258bac..ec481a5a 100644 --- a/site/content/docs/background/architecture.md +++ b/site/content/docs/background/architecture.md @@ -91,7 +91,7 @@ cleanly enable this integration. Pinniped supports the following cluster integration strategies. -* Kube Cluster Signing Certificate: Pinniped hosts a credential exchange API endpoint via a Kubernetes aggregated API server. +* Token Credential Request API: Pinniped hosts a credential exchange API endpoint via a Kubernetes aggregated API server. This API returns a new cluster-specific credential using the cluster's signing keypair to issue short-lived cluster certificates. (In the future, when the Kubernetes CSR API provides a way to issue short-lived certificates, then the Pinniped credential exchange API diff --git a/site/content/docs/reference/supported-clusters.md b/site/content/docs/reference/supported-clusters.md index 278e51d4..28ba554b 100644 --- a/site/content/docs/reference/supported-clusters.md +++ b/site/content/docs/reference/supported-clusters.md @@ -23,12 +23,15 @@ menu: The Pinniped Concierge has two strategies available to support clusters, under the following conditions: -1. Kube Cluster Signing Certificate: Can be run on any Kubernetes cluster where a custom pod can be executed on the same node running `kube-controller-manager`. +1. Token Credential Request API: Can be run on any Kubernetes cluster where a custom pod can be executed on the same node running `kube-controller-manager`. This type of cluster is typically called "self-hosted" because the cluster's control plane is running on nodes that are part of the cluster itself. Most managed Kubernetes services do not support this. 2. Impersonation Proxy: Can be run on any Kubernetes cluster where a `LoadBalancer` service can be created. Most cloud-hosted Kubernetes environments have this capability. The Impersonation Proxy automatically provisions a `LoadBalancer` for ingress to the impersonation endpoint. -If a cluster is capable of supporting both strategies, the Pinniped Concierge will use the -kube cluster signing certificate strategy. +If a cluster is capable of supporting both strategies, the Pinniped CLI will use the +token credential request API strategy by default. + +To choose the strategy to use with the concierge, use the `--concierge-mode` flag with `pinniped get kubeconfig`. +Possible values are `ImpersonationProxy` and `TokenCredentialRequestAPI`. From 4470d3d2d1bf855abb4945620788ad40b3f9258e Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 16 Mar 2021 11:00:31 -0700 Subject: [PATCH 192/203] Fix broken links to architecture page --- site/content/docs/tutorials/concierge-and-supervisor-demo.md | 2 +- site/content/docs/tutorials/concierge-only-demo.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/site/content/docs/tutorials/concierge-and-supervisor-demo.md b/site/content/docs/tutorials/concierge-and-supervisor-demo.md index 382cdfc0..3d2ae8c2 100644 --- a/site/content/docs/tutorials/concierge-and-supervisor-demo.md +++ b/site/content/docs/tutorials/concierge-and-supervisor-demo.md @@ -11,7 +11,7 @@ menu: ## Prerequisites -1. A Kubernetes cluster of a type supported by Pinniped Concierge as described in [architecture](/docs/architecture). +1. A Kubernetes cluster of a type supported by Pinniped Concierge as described in [architecture](/docs/background/architecture). Don't have a cluster handy? Consider using [kind](https://kind.sigs.k8s.io/) on your local machine. See below for an example of using kind. diff --git a/site/content/docs/tutorials/concierge-only-demo.md b/site/content/docs/tutorials/concierge-only-demo.md index 22943c84..e625a222 100644 --- a/site/content/docs/tutorials/concierge-only-demo.md +++ b/site/content/docs/tutorials/concierge-only-demo.md @@ -12,12 +12,12 @@ menu: ## Prerequisites -1. A Kubernetes cluster of a type supported by Pinniped as described in [architecture](/docs/architecture). +1. A Kubernetes cluster of a type supported by Pinniped as described in [architecture](/docs/background/architecture). Don't have a cluster handy? Consider using [kind](https://kind.sigs.k8s.io/) on your local machine. See below for an example of using kind. -1. An authenticator of a type supported by Pinniped as described in [architecture](/docs/architecture). +1. An authenticator of a type supported by Pinniped as described in [architecture](/docs/background/architecture). Don't have an authenticator of a type supported by Pinniped handy? No problem, there is a demo authenticator available. Start by installing local-user-authenticator on the same cluster where you would like to try Pinniped From 331fef8faebf9ab4b60d4e7c94ce42fe72da033e Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 16 Mar 2021 14:09:53 -0700 Subject: [PATCH 193/203] Tweaked some wording, updated the cli page --- site/content/docs/background/architecture.md | 6 +++--- site/content/docs/reference/cli.md | 3 +++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/site/content/docs/background/architecture.md b/site/content/docs/background/architecture.md index ec481a5a..89c879a0 100644 --- a/site/content/docs/background/architecture.md +++ b/site/content/docs/background/architecture.md @@ -22,7 +22,8 @@ to be passed on to clusters based on the user information from the IDP. 1. The Pinniped Concierge is a credential exchange API which takes as input a credential from an identity source (e.g., Pinniped Supervisor, proprietary IDP), authenticates the user via that credential, and returns another credential which is -understood by the host Kubernetes cluster. +understood by the host Kubernetes cluster or by an impersonation proxy which acts +on behalf of the user. ![Pinniped Architecture Sketch](/docs/img/pinniped_architecture_concierge_supervisor.svg) @@ -97,8 +98,7 @@ issue short-lived cluster certificates. (In the future, when the Kubernetes CSR provides a way to issue short-lived certificates, then the Pinniped credential exchange API will use that instead of using the cluster's signing keypair.) * Impersonation Proxy: Pinniped hosts an [impersonation](https://kubernetes.io/docs/reference/access-authn-authz/authentication/#user-impersonation) -proxy that performs actions on behalf of the end user. The impersonation proxy accepts and modifies user requests before passing them through to the -Kubernetes API server. +proxy that sends requests to the Kubernetes API server with user information and permissions based on a token. ## kubectl Integration diff --git a/site/content/docs/reference/cli.md b/site/content/docs/reference/cli.md index de1c4e34..15d9691b 100644 --- a/site/content/docs/reference/cli.md +++ b/site/content/docs/reference/cli.md @@ -43,6 +43,9 @@ pinniped get kubeconfig [flags] - `--concierge-authenticator-type string`: Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) +- `--concierge-mode`: + +Concierge mode of operation (e.g. 'ImpersonationProxy', 'TokenCredentialRequestAPI')(default: TokenCredentialRequestAPI) - `--kubeconfig string`: Path to kubeconfig file From fdfc854f8ce1826d2cd3994edb71637a5e178264 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 19 Mar 2021 13:57:08 -0700 Subject: [PATCH 194/203] Incorporating suggestions: - a credential that is understood by -> a credential that can be used to authenticate to - This is more neutral to whether its going directly to k8s or through the impersonation proxy --- site/content/docs/background/architecture.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/site/content/docs/background/architecture.md b/site/content/docs/background/architecture.md index 89c879a0..5eff94f1 100644 --- a/site/content/docs/background/architecture.md +++ b/site/content/docs/background/architecture.md @@ -117,8 +117,8 @@ both, or neither. ### Full Integration-- Concierge, Supervisor, and CLI Users can authenticate with the help of the Supervisor, which will issue tokens that -can be exchanged at the Concierge for a credential that is understood by the host Kubernetes -cluster. +can be exchanged at the Concierge for a credential that can be used to authenticate to +the host Kubernetes cluster. The Supervisor enables users to log in to their external identity provider once per day and access each cluster in a domain with a distinct scoped-down token. @@ -137,8 +137,8 @@ JWT authenticator via the Pinniped Concierge. ### Dynamic Cluster Authentication-- Concierge and CLI Users can authenticate directly with their OIDC compliant external identity provider to get credentials which -can be exchanged at the Concierge for a credential that is understood by the host Kubernetes -cluster. +can be exchanged at the Concierge for a credential that can be used to authenticate to +the host Kubernetes cluster. The diagram below shows the components involved in the login flow when the Concierge is configured. From a53728760195e56a312576bbb67dadca5d4e1c60 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Fri, 19 Mar 2021 14:34:35 -0700 Subject: [PATCH 195/203] Regenerate cli.md based on output of help message --- site/content/docs/reference/cli.md | 32 ++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/site/content/docs/reference/cli.md b/site/content/docs/reference/cli.md index 15d9691b..9cb92b33 100644 --- a/site/content/docs/reference/cli.md +++ b/site/content/docs/reference/cli.md @@ -32,8 +32,7 @@ pinniped get kubeconfig [flags] - `-h`, `--help`: - help for kubeconfig - + help for kubeconfig - `--concierge-api-group-suffix string`: Concierge API group suffix (default "pinniped.dev") @@ -43,9 +42,21 @@ pinniped get kubeconfig [flags] - `--concierge-authenticator-type string`: Concierge authenticator type (e.g., 'webhook', 'jwt') (default: autodiscover) -- `--concierge-mode`: +- `--concierge-ca-bundle path`: -Concierge mode of operation (e.g. 'ImpersonationProxy', 'TokenCredentialRequestAPI')(default: TokenCredentialRequestAPI) + Path to TLS certificate authority bundle (PEM format, optional, can be repeated) to use when connecting to the Concierge +- `--concierge-credential-issuer string`: + + Concierge CredentialIssuer object to use for autodiscovery (default: autodiscover) +- `--concierge-endpoint string`: + + API base for the Concierge endpoint +- `--concierge-mode mode`: + + Concierge mode of operation (default TokenCredentialRequestAPI) +- `--concierge-skip-wait`: + + Skip waiting for any pending Concierge strategies to become ready (default: false) - `--kubeconfig string`: Path to kubeconfig file @@ -54,8 +65,8 @@ Concierge mode of operation (e.g. 'ImpersonationProxy', 'TokenCredentialRequestA Kubeconfig context name (default: current active context) - `--no-concierge`: - Generate a configuration which does not use the concierge, but sends the credential to the cluster directly -- `--oidc-ca-bundle strings`: + Generate a configuration which does not use the Concierge, but sends the credential to the cluster directly +- `--oidc-ca-bundle path`: Path to TLS certificate authority bundle (PEM format, optional, can be repeated) - `--oidc-client-id string`: @@ -79,9 +90,18 @@ Concierge mode of operation (e.g. 'ImpersonationProxy', 'TokenCredentialRequestA - `--oidc-skip-browser`: During OpenID Connect login, skip opening the browser (just print the URL) +- `-o`, `--output string`: + + Output file path (default: stdout) +- `--skip-validation`: + + Skip final validation of the kubeconfig (default: false) - `--static-token string`: Instead of doing an OIDC-based login, specify a static token - `--static-token-env string`: Instead of doing an OIDC-based login, read a static token from the environment +- `--timeout duration`: + + Timeout for autodiscovery and validation (default 10m0s) From d7e9568137390866e5716d034894cd23f8c7134c Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Mon, 22 Mar 2021 09:43:40 -0700 Subject: [PATCH 196/203] Unparallelize a couple --- test/integration/concierge_impersonation_proxy_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 544e2252..02e7bf62 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -275,7 +275,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } t.Run("kubectl port-forward and keeping the connection open for over a minute", func(t *testing.T) { - t.Parallel() kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) @@ -315,7 +314,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("using and watching all the basic verbs", func(t *testing.T) { - t.Parallel() // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := createTestNamespace(t, adminClient) From 7683a9879236bda2efc3439b35f1f93cee20b395 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Mon, 22 Mar 2021 09:45:51 -0700 Subject: [PATCH 197/203] Unparallelize run all the verbs and port-forward tests --- test/integration/concierge_impersonation_proxy_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 02e7bf62..0ecdc775 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -275,7 +275,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } t.Run("kubectl port-forward and keeping the connection open for over a minute", func(t *testing.T) { - kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM) // Run the kubectl port-forward command. @@ -314,7 +313,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("using and watching all the basic verbs", func(t *testing.T) { - // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := createTestNamespace(t, adminClient) From d90398815bd2b55f874a3bf0b09b59f1ad6798b4 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Mon, 22 Mar 2021 10:48:09 -0700 Subject: [PATCH 198/203] Nothing in parallel in the impersonation proxy integration test --- test/integration/concierge_impersonation_proxy_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 0ecdc775..7f196484 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -433,8 +433,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { - t.Parallel() - // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. doubleImpersonationKubeClient := newImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes @@ -459,8 +457,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // authorization treatment from the Kube API server code that we are using, and we want to ensure that we are blocking // double impersonation even for the cluster admin. t.Run("double impersonation as a cluster admin user is blocked", func(t *testing.T) { - t.Parallel() - // Copy the admin credentials from the admin kubeconfig. adminClientRestConfig := library.NewClientConfig(t) @@ -492,8 +488,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { - t.Parallel() - // Test using the TokenCredentialRequest for authentication. impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient( impersonationProxyURL, impersonationProxyCACertPEM, "", @@ -622,8 +616,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("websocket client", func(t *testing.T) { - t.Parallel() - namespaceName := createTestNamespace(t, adminClient) impersonationRestConfig := impersonationProxyRestConfig(refreshCredential(), impersonationProxyURL, impersonationProxyCACertPEM, "") @@ -694,8 +686,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("http2 client", func(t *testing.T) { - t.Parallel() - namespaceName := createTestNamespace(t, adminClient) wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" From 75cfda0ffe240fdfb8db8ef7cfd3bfd392256f9f Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 22 Mar 2021 16:54:11 -0700 Subject: [PATCH 199/203] prepare-for-integration-tests.sh: Check Chrome and chromedriver versions They usually need to match, or at least be close, so added some code to help us remember to do that. --- hack/prepare-for-integration-tests.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hack/prepare-for-integration-tests.sh b/hack/prepare-for-integration-tests.sh index a14aa13f..e317c8f9 100755 --- a/hack/prepare-for-integration-tests.sh +++ b/hack/prepare-for-integration-tests.sh @@ -116,6 +116,22 @@ check_dependency htpasswd "Please install htpasswd. Should be pre-installed on M check_dependency openssl "Please install openssl. Should be pre-installed on MacOS." check_dependency chromedriver "Please install chromedriver. e.g. 'brew install chromedriver' for MacOS" +# Check that Chrome and chromedriver versions match. If chromedriver falls a couple versions behind +# then usually tests start to fail with strange error messages. +if [[ "$OSTYPE" == "darwin"* ]]; then + chrome_version=$(/Applications/Google\ Chrome.app/Contents/MacOS/Google\ Chrome --version | cut -d ' ' -f3 | cut -d '.' -f1) +else + chrome_version=$(google-chrome --version | cut -d ' ' -f3 | cut -d '.' -f1) +fi +chromedriver_version=$(chromedriver --version | cut -d ' ' -f2 | cut -d '.' -f1) +if [[ "$chrome_version" != "$chromedriver_version" ]]; then + log_error "It appears that you are using Chrome $chrome_version with chromedriver $chromedriver_version." + log_error "Please use the same version of chromedriver as Chrome." + log_error "If you are using the latest version of Chrome, then you can upgrade" + log_error "to the latest chromedriver, e.g. 'brew upgrade chromedriver' on MacOS." + exit 1 +fi + # Require kubectl >= 1.18.x if [ "$(kubectl version --client=true --short | cut -d '.' -f 2)" -lt 18 ]; then log_error "kubectl >= 1.18.x is required, you have $(kubectl version --client=true --short | cut -d ':' -f2)" From 95011682652871b52f78ef51a4ed50b762bf0c4d Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 23 Mar 2021 10:26:04 -0500 Subject: [PATCH 200/203] Simplify TestCLIGetKubeconfigStaticToken now that there's only a single table case. Signed-off-by: Matt Moyer --- test/integration/cli_test.go | 120 +++++++++++++++-------------------- 1 file changed, 51 insertions(+), 69 deletions(-) diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index 5f93ed40..1c239243 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -49,78 +49,60 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { // Build pinniped CLI. pinnipedExe := library.PinnipedCLIPath(t) - for _, tt := range []struct { - name string - args []string - expectStderrContains []string - }{ - { - name: "newer command, but still using static parameters", - args: []string{ - "get", "kubeconfig", - "--static-token", env.TestUser.Token, - "--concierge-api-group-suffix", env.APIGroupSuffix, - "--concierge-authenticator-type", "webhook", - "--concierge-authenticator-name", authenticator.Name, - }, - expectStderrContains: []string{ - "discovered CredentialIssuer", - "discovered Concierge endpoint", - "discovered Concierge certificate authority bundle", - "validated connection to the cluster", - }, - }, - } { - tt := tt - t.Run(tt.name, func(t *testing.T) { - stdout, stderr := runPinnipedCLI(t, nil, pinnipedExe, tt.args...) - for _, s := range tt.expectStderrContains { - assert.Contains(t, stderr, s) - } + stdout, stderr := runPinnipedCLI(t, nil, pinnipedExe, "get", "kubeconfig", + "--static-token", env.TestUser.Token, + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--concierge-authenticator-type", "webhook", + "--concierge-authenticator-name", authenticator.Name, + ) + assert.Contains(t, stderr, "discovered CredentialIssuer") + assert.Contains(t, stderr, "discovered Concierge endpoint") + assert.Contains(t, stderr, "discovered Concierge certificate authority bundle") + assert.Contains(t, stderr, "validated connection to the cluster") - // Even the deprecated command should now generate a kubeconfig with the new "pinniped login static" command. - restConfig := library.NewRestConfigFromKubeconfig(t, stdout) - require.NotNil(t, restConfig.ExecProvider) - require.Equal(t, []string{"login", "static"}, restConfig.ExecProvider.Args[:2]) + // Even the deprecated command should now generate a kubeconfig with the new "pinniped login static" command. + restConfig := library.NewRestConfigFromKubeconfig(t, stdout) + require.NotNil(t, restConfig.ExecProvider) + require.Equal(t, []string{"login", "static"}, restConfig.ExecProvider.Args[:2]) - // In addition to the client-go based testing below, also try the kubeconfig - // with kubectl to validate that it works. - t.Run( - "access as user with kubectl", - library.AccessAsUserWithKubectlTest(stdout, env.TestUser.ExpectedUsername, env.ConciergeNamespace), - ) - for _, group := range env.TestUser.ExpectedGroups { - group := group - t.Run( - "access as group "+group+" with kubectl", - library.AccessAsGroupWithKubectlTest(stdout, group, env.ConciergeNamespace), - ) - } - - // Create Kubernetes client with kubeconfig from pinniped CLI. - kubeClient := library.NewClientsetForKubeConfig(t, stdout) - - // Validate that we can auth to the API via our user. - t.Run("access as user with client-go", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, kubeClient)) - for _, group := range env.TestUser.ExpectedGroups { - group := group - t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, group, kubeClient)) - } - - // Validate that `pinniped whoami` returns the correct identity. - kubeconfigPath := filepath.Join(testutil.TempDir(t), "whoami-kubeconfig") - require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(stdout), 0600)) - assertWhoami( - ctx, - t, - false, - pinnipedExe, - kubeconfigPath, - env.TestUser.ExpectedUsername, - append(env.TestUser.ExpectedGroups, "system:authenticated"), - ) - }) + // In addition to the client-go based testing below, also try the kubeconfig + // with kubectl to validate that it works. + t.Run( + "access as user with kubectl", + library.AccessAsUserWithKubectlTest(stdout, env.TestUser.ExpectedUsername, env.ConciergeNamespace), + ) + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run( + "access as group "+group+" with kubectl", + library.AccessAsGroupWithKubectlTest(stdout, group, env.ConciergeNamespace), + ) } + + // Create Kubernetes client with kubeconfig from pinniped CLI. + kubeClient := library.NewClientsetForKubeConfig(t, stdout) + + // Validate that we can auth to the API via our user. + t.Run("access as user with client-go", library.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, kubeClient)) + for _, group := range env.TestUser.ExpectedGroups { + group := group + t.Run("access as group "+group+" with client-go", library.AccessAsGroupTest(ctx, group, kubeClient)) + } + + t.Run("whoami", func(t *testing.T) { + // Validate that `pinniped whoami` returns the correct identity. + kubeconfigPath := filepath.Join(testutil.TempDir(t), "whoami-kubeconfig") + require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(stdout), 0600)) + assertWhoami( + ctx, + t, + false, + pinnipedExe, + kubeconfigPath, + env.TestUser.ExpectedUsername, + append(env.TestUser.ExpectedGroups, "system:authenticated"), + ) + }) } func runPinnipedCLI(t *testing.T, envVars []string, pinnipedExe string, args ...string) (string, string) { From 176fb6a139aed75c123fc9937cfb66b6ccaea171 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 23 Mar 2021 10:33:05 -0500 Subject: [PATCH 201/203] Authenticators are no longer namespaced, so clean up these test logs. Signed-off-by: Matt Moyer --- test/library/client.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/library/client.go b/test/library/client.go index c53acc6f..789ffb2d 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -167,11 +167,11 @@ func CreateTestWebhookAuthenticator(ctx context.Context, t *testing.T) corev1.Ty Spec: testEnv.TestWebhook, }, metav1.CreateOptions{}) require.NoError(t, err, "could not create test WebhookAuthenticator") - t.Logf("created test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) + t.Logf("created test WebhookAuthenticator %s", webhook.Name) t.Cleanup(func() { t.Helper() - t.Logf("cleaning up test WebhookAuthenticator %s/%s", webhook.Namespace, webhook.Name) + t.Logf("cleaning up test WebhookAuthenticator %s", webhook.Name) deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := webhooks.Delete(deleteCtx, webhook.Name, metav1.DeleteOptions{}) @@ -230,7 +230,7 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp Spec: spec, }, metav1.CreateOptions{}) require.NoError(t, err, "could not create test JWTAuthenticator") - t.Logf("created test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) + t.Logf("created test JWTAuthenticator %s", jwtAuthenticator.Name) t.Cleanup(func() { t.Helper() @@ -238,7 +238,7 @@ func CreateTestJWTAuthenticator(ctx context.Context, t *testing.T, spec auth1alp deleteCtx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() err := jwtAuthenticators.Delete(deleteCtx, jwtAuthenticator.Name, metav1.DeleteOptions{}) - require.NoErrorf(t, err, "could not cleanup test JWTAuthenticator %s/%s", jwtAuthenticator.Namespace, jwtAuthenticator.Name) + require.NoErrorf(t, err, "could not cleanup test JWTAuthenticator %s", jwtAuthenticator.Name) }) return corev1.TypedLocalObjectReference{ From ce5b05f9121155a4f4a8b177cf1b2292275b9baf Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 23 Mar 2021 12:06:35 -0500 Subject: [PATCH 202/203] Add some debug logging to measure how long the CLI build takes. Signed-off-by: Matt Moyer --- test/library/cli.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/library/cli.go b/test/library/cli.go index d68e2439..dac7b8bc 100644 --- a/test/library/cli.go +++ b/test/library/cli.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package library @@ -9,6 +9,7 @@ import ( "path/filepath" "sync" "testing" + "time" "github.com/stretchr/testify/require" @@ -34,8 +35,10 @@ func PinnipedCLIPath(t *testing.T) string { } t.Log("building pinniped CLI binary") + start := time.Now() output, err := exec.Command("go", "build", "-o", path, "go.pinniped.dev/cmd/pinniped").CombinedOutput() require.NoError(t, err, string(output)) + t.Logf("built CLI binary in %s", time.Since(start).Round(time.Millisecond)) // Fill our cache so we don't have to do this again. pinnipedCLIBinaryCache.buf, err = ioutil.ReadFile(path) From c0d32f10b2b3e817981443c7da0e6d85e97c65f4 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 23 Mar 2021 12:07:34 -0500 Subject: [PATCH 203/203] Add some test debug logging when running the CLI. Signed-off-by: Matt Moyer --- test/integration/cli_test.go | 2 ++ test/library/iotest.go | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index 1c239243..1221a604 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -107,12 +107,14 @@ func TestCLIGetKubeconfigStaticToken(t *testing.T) { func runPinnipedCLI(t *testing.T, envVars []string, pinnipedExe string, args ...string) (string, string) { t.Helper() + start := time.Now() var stdout, stderr bytes.Buffer cmd := exec.Command(pinnipedExe, args...) cmd.Stdout = &stdout cmd.Stderr = &stderr cmd.Env = envVars require.NoErrorf(t, cmd.Run(), "stderr:\n%s\n\nstdout:\n%s\n\n", stderr.String(), stdout.String()) + t.Logf("ran %q in %s", library.MaskTokens("pinniped "+strings.Join(args, " ")), time.Since(start).Round(time.Millisecond)) return stdout.String(), stderr.String() } diff --git a/test/library/iotest.go b/test/library/iotest.go index 6e2f1e58..7ac175b3 100644 --- a/test/library/iotest.go +++ b/test/library/iotest.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package library @@ -43,6 +43,10 @@ func MaskTokens(in string) string { if strings.Count(t, ".") >= 4 { return t } + // Another heuristic, things that start with "--" are probably CLI flags. + if strings.HasPrefix(t, "--") { + return t + } return fmt.Sprintf("[...%d bytes...]", len(t)) }) }