From 521adffb174c0d59245480db6344b076d7a4685d Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Fri, 9 Apr 2021 17:52:53 -0400 Subject: [PATCH 01/10] impersonation proxy: add nested impersonation support This change updates the impersonator logic to use the delegated authorizer for all non-rest verbs such as impersonate. This allows it to correctly perform authorization checks for incoming requests that set impersonation headers while not performing unnecessary checks that are already handled by KAS. The audit layer is enabled to track the original user who made the request. This information is then included in a reserved extra field original-user-info.impersonation-proxy.concierge.pinniped.dev as a JSON blob. Signed-off-by: Monis Khan --- internal/concierge/impersonator/doc.go | 42 ++ .../concierge/impersonator/impersonator.go | 160 +++- .../impersonator/impersonator_test.go | 702 +++++++++++++++++- .../concierge_impersonation_proxy_test.go | 485 +++++++++--- 4 files changed, 1228 insertions(+), 161 deletions(-) create mode 100644 internal/concierge/impersonator/doc.go diff --git a/internal/concierge/impersonator/doc.go b/internal/concierge/impersonator/doc.go new file mode 100644 index 00000000..d8504603 --- /dev/null +++ b/internal/concierge/impersonator/doc.go @@ -0,0 +1,42 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/* +Package impersonator implements an HTTP server that reverse proxies all requests +to the Kubernetes API server with impersonation headers set to match the calling +user. Since impersonation cannot be disabled, this allows us to dynamically +configure authentication on any cluster, even the cloud hosted ones. + +The specifics of how it is implemented are of interest. The most novel detail +about the implementation is that we use the "front-end" of the aggregated API +server logic, mainly the DefaultBuildHandlerChain func, to handle how incoming +requests are authenticated, authorized, etc. The "back-end" of the proxy is a +reverse proxy that impersonates the user (instead of serving REST APIs). + +In terms of authentication, we aim to handle every type of authentication that +the Kubernetes API server supports by delegating most of the checks to it. We +also honor client certs from a CA that is specific to the impersonation proxy. +This approach allows clients to use the Token Credential Request API even when +we do not have the cluster's signing key. + +In terms of authorization, we rely mostly on the Kubernetes API server. Since we +impersonate the user, the proxied request will be authorized against that user. +Thus for all regular REST verbs, we perform no authorization checks. + +Nested impersonation is handled by performing the same authorization checks the +Kubernetes API server would (we get this mostly for free by using the aggregated +API server code). We preserve the original user in the reserved extra key +original-user-info.impersonation-proxy.concierge.pinniped.dev as a JSON blob of +the authenticationv1.UserInfo struct. This is necessary to make sure that the +Kubernetes audit log contains all three identities (original user, impersonated +user and the impersonation proxy's service account). Capturing the original +user information requires that we enable the auditing stack (WithImpersonation +only shares this information with the audit stack). To keep things simple, +we use the fake audit backend at the Metadata level for all requests. This +guarantees that we always have an audit event on every request. + +For all normal requests, we only use http/2.0 when proxying to the API server. +For upgrade requests, we only use http/1.1 since these always go from http/1.1 +to either websockets or SPDY. +*/ +package impersonator diff --git a/internal/concierge/impersonator/impersonator.go b/internal/concierge/impersonator/impersonator.go index f3a7372d..087329dd 100644 --- a/internal/concierge/impersonator/impersonator.go +++ b/internal/concierge/impersonator/impersonator.go @@ -4,11 +4,14 @@ package impersonator import ( + "context" + "encoding/json" "fmt" "net" "net/http" "net/http/httputil" "net/url" + "regexp" "strings" "time" @@ -21,6 +24,8 @@ import ( "k8s.io/apimachinery/pkg/util/httpstream" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" + auditinternal "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/audit/policy" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/authorization/authorizer" "k8s.io/apiserver/pkg/endpoints/filterlatency" @@ -31,6 +36,7 @@ import ( "k8s.io/apiserver/pkg/server/dynamiccertificates" "k8s.io/apiserver/pkg/server/filters" genericoptions "k8s.io/apiserver/pkg/server/options" + auditfake "k8s.io/apiserver/plugin/pkg/audit/fake" "k8s.io/client-go/rest" "k8s.io/client-go/transport" @@ -100,7 +106,6 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. if err != nil { return nil, err } - recommendedOptions.Authentication.ClientCert.ClientCA = "---irrelevant-but-needs-to-be-non-empty---" // drop when we pick up https://github.com/kubernetes/kubernetes/pull/100055 recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider( impersonationProxySignerCA, kubeClientCA, ) @@ -163,35 +168,55 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. })) handler = filterlatency.TrackStarted(handler, "impersonationproxy") + handler = filterlatency.TrackCompleted(handler) + handler = deleteKnownImpersonationHeaders(handler) + handler = filterlatency.TrackStarted(handler, "deleteimpersonationheaders") + // The standard Kube handler chain (authn, authz, impersonation, audit, etc). // See the genericapiserver.DefaultBuildHandlerChain func for details. handler = defaultBuildHandlerChainFunc(handler, c) // Always set security headers so browsers do the right thing. + handler = filterlatency.TrackCompleted(handler) handler = securityheader.Wrap(handler) + handler = filterlatency.TrackStarted(handler, "securityheaders") return handler } - // Overwrite the delegating authorizer with one that only cares about impersonation. - // Empty string is disallowed because request info has had bugs in the past where it would leave it empty. - disallowedVerbs := sets.NewString("", "impersonate") - noImpersonationAuthorizer := &comparableAuthorizer{ - AuthorizerFunc: func(a authorizer.Attributes) (authorizer.Decision, string, error) { - // Supporting impersonation is not hard, it would just require a bunch of testing - // and configuring the audit layer (to preserve the caller) which we can do later. - // We would also want to delete the incoming impersonation headers - // instead of overwriting the delegating authorizer, we would - // actually use it to make the impersonation authorization checks. - if disallowedVerbs.Has(a.GetVerb()) { - return authorizer.DecisionDeny, "impersonation is not allowed or invalid verb", nil - } + // wire up a fake audit backend at the metadata level so we can preserve the original user during nested impersonation + // TODO: wire up the real std out logging audit backend based on plog log level + serverConfig.AuditPolicyChecker = policy.FakeChecker(auditinternal.LevelMetadata, nil) + serverConfig.AuditBackend = &auditfake.Backend{} - return authorizer.DecisionAllow, "deferring authorization to kube API server", nil + delegatingAuthorizer := serverConfig.Authorization.Authorizer + nestedImpersonationAuthorizer := &comparableAuthorizer{ + authorizerFunc: func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) { + switch a.GetVerb() { + case "": + // Empty string is disallowed because request info has had bugs in the past where it would leave it empty. + return authorizer.DecisionDeny, "invalid verb", nil + case "create", + "update", + "delete", + "deletecollection", + "get", + "list", + "watch", + "patch", + "proxy": + // we know these verbs are from the request info parsing which is safe to delegate to KAS + return authorizer.DecisionAllow, "deferring standard verb authorization to kube API server", nil + default: + // assume everything else is internal SAR checks that we need to run against the requesting user + // because when KAS does the check, it may run the check against our service account and not the + // requesting user. This also handles the impersonate verb to allow for nested impersonation. + return delegatingAuthorizer.Authorize(ctx, a) + } }, } // Set our custom authorizer before calling Compete(), which will use it. - serverConfig.Authorization.Authorizer = noImpersonationAuthorizer + serverConfig.Authorization.Authorizer = nestedImpersonationAuthorizer impersonationProxyServer, err := serverConfig.Complete().New("impersonation-proxy", genericapiserver.NewEmptyDelegate()) if err != nil { @@ -201,7 +226,7 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. preparedRun := impersonationProxyServer.PrepareRun() // Sanity check. Make sure that our custom authorizer is still in place and did not get changed or wrapped. - if preparedRun.Authorizer != noImpersonationAuthorizer { + if preparedRun.Authorizer != nestedImpersonationAuthorizer { return nil, constable.Error("invalid mutation of impersonation authorizer detected") } @@ -225,9 +250,44 @@ func newInternal( //nolint:funlen // yeah, it's kind of long. return result, nil } +func deleteKnownImpersonationHeaders(delegate http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // remove known impersonation headers while avoiding mutation of input request + // unknown future impersonation headers will still get caught by our later checks + if ensureNoImpersonationHeaders(r) != nil { + r = r.Clone(r.Context()) + + impersonationHeaders := []string{ + transport.ImpersonateUserHeader, + transport.ImpersonateGroupHeader, + } + + for k := range r.Header { + if !strings.HasPrefix(k, transport.ImpersonateUserExtraHeaderPrefix) { + continue + } + impersonationHeaders = append(impersonationHeaders, k) + } + + for _, header := range impersonationHeaders { + r.Header.Del(header) // delay mutation until the end when we are done iterating over the map + } + } + + delegate.ServeHTTP(w, r) + }) +} + // No-op wrapping around AuthorizerFunc to allow for comparisons. type comparableAuthorizer struct { - authorizer.AuthorizerFunc + authorizerFunc +} + +// TODO: delete when we pick up https://github.com/kubernetes/kubernetes/pull/100963 +type authorizerFunc func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) + +func (f authorizerFunc) Authorize(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) { + return f(ctx, a) } func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapiserver.Config) http.Handler, error) { @@ -258,7 +318,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi } if err := ensureNoImpersonationHeaders(r); err != nil { - plog.Error("noImpersonationAuthorizer logic did not prevent nested impersonation but it is always supposed to do so", + plog.Error("unknown impersonation header seen", err, "url", r.URL.String(), "method", r.Method, @@ -277,6 +337,16 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi return } + ae := request.AuditEventFrom(r.Context()) + if ae == nil { + plog.Warning("aggregated API server logic did not set audit event but it is always supposed to do so", + "url", r.URL.String(), + "method", r.Method, + ) + newInternalErrResponse(w, r, c.Serializer, "invalid audit event") + return + } + // KAS only supports upgrades via http/1.1 to websockets/SPDY (upgrades never use http/2.0) // Thus we default to using http/2.0 when the request is not an upgrade, otherwise we use http/1.1 baseRT := http2RoundTripper @@ -285,7 +355,7 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi baseRT = http1RoundTripper } - rt, err := getTransportForUser(userInfo, baseRT) + rt, err := getTransportForUser(userInfo, baseRT, ae) if err != nil { plog.WarningErr("rejecting request as we cannot act as the current user", err, "url", r.URL.String(), @@ -332,6 +402,9 @@ func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapi func ensureNoImpersonationHeaders(r *http.Request) error { for key := range r.Header { + // even though we have unit tests that try to cover this case, it is hard to tell if Go does + // client side canonicalization on encode, server side canonicalization on decode, or both + key := http.CanonicalHeaderKey(key) if strings.HasPrefix(key, "Impersonate") { return fmt.Errorf("%q header already exists", key) } @@ -340,12 +413,17 @@ func ensureNoImpersonationHeaders(r *http.Request) error { return nil } -func getTransportForUser(userInfo user.Info, delegate http.RoundTripper) (http.RoundTripper, error) { +func getTransportForUser(userInfo user.Info, delegate http.RoundTripper, ae *auditinternal.Event) (http.RoundTripper, error) { if len(userInfo.GetUID()) == 0 { + extra, err := buildExtra(userInfo.GetExtra(), ae) + if err != nil { + return nil, err + } + impersonateConfig := transport.ImpersonationConfig{ UserName: userInfo.GetName(), Groups: userInfo.GetGroups(), - Extra: userInfo.GetExtra(), + Extra: extra, } // transport.NewImpersonatingRoundTripper clones the request before setting headers // thus it will not accidentally mutate the input request (see http.Handler docs) @@ -365,6 +443,44 @@ func getTransportForUser(userInfo user.Info, delegate http.RoundTripper) (http.R return nil, constable.Error("unexpected uid") } +func buildExtra(extra map[string][]string, ae *auditinternal.Event) (map[string][]string, error) { + const reservedImpersonationProxySuffix = ".impersonation-proxy.concierge.pinniped.dev" + + // always validate that the extra is something we support irregardless of nested impersonation + for k := range extra { + if !extraKeyRegexp.MatchString(k) { + return nil, fmt.Errorf("disallowed extra key seen: %s", k) + } + + if strings.HasSuffix(k, reservedImpersonationProxySuffix) { + return nil, fmt.Errorf("disallowed extra key with reserved prefix seen: %s", k) + } + } + + if ae.ImpersonatedUser == nil { + return extra, nil // just return the given extra since nested impersonation is not being used + } + + // avoid mutating input map, preallocate new map to store original user info + out := make(map[string][]string, len(extra)+1) + + for k, v := range extra { + out[k] = v // shallow copy of slice since we are not going to mutate it + } + + origUserInfoJSON, err := json.Marshal(ae.User) + if err != nil { + return nil, err + } + + out["original-user-info"+reservedImpersonationProxySuffix] = []string{string(origUserInfoJSON)} + + return out, nil +} + +// extraKeyRegexp is a very conservative regex to handle impersonation's extra key fidelity limitations such as casing and escaping. +var extraKeyRegexp = regexp.MustCompile(`^[a-z0-9/\-._]+$`) + func newInternalErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, msg string) { newStatusErrResponse(w, r, s, apierrors.NewInternalError(constable.Error(msg))) } diff --git a/internal/concierge/impersonator/impersonator_test.go b/internal/concierge/impersonator/impersonator_test.go index 7633c9df..dfd765c3 100644 --- a/internal/concierge/impersonator/impersonator_test.go +++ b/internal/concierge/impersonator/impersonator_test.go @@ -15,11 +15,13 @@ import ( "time" "github.com/stretchr/testify/require" + authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apimachinery/pkg/util/httpstream" + auditinternal "k8s.io/apiserver/pkg/apis/audit" "k8s.io/apiserver/pkg/authentication/user" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" @@ -39,8 +41,6 @@ import ( ) func TestImpersonator(t *testing.T) { - const port = 9444 - ca, err := certauthority.New("ca", time.Hour) require.NoError(t, err) caKey, err := ca.PrivateKeyToPEM() @@ -58,13 +58,7 @@ func TestImpersonator(t *testing.T) { unrelatedCA, err := certauthority.New("ca", time.Hour) require.NoError(t, err) - // Punch out just enough stuff to make New actually run without error. - recOpts := func(options *genericoptions.RecommendedOptions) { - options.Authentication.RemoteKubeConfigFileOptional = true - options.Authorization.RemoteKubeConfigFileOptional = true - options.CoreAPI = nil - options.Admission = nil - } + // turn off this code path for all tests because it does not handle the config we remove correctly defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.APIPriorityAndFairness, false)() tests := []struct { @@ -140,7 +134,7 @@ func TestImpersonator(t *testing.T) { clientCert: newClientCert(t, ca, "test-username2", []string{"test-group3", "test-group4"}), kubeAPIServerClientBearerTokenFile: "required-to-be-set", clientMutateHeaders: func(header http.Header) { - header.Add("x-FORWARDED-for", "example.com") + header["x-FORWARDED-for"] = append(header["x-FORWARDED-for"], "example.com") }, wantKubeAPIServerRequestHeaders: http.Header{ "Impersonate-User": {"test-username2"}, @@ -189,20 +183,128 @@ func TestImpersonator(t *testing.T) { wantError: "Unauthorized", }, { - name: "double impersonation is not allowed by regular users", + name: "nested impersonation by regular users calls delegating authorizer", clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, kubeAPIServerClientBearerTokenFile: "required-to-be-set", + // this fails because the delegating authorizer in this test only allows system:masters and fails everything else wantError: `users "some-other-username" is forbidden: User "test-username" ` + - `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + `cannot impersonate resource "users" in API group "" at the cluster scope`, }, { - name: "double impersonation is not allowed by admin users", - clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), - clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + name: "nested impersonation by admin users calls delegating authorizer", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{ + UserName: "fire", + Groups: []string{"elements"}, + Extra: map[string][]string{ + "colors": {"red", "orange", "blue"}, + + // gke + "iam.gke.io/user-assertion": {"good", "stuff"}, + "user-assertion.cloud.google.com": {"smaller", "things"}, + + // openshift + "scopes.authorization.openshift.io": {"user:info", "user:full", "user:check-access"}, + + // openstack + "alpha.kubernetes.io/identity/roles": {"a-role1", "a-role2"}, + "alpha.kubernetes.io/identity/project/id": {"a-project-id"}, + "alpha.kubernetes.io/identity/project/name": {"a-project-name"}, + "alpha.kubernetes.io/identity/user/domain/id": {"a-domain-id"}, + "alpha.kubernetes.io/identity/user/domain/name": {"a-domain-name"}, + }, + }, kubeAPIServerClientBearerTokenFile: "required-to-be-set", - wantError: `users "some-other-username" is forbidden: User "test-admin" ` + - `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"fire"}, + "Impersonate-Group": {"elements", "system:authenticated"}, + "Impersonate-Extra-Colors": {"red", "orange", "blue"}, + "Impersonate-Extra-Iam.gke.io%2fuser-Assertion": {"good", "stuff"}, + "Impersonate-Extra-User-Assertion.cloud.google.com": {"smaller", "things"}, + "Impersonate-Extra-Scopes.authorization.openshift.io": {"user:info", "user:full", "user:check-access"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2froles": {"a-role1", "a-role2"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fproject%2fid": {"a-project-id"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fproject%2fname": {"a-project-name"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fuser%2fdomain%2fid": {"a-domain-id"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fuser%2fdomain%2fname": {"a-domain-name"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxy.concierge.pinniped.dev": {`{"username":"test-admin","groups":["test-group2","system:masters","system:authenticated"]}`}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, + }, + { + name: "nested impersonation by admin users cannot impersonate UID", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + clientMutateHeaders: func(header http.Header) { + header["Impersonate-Uid"] = []string{"root"} + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: invalid impersonation", + }, + { + name: "nested impersonation by admin users cannot impersonate UID header canonicalization", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{UserName: "some-other-username"}, + clientMutateHeaders: func(header http.Header) { + header["imPerSoNaTE-uid"] = []string{"magic"} + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: invalid impersonation", + }, + { + name: "nested impersonation by admin users cannot use reserved key", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{ + UserName: "other-user-to-impersonate", + Groups: []string{"other-peeps"}, + Extra: map[string][]string{ + "key": {"good"}, + "something.impersonation-proxy.concierge.pinniped.dev": {"bad data"}, + }, + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: unimplemented functionality - unable to act as current user", + }, + { + name: "nested impersonation by admin users cannot use invalid key", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{ + UserName: "panda", + Groups: []string{"other-peeps"}, + Extra: map[string][]string{ + "party~~time": {"danger"}, + }, + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantError: "Internal error occurred: unimplemented functionality - unable to act as current user", + }, + { + name: "nested impersonation by admin users can use uppercase key because impersonation is lossy", + clientCert: newClientCert(t, ca, "test-admin", []string{"system:masters", "test-group2"}), + clientImpersonateUser: rest.ImpersonationConfig{ + UserName: "panda", + Groups: []string{"other-peeps"}, + Extra: map[string][]string{ + "ROAR": {"tiger"}, // by the time our code sees this key, it is lowercased to "roar" + }, + }, + kubeAPIServerClientBearerTokenFile: "required-to-be-set", + wantKubeAPIServerRequestHeaders: http.Header{ + "Impersonate-User": {"panda"}, + "Impersonate-Group": {"other-peeps", "system:authenticated"}, + "Impersonate-Extra-Roar": {"tiger"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxy.concierge.pinniped.dev": {`{"username":"test-admin","groups":["test-group2","system:masters","system:authenticated"]}`}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-agent"}, + "Accept": {"application/vnd.kubernetes.protobuf,application/json"}, + "Accept-Encoding": {"gzip"}, + "X-Forwarded-For": {"127.0.0.1"}, + }, }, { name: "no bearer token file in Kube API server client config", @@ -212,17 +314,17 @@ func TestImpersonator(t *testing.T) { name: "header canonicalization user header", clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), clientMutateHeaders: func(header http.Header) { - header.Set("imPerSonaTE-USer", "PANDA") + header["imPerSonaTE-USer"] = []string{"PANDA"} }, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: `users "PANDA" is forbidden: User "test-username" ` + - `cannot impersonate resource "users" in API group "" at the cluster scope: impersonation is not allowed or invalid verb`, + `cannot impersonate resource "users" in API group "" at the cluster scope`, }, { name: "header canonicalization future UID header", clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), clientMutateHeaders: func(header http.Header) { - header.Set("imPerSonaTE-uid", "007") + header["imPerSonaTE-uid"] = []string{"007"} }, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: "Internal error occurred: invalid impersonation", @@ -231,7 +333,7 @@ func TestImpersonator(t *testing.T) { name: "future UID header", clientCert: newClientCert(t, ca, "test-username", []string{"test-group1", "test-group2"}), clientMutateHeaders: func(header http.Header) { - header.Set("Impersonate-Uid", "008") + header["Impersonate-Uid"] = []string{"008"} }, kubeAPIServerClientBearerTokenFile: "required-to-be-set", wantError: "Internal error occurred: invalid impersonation", @@ -239,8 +341,14 @@ func TestImpersonator(t *testing.T) { } for _, tt := range tests { tt := tt - // This is a serial test because the production code binds to the port. t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // we need to create this listener ourselves because the API server + // code treats (port == 0 && listener == nil) to mean "do nothing" + listener, port, err := genericoptions.CreateListener("", "127.0.0.1:0", net.ListenConfig{}) + require.NoError(t, err) + // After failing to start and after shutdown, the impersonator port should be available again. defer requireCanBindToPort(t, port) @@ -293,8 +401,17 @@ func TestImpersonator(t *testing.T) { } clientOpts := []kubeclient.Option{kubeclient.WithConfig(&testKubeAPIServerKubeconfig)} - // Create an impersonator. - runner, constructionErr := newInternal(port, certKeyContent, caContent, clientOpts, recOpts) + // Punch out just enough stuff to make New actually run without error. + recOpts := func(options *genericoptions.RecommendedOptions) { + options.Authentication.RemoteKubeConfigFileOptional = true + options.Authorization.RemoteKubeConfigFileOptional = true + options.CoreAPI = nil + options.Admission = nil + options.SecureServing.Listener = listener // use our listener with the dynamic port + } + + // Create an impersonator. Use an invalid port number to make sure our listener override works. + runner, constructionErr := newInternal(-1000, certKeyContent, caContent, clientOpts, recOpts) if len(tt.wantConstructionError) > 0 { require.EqualError(t, constructionErr, tt.wantConstructionError) require.Nil(t, runner) @@ -383,20 +500,30 @@ func TestImpersonatorHTTPHandler(t *testing.T) { } validURL, _ := url.Parse("http://pinniped.dev/blah") - newRequest := func(h http.Header, userInfo user.Info) *http.Request { + newRequest := func(h http.Header, userInfo user.Info, event *auditinternal.Event) *http.Request { ctx := context.Background() + if userInfo != nil { ctx = request.WithUser(ctx, userInfo) } - r, err := http.NewRequestWithContext(ctx, http.MethodGet, validURL.String(), nil) - require.NoError(t, err) - r.Header = h + + ae := &auditinternal.Event{Level: auditinternal.LevelMetadata} + if event != nil { + ae = event + } + ctx = request.WithAuditEvent(ctx, ae) + reqInfo := &request.RequestInfo{ IsResourceRequest: false, Path: validURL.Path, Verb: "get", } - r = r.WithContext(request.WithRequestInfo(ctx, reqInfo)) + ctx = request.WithRequestInfo(ctx, reqInfo) + + r, err := http.NewRequestWithContext(ctx, http.MethodGet, validURL.String(), nil) + require.NoError(t, err) + r.Header = h + return r } @@ -436,43 +563,123 @@ func TestImpersonatorHTTPHandler(t *testing.T) { }, { name: "Impersonate-User header already in request", - request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}, nil), + request: newRequest(map[string][]string{"Impersonate-User": {"some-user"}}, nil, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-Group header already in request", - request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}, nil), + request: newRequest(map[string][]string{"Impersonate-Group": {"some-group"}}, nil, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-Extra header already in request", - request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}, nil), + request: newRequest(map[string][]string{"Impersonate-Extra-something": {"something"}}, nil, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "Impersonate-* header already in request", - request: newRequest(map[string][]string{"Impersonate-Something": {"some-newfangled-impersonate-header"}}, nil), + request: newRequest(map[string][]string{"Impersonate-Something": {"some-newfangled-impersonate-header"}}, nil, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid impersonation","reason":"InternalError","details":{"causes":[{"message":"invalid impersonation"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "unexpected authorization header", - request: newRequest(map[string][]string{"Authorization": {"panda"}}, nil), + request: newRequest(map[string][]string{"Authorization": {"panda"}}, nil, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid authorization header","reason":"InternalError","details":{"causes":[{"message":"invalid authorization header"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "missing user", - request: newRequest(map[string][]string{}, nil), + request: newRequest(map[string][]string{}, nil, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid user","reason":"InternalError","details":{"causes":[{"message":"invalid user"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, { name: "unexpected UID", - request: newRequest(map[string][]string{}, &user.DefaultInfo{UID: "007"}), + request: newRequest(map[string][]string{}, &user.DefaultInfo{UID: "007"}, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: unimplemented functionality - unable to act as current user","reason":"InternalError","details":{"causes":[{"message":"unimplemented functionality - unable to act as current user"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "authenticated user but missing audit event", + request: func() *http.Request { + req := newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Other-Header": {"test-header-value-1"}, + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, + }, nil) + ctx := request.WithAuditEvent(req.Context(), nil) + req = req.WithContext(ctx) + return req + }(), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: invalid audit event","reason":"InternalError","details":{"causes":[{"message":"invalid audit event"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "authenticated user with upper case extra", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: map[string][]string{ + "valid-key": {"valid-value"}, + "Invalid-key": {"still-valid-value"}, + }, + }, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: unimplemented functionality - unable to act as current user","reason":"InternalError","details":{"causes":[{"message":"unimplemented functionality - unable to act as current user"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "authenticated user with upper case extra across multiple lines", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: map[string][]string{ + "valid-key": {"valid-value"}, + "valid-data\nInvalid-key": {"still-valid-value"}, + }, + }, nil), + wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: unimplemented functionality - unable to act as current user","reason":"InternalError","details":{"causes":[{"message":"unimplemented functionality - unable to act as current user"}]},"code":500}` + "\n", + wantHTTPStatus: http.StatusInternalServerError, + }, + { + name: "authenticated user with reserved extra key", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: map[string][]string{ + "valid-key": {"valid-value"}, + "foo.impersonation-proxy.concierge.pinniped.dev": {"still-valid-value"}, + }, + }, nil), wantHTTPBody: `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Internal error occurred: unimplemented functionality - unable to act as current user","reason":"InternalError","details":{"causes":[{"message":"unimplemented functionality - unable to act as current user"}]},"code":500}` + "\n", wantHTTPStatus: http.StatusInternalServerError, }, @@ -492,7 +699,7 @@ func TestImpersonatorHTTPHandler(t *testing.T) { Name: testUser, Groups: testGroups, Extra: testExtra, - }), + }, nil), wantKubeAPIServerRequestHeaders: map[string][]string{ "Authorization": {"Bearer some-service-account-token"}, "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, @@ -510,6 +717,318 @@ func TestImpersonatorHTTPHandler(t *testing.T) { wantHTTPBody: "successful proxied response", wantHTTPStatus: http.StatusOK, }, + { + name: "authenticated gke user", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: "username@company.com", + Groups: []string{"system:authenticated"}, + Extra: map[string][]string{ + // make sure we can handle these keys + "iam.gke.io/user-assertion": {"ABC"}, + "user-assertion.cloud.google.com": {"XYZ"}, + }, + }, nil), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Iam.gke.io%2fuser-Assertion": {"ABC"}, + "Impersonate-Extra-User-Assertion.cloud.google.com": {"XYZ"}, + "Impersonate-Group": {"system:authenticated"}, + "Impersonate-User": {"username@company.com"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "authenticated openshift/openstack user", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: "kube:admin", + // both of these auth stacks set UID but we cannot handle it today + // UID: "user-id", + Groups: []string{"system:cluster-admins", "system:authenticated"}, + Extra: map[string][]string{ + // openshift + "scopes.authorization.openshift.io": {"user:info", "user:full"}, + + // openstack + "alpha.kubernetes.io/identity/roles": {"role1", "role2"}, + "alpha.kubernetes.io/identity/project/id": {"project-id"}, + "alpha.kubernetes.io/identity/project/name": {"project-name"}, + "alpha.kubernetes.io/identity/user/domain/id": {"domain-id"}, + "alpha.kubernetes.io/identity/user/domain/name": {"domain-name"}, + }, + }, nil), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Scopes.authorization.openshift.io": {"user:info", "user:full"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2froles": {"role1", "role2"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fproject%2fid": {"project-id"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fproject%2fname": {"project-name"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fuser%2fdomain%2fid": {"domain-id"}, + "Impersonate-Extra-Alpha.kubernetes.io%2fidentity%2fuser%2fdomain%2fname": {"domain-name"}, + "Impersonate-Group": {"system:cluster-admins", "system:authenticated"}, + "Impersonate-User": {"kube:admin"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "authenticated user with almost reserved key", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: "username@company.com", + Groups: []string{"system:authenticated"}, + Extra: map[string][]string{ + "foo.iimpersonation-proxy.concierge.pinniped.dev": {"still-valid-value"}, + }, + }, nil), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Foo.iimpersonation-Proxy.concierge.pinniped.dev": {"still-valid-value"}, + "Impersonate-Group": {"system:authenticated"}, + "Impersonate-User": {"username@company.com"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "authenticated user with almost reserved key and nested impersonation", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: "username@company.com", + Groups: []string{"system:authenticated"}, + Extra: map[string][]string{ + "original-user-info.impersonation-proxyy.concierge.pinniped.dev": {"log confusion stuff here"}, + }, + }, + &auditinternal.Event{ + User: authenticationv1.UserInfo{ + Username: "panda", + UID: "0x001", + Groups: []string{"bears", "friends"}, + Extra: map[string]authenticationv1.ExtraValue{ + "original-user-info.impersonation-proxy.concierge.pinniped.dev": {"this is allowed"}, + }, + }, + ImpersonatedUser: &authenticationv1.UserInfo{}, + }, + ), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxyy.concierge.pinniped.dev": {"log confusion stuff here"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxy.concierge.pinniped.dev": {`{"username":"panda","uid":"0x001","groups":["bears","friends"],"extra":{"original-user-info.impersonation-proxy.concierge.pinniped.dev":["this is allowed"]}}`}, + "Impersonate-Group": {"system:authenticated"}, + "Impersonate-User": {"username@company.com"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "authenticated user with nested impersonation", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, + }, + &auditinternal.Event{ + User: authenticationv1.UserInfo{ + Username: "panda", + UID: "0x001", + Groups: []string{"bears", "friends"}, + Extra: map[string]authenticationv1.ExtraValue{ + "assertion": {"sha", "md5"}, + "req-id": {"0123"}, + }, + }, + ImpersonatedUser: &authenticationv1.UserInfo{}, + }, + ), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxy.concierge.pinniped.dev": {`{"username":"panda","uid":"0x001","groups":["bears","friends"],"extra":{"assertion":["sha","md5"],"req-id":["0123"]}}`}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "authenticated gke user with nested impersonation", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: testUser, + Groups: testGroups, + Extra: testExtra, + }, + &auditinternal.Event{ + User: authenticationv1.UserInfo{ + Username: "username@company.com", + Groups: []string{"system:authenticated"}, + Extra: map[string]authenticationv1.ExtraValue{ + // make sure we can handle these keys + "iam.gke.io/user-assertion": {"ABC"}, + "user-assertion.cloud.google.com": {"999"}, + }, + }, + ImpersonatedUser: &authenticationv1.UserInfo{}, + }, + ), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxy.concierge.pinniped.dev": {`{"username":"username@company.com","groups":["system:authenticated"],"extra":{"iam.gke.io/user-assertion":["ABC"],"user-assertion.cloud.google.com":["999"]}}`}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, + { + name: "authenticated user with nested impersonation of gke user", + request: newRequest(map[string][]string{ + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, // the value "Upgrade" is handled in a special way by `httputil.NewSingleHostReverseProxy` + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Content-Length": {"some-length"}, + "Other-Header": {"test-header-value-1"}, // this header will be passed through + }, &user.DefaultInfo{ + Name: "username@company.com", + Groups: []string{"system:authenticated"}, + Extra: map[string][]string{ + // make sure we can handle these keys + "iam.gke.io/user-assertion": {"DEF"}, + "user-assertion.cloud.google.com": {"XYZ"}, + }, + }, + &auditinternal.Event{ + User: authenticationv1.UserInfo{ + Username: "panda", + UID: "0x001", + Groups: []string{"bears", "friends"}, + Extra: map[string]authenticationv1.ExtraValue{ + "assertion": {"sha", "md5"}, + "req-id": {"0123"}, + }, + }, + ImpersonatedUser: &authenticationv1.UserInfo{}, + }, + ), + wantKubeAPIServerRequestHeaders: map[string][]string{ + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Iam.gke.io%2fuser-Assertion": {"DEF"}, + "Impersonate-Extra-User-Assertion.cloud.google.com": {"XYZ"}, + "Impersonate-Group": {"system:authenticated"}, + "Impersonate-User": {"username@company.com"}, + "User-Agent": {"test-user-agent"}, + "Accept": {"some-accepted-format"}, + "Accept-Encoding": {"some-accepted-encoding"}, + "Connection": {"Upgrade"}, + "Upgrade": {"some-upgrade"}, + "Content-Type": {"some-type"}, + "Other-Header": {"test-header-value-1"}, + "Impersonate-Extra-Original-User-Info.impersonation-Proxy.concierge.pinniped.dev": {`{"username":"panda","uid":"0x001","groups":["bears","friends"],"extra":{"assertion":["sha","md5"],"req-id":["0123"]}}`}, + }, + wantHTTPBody: "successful proxied response", + wantHTTPStatus: http.StatusOK, + }, { name: "user is authenticated but the kube API request returns an error", request: newRequest(map[string][]string{ @@ -518,7 +1037,7 @@ func TestImpersonatorHTTPHandler(t *testing.T) { Name: testUser, Groups: testGroups, Extra: testExtra, - }), + }, nil), kubeAPIServerStatusCode: http.StatusNotFound, wantKubeAPIServerRequestHeaders: map[string][]string{ "Accept-Encoding": {"gzip"}, // because the rest client used in this test does not disable compression @@ -623,6 +1142,7 @@ type clientCert struct { } func newClientCert(t *testing.T, ca *certauthority.CA, username string, groups []string) *clientCert { + t.Helper() certPEM, keyPEM, err := ca.IssueClientCertPEM(username, groups, time.Hour) require.NoError(t, err) return &clientCert{ @@ -632,7 +1152,113 @@ func newClientCert(t *testing.T, ca *certauthority.CA, username string, groups [ } func requireCanBindToPort(t *testing.T, port int) { + t.Helper() ln, _, listenErr := genericoptions.CreateListener("", "0.0.0.0:"+strconv.Itoa(port), net.ListenConfig{}) require.NoError(t, listenErr) require.NoError(t, ln.Close()) } + +func Test_deleteKnownImpersonationHeaders(t *testing.T) { + tests := []struct { + name string + headers, want http.Header + }{ + { + name: "no impersonation", + headers: map[string][]string{ + "a": {"b"}, + "Accept-Encoding": {"gzip"}, + "User-Agent": {"test-user-agent"}, + }, + want: map[string][]string{ + "a": {"b"}, + "Accept-Encoding": {"gzip"}, + "User-Agent": {"test-user-agent"}, + }, + }, + { + name: "impersonate user header is dropped", + headers: map[string][]string{ + "a": {"b"}, + "Impersonate-User": {"panda"}, + "Accept-Encoding": {"gzip"}, + "User-Agent": {"test-user-agent"}, + }, + want: map[string][]string{ + "a": {"b"}, + "Accept-Encoding": {"gzip"}, + "User-Agent": {"test-user-agent"}, + }, + }, + { + name: "all known impersonate headers are dropped", + headers: map[string][]string{ + "Accept-Encoding": {"gzip"}, + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "User-Agent": {"test-user-agent"}, + }, + want: map[string][]string{ + "Accept-Encoding": {"gzip"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-user-agent"}, + }, + }, + { + name: "future UID header is not dropped", + headers: map[string][]string{ + "Accept-Encoding": {"gzip"}, + "Authorization": {"Bearer some-service-account-token"}, + "Impersonate-Extra-Extra-1": {"some", "extra", "stuff"}, + "Impersonate-Extra-Extra-2": {"some", "more", "extra", "stuff"}, + "Impersonate-Group": {"test-group-1", "test-group-2"}, + "Impersonate-User": {"test-user"}, + "Impersonate-Uid": {"008"}, + "User-Agent": {"test-user-agent"}, + }, + want: map[string][]string{ + "Accept-Encoding": {"gzip"}, + "Authorization": {"Bearer some-service-account-token"}, + "User-Agent": {"test-user-agent"}, + "Impersonate-Uid": {"008"}, + }, + }, + { + name: "future UID header is not dropped, no other headers", + headers: map[string][]string{ + "Impersonate-Uid": {"009"}, + }, + want: map[string][]string{ + "Impersonate-Uid": {"009"}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + inputReq := (&http.Request{Header: tt.headers}).WithContext(context.Background()) + inputReqCopy := inputReq.Clone(inputReq.Context()) + + delegate := http.HandlerFunc(func(w http.ResponseWriter, outputReq *http.Request) { + require.Nil(t, w) + + // assert only headers mutated + outputReqCopy := outputReq.Clone(outputReq.Context()) + outputReqCopy.Header = tt.headers + require.Equal(t, inputReqCopy, outputReqCopy) + + require.Equal(t, tt.want, outputReq.Header) + + if ensureNoImpersonationHeaders(inputReq) == nil { + require.True(t, inputReq == outputReq, "expect req to passed through when no modification needed") + } + }) + + deleteKnownImpersonationHeaders(delegate).ServeHTTP(nil, inputReq) + require.Equal(t, inputReqCopy, inputReq) // assert no mutation occurred + }) + } +} diff --git a/test/integration/concierge_impersonation_proxy_test.go b/test/integration/concierge_impersonation_proxy_test.go index 372f7ca4..97a328ac 100644 --- a/test/integration/concierge_impersonation_proxy_test.go +++ b/test/integration/concierge_impersonation_proxy_test.go @@ -26,7 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/http2" - v1 "k8s.io/api/authorization/v1" + authenticationv1 "k8s.io/api/authentication/v1" + authorizationv1 "k8s.io/api/authorization/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -34,9 +35,14 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/request/bearertoken" + "k8s.io/apiserver/pkg/authentication/serviceaccount" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + "k8s.io/client-go/transport" + "k8s.io/client-go/util/keyutil" "sigs.k8s.io/yaml" conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" @@ -44,6 +50,7 @@ import ( loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" pinnipedconciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/concierge/impersonator" + "go.pinniped.dev/internal/httputil/roundtripper" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" "go.pinniped.dev/test/library" @@ -102,23 +109,13 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) - newImpersonationProxyClientWithCredentials := func(credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { - kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) - if !clusterSupportsLoadBalancers { - // Only if there is no possibility to send traffic through a load balancer, then send the traffic through the Squid proxy. - // Prefer to go through a load balancer because that's how the impersonator is intended to be used in the real world. - kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) - } - return library.NewKubeclient(t, kubeconfig) - } - - newAnonymousImpersonationProxyClient := func(impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { - emptyCredentials := &loginv1alpha1.ClusterCredential{} - return newImpersonationProxyClientWithCredentials(emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) - } - - var mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest + var ( + mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest + mostRecentTokenCredentialRequestResponseLock sync.Mutex + ) refreshCredential := func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential { + mostRecentTokenCredentialRequestResponseLock.Lock() + defer mostRecentTokenCredentialRequestResponseLock.Unlock() if mostRecentTokenCredentialRequestResponse == nil || credentialAlmostExpired(t, mostRecentTokenCredentialRequestResponse) { var err error // Make a TokenCredentialRequest. This can either return a cert signed by the Kube API server's CA (e.g. on kind) @@ -132,7 +129,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // what would normally happen when a user is using a kubeconfig where the server is the impersonation proxy, // so it more closely simulates the normal use case, and also because we want this to work on AKS clusters // which do not allow anonymous requests. - client := newAnonymousImpersonationProxyClient(impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge + client := newAnonymousImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge require.Eventually(t, func() bool { mostRecentTokenCredentialRequestResponse, err = createTokenCredentialRequest(credentialRequestSpecWithWorkingCredentials, client) if err != nil { @@ -154,19 +151,6 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl return mostRecentTokenCredentialRequestResponse.Status.Credential } - impersonationProxyViaSquidKubeClientWithoutCredential := func() kubernetes.Interface { - proxyURL := "https://" + proxyServiceEndpoint - kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, "") - kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) - return library.NewKubeclient(t, kubeconfig).Kubernetes - } - - newImpersonationProxyClient := func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, doubleImpersonateUser string) *kubeclient.Client { - refreshedCredentials := refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM).DeepCopy() - refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests - return newImpersonationProxyClientWithCredentials(refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, doubleImpersonateUser) - } - oldConfigMap, err := adminClient.CoreV1().ConfigMaps(env.ConciergeNamespace).Get(ctx, impersonationProxyConfigMapName(env), metav1.GetOptions{}) if !k8serrors.IsNotFound(err) { require.NoError(t, err) // other errors aside from NotFound are unexpected @@ -175,7 +159,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // At the end of the test, clean up the ConfigMap. t.Cleanup(func() { - ctx, cancel = context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() // Delete any version that was created by this test. @@ -249,7 +233,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. - _, err = impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err = impersonationProxyViaSquidKubeClientWithoutCredential(t, proxyServiceEndpoint).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) isErr, message := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) require.Truef(t, isErr, "wanted error %q to be service unavailable via squid error, but: %s", err, message) @@ -279,7 +263,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // so we don't have to keep repeating them. // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. impersonationProxyKubeClient := func(t *testing.T) kubernetes.Interface { - return newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, "").Kubernetes + return newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil, refreshCredential).Kubernetes } t.Run("positive tests", func(t *testing.T) { @@ -289,7 +273,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, ) // Wait for the above RBAC rule to take effect. - library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &v1.ResourceAttributes{ + library.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &authorizationv1.ResourceAttributes{ Verb: "get", Group: "", Version: "v1", Resource: "namespaces", }) @@ -323,12 +307,13 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } t.Run("kubectl port-forward and keeping the connection open for over a minute (non-idle)", func(t *testing.T) { + t.Parallel() kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator) // Run the kubectl port-forward command. timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "10443:8443") portForwardCmd.Env = envVarsWithProxy // Start, but don't wait for the command to finish. @@ -348,7 +333,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl defer cancelFunc() startTime := time.Now() for time.Now().Before(startTime.Add(70 * time.Second)) { - curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:8443") // -sS turns off the progressbar but still prints errors + curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10443") // -sS turns off the progressbar but still prints errors curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr curlErr := curlCmd.Run() @@ -364,7 +349,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // curl the endpoint once more, once 70 seconds has elapsed, to make sure the connection is still open. timeout, cancelFunc = context.WithTimeout(ctx, 30*time.Second) defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:8443") // -sS turns off the progressbar but still prints errors + curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10443") // -sS turns off the progressbar but still prints errors curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr curlErr := curlCmd.Run() @@ -376,16 +361,17 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // We expect this to 403, but all we care is that it gets through. require.NoError(t, curlErr) - require.Contains(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + require.Contains(t, curlStdOut.String(), `"forbidden: User \"system:anonymous\" cannot get path \"/\""`) }) t.Run("kubectl port-forward and keeping the connection open for over a minute (idle)", func(t *testing.T) { + t.Parallel() kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator) // Run the kubectl port-forward command. timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "8443:8443") + portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", env.ConciergeNamespace, conciergePod.Name, "10444:8443") portForwardCmd.Env = envVarsWithProxy // Start, but don't wait for the command to finish. @@ -401,7 +387,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() - curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:8443") // -sS turns off the progressbar but still prints errors + curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10444") // -sS turns off the progressbar but still prints errors var curlStdOut, curlStdErr bytes.Buffer curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr @@ -413,10 +399,11 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl } // We expect this to 403, but all we care is that it gets through. require.NoError(t, err) - require.Contains(t, curlStdOut.String(), "\"forbidden: User \\\"system:anonymous\\\" cannot get path \\\"/\\\"\"") + require.Contains(t, curlStdOut.String(), `"forbidden: User \"system:anonymous\" cannot get path \"/\""`) }) t.Run("using and watching all the basic verbs", func(t *testing.T) { + t.Parallel() // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := createTestNamespace(t, adminClient) @@ -536,10 +523,12 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Len(t, listResult.Items, 0) }) - t.Run("double impersonation as a regular user is blocked", func(t *testing.T) { + t.Run("nested impersonation as a regular user is allowed if they have enough RBAC permissions", func(t *testing.T) { + t.Parallel() // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. - doubleImpersonationKubeClient := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate").Kubernetes + nestedImpersonationClient := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, + &rest.ImpersonationConfig{UserName: "other-user-to-impersonate"}, refreshCredential) // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. // We could use any resource for this, but we happen to know that this one should exist. @@ -548,68 +537,236 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Now we'll see what happens when we add an impersonation header to the request. This should generate a // request similar to the one above, except that it will also have an impersonation header. - _, err = doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - // Double impersonation is not supported yet, so we should get an error. + _, err = nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // this user is not allowed to impersonate other users + require.True(t, k8serrors.IsForbidden(err), err) require.EqualError(t, err, fmt.Sprintf( `users "other-user-to-impersonate" is forbidden: `+ - `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ - `impersonation is not allowed or invalid verb`, + `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope`, env.TestUser.ExpectedUsername)) - }) - // This is a separate test from the above double impersonation test because the cluster admin user gets special - // authorization treatment from the Kube API server code that we are using, and we want to ensure that we are blocking - // double impersonation even for the cluster admin. - t.Run("double impersonation as a cluster admin user is blocked", func(t *testing.T) { - // Copy the admin credentials from the admin kubeconfig. - adminClientRestConfig := library.NewClientConfig(t) + // impersonate the GC service account instead which can read anything (the binding to edit allows this) + nestedImpersonationClientAsSA := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, + &rest.ImpersonationConfig{UserName: "system:serviceaccount:kube-system:generic-garbage-collector"}, refreshCredential) - if adminClientRestConfig.BearerToken == "" && adminClientRestConfig.CertData == nil && adminClientRestConfig.KeyData == nil { - t.Skip("The admin kubeconfig does not include credentials, so skipping this test.") + _, err = nestedImpersonationClientAsSA.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + require.NoError(t, err) + + expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups + expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...) + expectedGroups = append(expectedGroups, "system:authenticated") + expectedOriginalUserInfo := authenticationv1.UserInfo{ + Username: env.TestUser.ExpectedUsername, + Groups: expectedGroups, } + expectedOriginalUserInfoJSON, err := json.Marshal(expectedOriginalUserInfo) + require.NoError(t, err) - clusterAdminCredentials := &loginv1alpha1.ClusterCredential{ - Token: adminClientRestConfig.BearerToken, - ClientCertificateData: string(adminClientRestConfig.CertData), - ClientKeyData: string(adminClientRestConfig.KeyData), - } - - // Make a client using the admin credentials which will send requests through the impersonation proxy - // and will also add impersonate headers to the request. - doubleImpersonationKubeClient := newImpersonationProxyClientWithCredentials( - clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, "other-user-to-impersonate", - ).Kubernetes - - _, err := doubleImpersonationKubeClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) - // Double impersonation is not supported yet, so we should get an error. - require.Error(t, err) - require.Regexp(t, - `users "other-user-to-impersonate" is forbidden: `+ - `User ".*" cannot impersonate resource "users" in API group "" at the cluster scope: `+ - `impersonation is not allowed or invalid verb`, - err.Error(), - ) - }) - - t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { - // Test using the TokenCredentialRequest for authentication. - impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient(t, - impersonationProxyURL, impersonationProxyCACertPEM, "", - ).PinnipedConcierge - whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + // check that we impersonated the correct user and that the original user is retained in the extra + whoAmI, err := nestedImpersonationClientAsSA.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + "system:serviceaccount:kube-system:generic-garbage-collector", + []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"}, + map[string]identityv1alpha1.ExtraValue{ + "original-user-info.impersonation-proxy.concierge.pinniped.dev": {string(expectedOriginalUserInfoJSON)}, + }, + ), + whoAmI, + ) + + _, err = newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, + &rest.ImpersonationConfig{ + UserName: "system:serviceaccount:kube-system:generic-garbage-collector", + Extra: map[string][]string{ + "some-fancy-key": {"with a dangerous value"}, + }, + }, + refreshCredential).PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + // this user should not be able to impersonate extra + require.True(t, k8serrors.IsForbidden(err), err) + require.EqualError(t, err, fmt.Sprintf( + `userextras.authentication.k8s.io "with a dangerous value" is forbidden: `+ + `User "%s" cannot impersonate resource "userextras/some-fancy-key" in API group "authentication.k8s.io" at the cluster scope`, + env.TestUser.ExpectedUsername)) + }) + + t.Run("nested impersonation as a cluster admin user is allowed", func(t *testing.T) { + t.Parallel() + // Copy the admin credentials from the admin kubeconfig. + adminClientRestConfig := library.NewClientConfig(t) + clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig) + + // figure out who the admin user is + whoAmIAdmin, err := newImpersonationProxyClientWithCredentials(t, + clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil). + PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + + expectedExtra := make(map[string]authenticationv1.ExtraValue, len(whoAmIAdmin.Status.KubernetesUserInfo.User.Extra)) + for k, v := range whoAmIAdmin.Status.KubernetesUserInfo.User.Extra { + expectedExtra[k] = authenticationv1.ExtraValue(v) + } + expectedOriginalUserInfo := authenticationv1.UserInfo{ + Username: whoAmIAdmin.Status.KubernetesUserInfo.User.Username, + // The WhoAmI API is lossy so this will fail when the admin user actually does have a UID + UID: whoAmIAdmin.Status.KubernetesUserInfo.User.UID, + Groups: whoAmIAdmin.Status.KubernetesUserInfo.User.Groups, + Extra: expectedExtra, + } + expectedOriginalUserInfoJSON, err := json.Marshal(expectedOriginalUserInfo) + require.NoError(t, err) + + // Make a client using the admin credentials which will send requests through the impersonation proxy + // and will also add impersonate headers to the request. + nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t, + clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, + &rest.ImpersonationConfig{ + UserName: "other-user-to-impersonate", + Groups: []string{"other-group-1", "other-group-2"}, + Extra: map[string][]string{ + "this-key": {"to this value"}, + }, + }, + ) + + _, err = nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + // the impersonated user lacks the RBAC to perform this call + require.True(t, k8serrors.IsForbidden(err), err) + require.EqualError(t, err, fmt.Sprintf( + `secrets "%s" is forbidden: User "other-user-to-impersonate" cannot get resource "secrets" in API group "" in the namespace "%s"`, + impersonationProxyTLSSecretName(env), env.ConciergeNamespace, + )) + + // check that we impersonated the correct user and that the original user is retained in the extra + whoAmI, err := nestedImpersonationClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + "other-user-to-impersonate", + []string{"other-group-1", "other-group-2", "system:authenticated"}, + map[string]identityv1alpha1.ExtraValue{ + "this-key": {"to this value"}, + "original-user-info.impersonation-proxy.concierge.pinniped.dev": {string(expectedOriginalUserInfoJSON)}, + }, + ), + whoAmI, + ) + }) + + t.Run("nested impersonation as a cluster admin fails on reserved key", func(t *testing.T) { + t.Parallel() + adminClientRestConfig := library.NewClientConfig(t) + clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig) + + nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t, + clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, + &rest.ImpersonationConfig{ + UserName: "other-user-to-impersonate", + Groups: []string{"other-group-1", "other-group-2"}, + Extra: map[string][]string{ + "this-good-key": {"to this good value"}, + "something.impersonation-proxy.concierge.pinniped.dev": {"super sneaky value"}, + }, + }, + ) + + _, err := nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) + require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") + require.True(t, k8serrors.IsInternalError(err), err) + require.Equal(t, &k8serrors.StatusError{ + ErrStatus: metav1.Status{ + Status: metav1.StatusFailure, + Code: http.StatusInternalServerError, + Reason: metav1.StatusReasonInternalError, + Details: &metav1.StatusDetails{ + Causes: []metav1.StatusCause{ + { + Message: "unimplemented functionality - unable to act as current user", + }, + }, + }, + Message: "Internal error occurred: unimplemented functionality - unable to act as current user", + }, + }, err) + }) + + // this works because impersonation cannot set UID and thus the final user info the proxy sees has no UID + t.Run("nested impersonation as a service account is allowed if it has enough RBAC permissions", func(t *testing.T) { + t.Parallel() + namespaceName := createTestNamespace(t, adminClient) + saName, saToken, saUID := createServiceAccountToken(ctx, t, adminClient, namespaceName) + nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t, + &loginv1alpha1.ClusterCredential{Token: saToken}, impersonationProxyURL, impersonationProxyCACertPEM, + &rest.ImpersonationConfig{UserName: "system:serviceaccount:kube-system:root-ca-cert-publisher"}).PinnipedConcierge + _, err := nestedImpersonationClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + // this SA is not yet allowed to impersonate SAs + require.True(t, k8serrors.IsForbidden(err), err) + require.EqualError(t, err, fmt.Sprintf( + `serviceaccounts "root-ca-cert-publisher" is forbidden: `+ + `User "%s" cannot impersonate resource "serviceaccounts" in API group "" in the namespace "kube-system"`, + serviceaccount.MakeUsername(namespaceName, saName))) + + // webhook authorizer deny cache TTL is 10 seconds so we need to wait long enough for it to drain + time.Sleep(15 * time.Second) + + // allow the test SA to impersonate any SA + library.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: saName, Namespace: namespaceName}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, + ) + library.WaitForUserToHaveAccess(t, serviceaccount.MakeUsername(namespaceName, saName), []string{}, &authorizationv1.ResourceAttributes{ + Verb: "impersonate", Group: "", Version: "v1", Resource: "serviceaccounts", + }) + + whoAmI, err := nestedImpersonationClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + require.Equal(t, + expectedWhoAmIRequestResponse( + "system:serviceaccount:kube-system:root-ca-cert-publisher", + []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"}, + map[string]identityv1alpha1.ExtraValue{ + "original-user-info.impersonation-proxy.concierge.pinniped.dev": { + fmt.Sprintf(`{"username":"%s","uid":"%s","groups":["system:serviceaccounts","system:serviceaccounts:%s","system:authenticated"]}`, + serviceaccount.MakeUsername(namespaceName, saName), saUID, namespaceName), + }, + }, + ), + whoAmI, + ) + }) + + t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { + t.Parallel() + // Test using the TokenCredentialRequest for authentication. + impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient(t, + impersonationProxyURL, impersonationProxyCACertPEM, nil, refreshCredential, + ).PinnipedConcierge + whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). + Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) + require.NoError(t, err) + expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups + expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...) + expectedGroups = append(expectedGroups, "system:authenticated") require.Equal(t, expectedWhoAmIRequestResponse( env.TestUser.ExpectedUsername, - append(env.TestUser.ExpectedGroups, "system:authenticated"), + expectedGroups, + nil, ), whoAmI, ) // Test an unauthenticated request which does not include any credentials. impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient( - impersonationProxyURL, impersonationProxyCACertPEM, "", + t, impersonationProxyURL, impersonationProxyCACertPEM, nil, ).PinnipedConcierge whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) @@ -618,6 +775,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl expectedWhoAmIRequestResponse( "system:anonymous", []string{"system:unauthenticated"}, + nil, ), whoAmI, ) @@ -625,9 +783,10 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // Test using a service account token. Authenticating as Service Accounts through the impersonation // proxy is not supported, so it should fail. namespaceName := createTestNamespace(t, adminClient) - impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials( - &loginv1alpha1.ClusterCredential{Token: createServiceAccountToken(ctx, t, adminClient, namespaceName)}, - impersonationProxyURL, impersonationProxyCACertPEM, "").PinnipedConcierge + _, saToken, _ := createServiceAccountToken(ctx, t, adminClient, namespaceName) + impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials(t, + &loginv1alpha1.ClusterCredential{Token: saToken}, + impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge _, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") @@ -650,6 +809,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("kubectl as a client", func(t *testing.T) { + t.Parallel() kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator) // Try "kubectl exec" through the impersonation proxy. @@ -720,11 +880,12 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("websocket client", func(t *testing.T) { + t.Parallel() namespaceName := createTestNamespace(t, adminClient) impersonationRestConfig := impersonationProxyRestConfig( refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM), - impersonationProxyURL, impersonationProxyCACertPEM, "", + impersonationProxyURL, impersonationProxyCACertPEM, nil, ) tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) require.NoError(t, err) @@ -793,6 +954,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl }) t.Run("http2 client", func(t *testing.T) { + t.Parallel() namespaceName := createTestNamespace(t, adminClient) wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" @@ -811,7 +973,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl // create rest client restConfig := impersonationProxyRestConfig( refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM), - impersonationProxyURL, impersonationProxyCACertPEM, "", + impersonationProxyURL, impersonationProxyCACertPEM, nil, ) tlsConfig, err := rest.TLSConfigFor(restConfig) @@ -916,7 +1078,7 @@ func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's compl require.Eventually(t, func() bool { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. - _, err := impersonationProxyViaSquidKubeClientWithoutCredential().CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + _, err := impersonationProxyViaSquidKubeClientWithoutCredential(t, proxyServiceEndpoint).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) isErr, _ := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) return isErr }, 20*time.Second, 500*time.Millisecond) @@ -977,7 +1139,7 @@ func createTestNamespace(t *testing.T, adminClient kubernetes.Interface) string return namespace.Name } -func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, namespaceName string) string { +func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, namespaceName string) (name, token string, uid types.UID) { t.Helper() serviceAccount, err := adminClient.CoreV1().ServiceAccounts(namespaceName).Create(ctx, @@ -1011,10 +1173,10 @@ func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient ku return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil }, time.Minute, time.Second) - return string(secret.Data[corev1.ServiceAccountTokenKey]) + return serviceAccount.Name, string(secret.Data[corev1.ServiceAccountTokenKey]), serviceAccount.UID } -func expectedWhoAmIRequestResponse(username string, groups []string) *identityv1alpha1.WhoAmIRequest { +func expectedWhoAmIRequestResponse(username string, groups []string, extra map[string]identityv1alpha1.ExtraValue) *identityv1alpha1.WhoAmIRequest { return &identityv1alpha1.WhoAmIRequest{ Status: identityv1alpha1.WhoAmIRequestStatus{ KubernetesUserInfo: identityv1alpha1.KubernetesUserInfo{ @@ -1022,7 +1184,7 @@ func expectedWhoAmIRequestResponse(username string, groups []string) *identityv1 Username: username, UID: "", // no way to impersonate UID: https://github.com/kubernetes/kubernetes/issues/93699 Groups: groups, - Extra: nil, + Extra: extra, }, }, }, @@ -1031,6 +1193,7 @@ func expectedWhoAmIRequestResponse(username string, groups []string) *identityv1 func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *library.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface) (string, []byte) { t.Helper() + var impersonationProxyURL string var impersonationProxyCACertPEM []byte @@ -1105,6 +1268,7 @@ func requireDisabledStrategy(ctx context.Context, t *testing.T, env *library.Tes func credentialAlmostExpired(t *testing.T, credential *loginv1alpha1.TokenCredentialRequest) bool { t.Helper() + pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) require.NoError(t, err) @@ -1117,7 +1281,7 @@ func credentialAlmostExpired(t *testing.T, credential *loginv1alpha1.TokenCreden return false } -func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, doubleImpersonateUser string) *rest.Config { +func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *rest.Config { config := rest.Config{ Host: host, TLSClientConfig: rest.TLSClientConfig{ @@ -1135,8 +1299,8 @@ func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, h // We would like the impersonation proxy to imitate that behavior, so we test it here. BearerToken: credential.Token, } - if doubleImpersonateUser != "" { - config.Impersonate = rest.ImpersonationConfig{UserName: doubleImpersonateUser} + if nestedImpersonationConfig != nil { + config.Impersonate = *nestedImpersonationConfig } return &config } @@ -1144,6 +1308,7 @@ func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, h func kubeconfigProxyFunc(t *testing.T, squidProxyURL string) func(req *http.Request) (*url.URL, error) { return func(req *http.Request) (*url.URL, error) { t.Helper() + parsedSquidProxyURL, err := url.Parse(squidProxyURL) require.NoError(t, err) t.Logf("passing request for %s through proxy %s", req.URL, parsedSquidProxyURL.String()) @@ -1153,6 +1318,7 @@ func kubeconfigProxyFunc(t *testing.T, squidProxyURL string) func(req *http.Requ func impersonationProxyConfigMapForConfig(t *testing.T, env *library.TestEnv, config impersonator.Config) corev1.ConfigMap { t.Helper() + configString, err := yaml.Marshal(config) require.NoError(t, err) configMap := corev1.ConfigMap{ @@ -1244,6 +1410,8 @@ func getImpersonationKubeconfig(t *testing.T, env *library.TestEnv, impersonatio // func to create kubectl commands with a kubeconfig. func kubectlCommand(timeout context.Context, t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) { + t.Helper() + allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) //nolint:gosec // we are not performing malicious argument injection against ourselves kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) @@ -1296,6 +1464,7 @@ func isServiceUnavailableViaSquidError(err error, proxyServiceEndpoint string) ( func requireClose(t *testing.T, c chan struct{}, timeout time.Duration) { t.Helper() + timer := time.NewTimer(timeout) select { case <-c: @@ -1317,3 +1486,117 @@ func createTokenCredentialRequest( &loginv1alpha1.TokenCredentialRequest{Spec: spec}, metav1.CreateOptions{}, ) } + +func newImpersonationProxyClientWithCredentials(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client { + t.Helper() + + env := library.IntegrationEnv(t) + clusterSupportsLoadBalancers := env.HasCapability(library.HasExternalLoadBalancerProvider) + + kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) + if !clusterSupportsLoadBalancers { + // Only if there is no possibility to send traffic through a load balancer, then send the traffic through the Squid proxy. + // Prefer to go through a load balancer because that's how the impersonator is intended to be used in the real world. + kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) + } + return library.NewKubeclient(t, kubeconfig) +} + +func newAnonymousImpersonationProxyClient(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client { + t.Helper() + + emptyCredentials := &loginv1alpha1.ClusterCredential{} + return newImpersonationProxyClientWithCredentials(t, emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) +} + +func impersonationProxyViaSquidKubeClientWithoutCredential(t *testing.T, proxyServiceEndpoint string) kubernetes.Interface { + t.Helper() + + env := library.IntegrationEnv(t) + proxyURL := "https://" + proxyServiceEndpoint + kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, nil) + kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) + return library.NewKubeclient(t, kubeconfig).Kubernetes +} + +func newImpersonationProxyClient( + t *testing.T, + impersonationProxyURL string, + impersonationProxyCACertPEM []byte, + nestedImpersonationConfig *rest.ImpersonationConfig, + refreshCredentialFunc func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential, +) *kubeclient.Client { + t.Helper() + + refreshedCredentials := refreshCredentialFunc(t, impersonationProxyURL, impersonationProxyCACertPEM).DeepCopy() + refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests + return newImpersonationProxyClientWithCredentials(t, refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) +} + +// getCredForConfig is mostly just a hacky workaround for impersonationProxyRestConfig needing creds directly. +func getCredForConfig(t *testing.T, config *rest.Config) *loginv1alpha1.ClusterCredential { + t.Helper() + + out := &loginv1alpha1.ClusterCredential{} + + config = rest.CopyConfig(config) + + config.Wrap(func(rt http.RoundTripper) http.RoundTripper { + return roundtripper.Func(func(req *http.Request) (*http.Response, error) { + resp, err := rt.RoundTrip(req) + + r := req + if resp != nil && resp.Request != nil { + r = resp.Request + } + + _, _, _ = bearertoken.New(authenticator.TokenFunc(func(_ context.Context, token string) (*authenticator.Response, bool, error) { + out.Token = token + return nil, false, nil + })).AuthenticateRequest(r) + + return resp, err + }) + }) + + transportConfig, err := config.TransportConfig() + require.NoError(t, err) + + rt, err := transport.New(transportConfig) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, "GET", "https://localhost", nil) + require.NoError(t, err) + resp, _ := rt.RoundTrip(req) + if resp != nil && resp.Body != nil { + _ = resp.Body.Close() + } + + tlsConfig, err := transport.TLSConfigFor(transportConfig) + require.NoError(t, err) + + if tlsConfig != nil && tlsConfig.GetClientCertificate != nil { + cert, err := tlsConfig.GetClientCertificate(nil) + require.NoError(t, err) + require.Len(t, cert.Certificate, 1) + + publicKey := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Certificate[0], + }) + out.ClientCertificateData = string(publicKey) + + privateKey, err := keyutil.MarshalPrivateKeyToPEM(cert.PrivateKey) + require.NoError(t, err) + out.ClientKeyData = string(privateKey) + } + + if *out == (loginv1alpha1.ClusterCredential{}) { + t.Fatal("failed to get creds for config") + } + + return out +} From 73716f1b911b475b8bb461fb1ef5b1df77c2670b Mon Sep 17 00:00:00 2001 From: Monis Khan Date: Mon, 19 Apr 2021 06:23:09 -0400 Subject: [PATCH 02/10] Ignore client-side throttling in kubectl stderr Signed-off-by: Monis Khan --- test/integration/category_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/integration/category_test.go b/test/integration/category_test.go index 592ccb24..253d0d62 100644 --- a/test/integration/category_test.go +++ b/test/integration/category_test.go @@ -65,7 +65,10 @@ func requireCleanKubectlStderr(t *testing.T, stderr string) { if strings.Contains(line, "Throttling request took") { continue } - require.Failf(t, "unexpected kubectl stderr", "kubectl produced unexpected stderr output:\n%s\n\n", stderr) + if strings.Contains(line, "due to client-side throttling, not priority and fairness") { + continue + } + require.Failf(t, "unexpected kubectl stderr", "kubectl produced unexpected stderr:\n%s\n\n", stderr) return } } From d86b24ca2f1c136da5cf0e306726f9da2cfc3d8f Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Mon, 19 Apr 2021 16:10:20 -0400 Subject: [PATCH 03/10] hack: add prepare-webhook-on-kind.sh Inspired from 7bb5657c4d5. I used this to help accept 2 stories today. Signed-off-by: Andrew Keesler --- hack/prepare-webhook-on-kind.sh | 37 +++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 hack/prepare-webhook-on-kind.sh diff --git a/hack/prepare-webhook-on-kind.sh b/hack/prepare-webhook-on-kind.sh new file mode 100755 index 00000000..7f67e3ac --- /dev/null +++ b/hack/prepare-webhook-on-kind.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2021 the Pinniped contributors. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +# +# This script deploys a WebhookAuthenticator to use for manual testing. It +# assumes that you have run hack/prepare-for-integration-tests.sh while pointed +# at the current cluster. +# + +set -euo pipefail + +# Change working directory to the top of the repo. +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT" + +# Read the env vars output by hack/prepare-for-integration-tests.sh. +source /tmp/integration-test-env + +# Create WebhookAuthenticator. +cat <&2 +kind: WebhookAuthenticator +apiVersion: authentication.concierge.pinniped.dev/v1alpha1 +metadata: + name: my-webhook +spec: + endpoint: ${PINNIPED_TEST_WEBHOOK_ENDPOINT} + tls: + certificateAuthorityData: ${PINNIPED_TEST_WEBHOOK_CA_BUNDLE} +EOF + +# Use the CLI to get a kubeconfig that will use this WebhookAuthenticator. +go build -o /tmp/pinniped ./cmd/pinniped +/tmp/pinniped get kubeconfig --static-token "$PINNIPED_TEST_USER_TOKEN" >/tmp/kubeconfig-with-webhook-auth.yaml + +echo "export KUBECONFIG=/tmp/kubeconfig-with-webhook-auth.yaml" From 9f509d3f13f9d1b5aac76dfe99da923f25d0fbc5 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Wed, 21 Apr 2021 08:58:20 -0400 Subject: [PATCH 04/10] internal/kubeclient: match plog level with klog level Signed-off-by: Andrew Keesler --- internal/kubeclient/copied.go | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/internal/kubeclient/copied.go b/internal/kubeclient/copied.go index a822deaa..3b4efd9b 100644 --- a/internal/kubeclient/copied.go +++ b/internal/kubeclient/copied.go @@ -6,7 +6,6 @@ package kubeclient import ( "bytes" "encoding/hex" - "fmt" "net/url" "k8s.io/apimachinery/pkg/runtime/schema" @@ -32,39 +31,17 @@ func defaultServerUrlFor(config *restclient.Config) (*url.URL, string, error) { return restclient.DefaultServerURL(host, config.APIPath, schema.GroupVersion{}, defaultTLS) } -// truncateBody was copied from k8s.io/client-go/rest/request.go -// ...except i changed klog invocations to analogous plog invocations -// -// truncateBody decides if the body should be truncated, based on the glog Verbosity. -func truncateBody(body string) string { - max := 0 - switch { - case plog.Enabled(plog.LevelAll): - return body - case plog.Enabled(plog.LevelTrace): - max = 10240 - case plog.Enabled(plog.LevelDebug): - max = 1024 - } - - if len(body) <= max { - return body - } - - return body[:max] + fmt.Sprintf(" [truncated %d chars]", len(body)-max) -} - // glogBody logs a body output that could be either JSON or protobuf. It explicitly guards against // allocating a new string for the body output unless necessary. Uses a simple heuristic to determine // whether the body is printable. func glogBody(prefix string, body []byte) { - if plog.Enabled(plog.LevelDebug) { + if plog.Enabled(plog.LevelAll) { if bytes.IndexFunc(body, func(r rune) bool { return r < 0x0a }) != -1 { - plog.Debug(prefix, "body", truncateBody(hex.Dump(body))) + plog.Debug(prefix, "body", hex.Dump(body)) } else { - plog.Debug(prefix, "body", truncateBody(string(body))) + plog.Debug(prefix, "body", string(body)) } } } From 638d9235a27694c48eddea168db358f7cae44285 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Thu, 22 Apr 2021 10:25:44 -0500 Subject: [PATCH 05/10] Remove unneeded OIDC-related sleeps in tests. Now that we have the fix from https://github.com/kubernetes/kubernetes/pull/97693, we no longer need these sleeps. The underlying authenticator initialization is still asynchronous, but should happen within a few milliseconds. Signed-off-by: Matt Moyer --- .../jwtcachefiller/jwtcachefiller_test.go | 29 ++++++++++++------- test/integration/e2e_test.go | 5 ---- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go b/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go index 5a43751e..19eb1731 100644 --- a/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go +++ b/internal/controller/authenticator/jwtcachefiller/jwtcachefiller_test.go @@ -16,6 +16,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -25,6 +26,7 @@ import ( "gopkg.in/square/go-jose.v2/jwt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" @@ -345,15 +347,6 @@ func TestController(t *testing.T) { return // end of test unless we wanted to run tests on the resulting authenticator from the cache } - // The implementation of AuthenticateToken() that we use waits 10 seconds after creation to - // perform OIDC discovery. Therefore, the JWTAuthenticator is not functional for the first 10 - // seconds. We sleep for 13 seconds in this unit test to give a little bit of cushion to that 10 - // second delay. - // - // We should get rid of this 10 second delay. See - // https://github.com/vmware-tanzu/pinniped/issues/260. - time.Sleep(time.Second * 13) - // We expected the cache to have an entry, so pull that entry from the cache and test it. expectedCacheKey := authncache.Key{ APIGroup: auth1alpha1.GroupName, @@ -428,7 +421,17 @@ func TestController(t *testing.T) { tt.wantUsernameClaim, username, ) - rsp, authenticated, err := cachedAuthenticator.AuthenticateToken(context.Background(), jwt) + + // Loop for a while here to allow the underlying OIDC authenticator to initialize itself asynchronously. + var ( + rsp *authenticator.Response + authenticated bool + err error + ) + _ = wait.PollImmediate(10*time.Millisecond, 5*time.Second, func() (bool, error) { + rsp, authenticated, err = cachedAuthenticator.AuthenticateToken(context.Background(), jwt) + return !isNotInitialized(err), nil + }) if test.wantErrorRegexp != "" { require.Error(t, err) require.Regexp(t, test.wantErrorRegexp, err.Error()) @@ -443,6 +446,12 @@ func TestController(t *testing.T) { } } +// isNotInitialized checks if the error is the internally-defined "oidc: authenticator not initialized" error from +// the underlying OIDC authenticator, which is initialized asynchronously. +func isNotInitialized(err error) bool { + return err != nil && strings.Contains(err.Error(), "authenticator not initialized") +} + func testTableForAuthenticateTokenTests( t *testing.T, goodRSASigningKey *rsa.PrivateKey, diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index 2cb207e7..b70a5053 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -151,11 +151,6 @@ func TestE2EFullIntegration(t *testing.T) { kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") require.NoError(t, ioutil.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) - // Wait 10 seconds for the JWTAuthenticator to become initialized. - // TODO: remove this sleep once we have fixed the initialization problem. - t.Log("sleeping 10s to wait for JWTAuthenticator to become initialized") - time.Sleep(10 * time.Second) - // Run "kubectl get namespaces" which should trigger a browser login via the plugin. start := time.Now() kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) From 2843c4f8cbf23260854c0b2b8e7aa1ce3500d904 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 20 Apr 2021 14:55:28 -0500 Subject: [PATCH 06/10] Refactor kube-cert-agent controllers to use a Deployment. This is a relatively large rewrite of much of the kube-cert-agent controllers. Instead of managing raw Pod objects, they now create a single Deployment and let the builtin k8s controller handle it from there. This reduces the amount of code we need and should handle a number of edge cases better, especially those where a Pod becomes "wedged" and needs to be recreated. Signed-off-by: Matt Moyer --- deploy/concierge/rbac.yaml | 13 +- .../controller/kubecertagent/annotater.go | 219 ---- .../kubecertagent/annotater_test.go | 727 ------------- internal/controller/kubecertagent/creater.go | 185 ---- .../controller/kubecertagent/creater_test.go | 623 ----------- internal/controller/kubecertagent/deleter.go | 87 -- .../controller/kubecertagent/deleter_test.go | 506 --------- internal/controller/kubecertagent/execer.go | 232 ---- .../controller/kubecertagent/execer_test.go | 733 ------------- .../controller/kubecertagent/kubecertagent.go | 712 ++++++++----- .../kubecertagent/kubecertagent_test.go | 998 ++++++++++++++---- .../kubecertagent/pod_command_executor.go | 1 + .../controllermanager/prepare_controllers.go | 68 +- .../concierge_credentialissuer_test.go | 7 +- .../concierge_kubecertagent_test.go | 200 +--- 15 files changed, 1343 insertions(+), 3968 deletions(-) delete mode 100644 internal/controller/kubecertagent/annotater.go delete mode 100644 internal/controller/kubecertagent/annotater_test.go delete mode 100644 internal/controller/kubecertagent/creater.go delete mode 100644 internal/controller/kubecertagent/creater_test.go delete mode 100644 internal/controller/kubecertagent/deleter.go delete mode 100644 internal/controller/kubecertagent/deleter_test.go delete mode 100644 internal/controller/kubecertagent/execer.go delete mode 100644 internal/controller/kubecertagent/execer_test.go diff --git a/deploy/concierge/rbac.yaml b/deploy/concierge/rbac.yaml index 6370d380..6bd56fe0 100644 --- a/deploy/concierge/rbac.yaml +++ b/deploy/concierge/rbac.yaml @@ -47,7 +47,7 @@ rules: - apiGroups: - #@ pinnipedDevAPIGroupWithPrefix("config.concierge") resources: [ credentialissuers/status ] - verbs: [get, patch, update] + verbs: [ get, patch, update ] - apiGroups: - #@ pinnipedDevAPIGroupWithPrefix("authentication.concierge") resources: [ jwtauthenticators, webhookauthenticators ] @@ -82,16 +82,21 @@ rules: - apiGroups: [ "" ] resources: [ secrets ] verbs: [ create, get, list, patch, update, watch, delete ] - #! We need to be able to CRUD pods in our namespace so we can reconcile the kube-cert-agent pods. + #! We need to be able to watch pods in our namespace so we can find the kube-cert-agent pods. - apiGroups: [ "" ] resources: [ pods ] - verbs: [ create, get, list, patch, update, watch, delete ] + verbs: [ get, list, watch ] #! We need to be able to exec into pods in our namespace so we can grab the API server's private key - apiGroups: [ "" ] resources: [ pods/exec ] verbs: [ create ] + #! We need to be able to create and update deployments in our namespace so we can manage the kube-cert-agent Deployment. - apiGroups: [ apps ] - resources: [ replicasets,deployments ] + resources: [ deployments ] + verbs: [ create, get, list, patch, update, watch ] + #! We need to be able to get replicasets so we can form the correct owner references on our generated objects. + - apiGroups: [ apps ] + resources: [ replicasets ] verbs: [ get ] - apiGroups: [ "" ] resources: [ configmaps ] diff --git a/internal/controller/kubecertagent/annotater.go b/internal/controller/kubecertagent/annotater.go deleted file mode 100644 index fa640d21..00000000 --- a/internal/controller/kubecertagent/annotater.go +++ /dev/null @@ -1,219 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "context" - "fmt" - - "github.com/spf13/pflag" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/clock" - corev1informers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/util/retry" - "k8s.io/klog/v2" - - pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" - pinnipedcontroller "go.pinniped.dev/internal/controller" - "go.pinniped.dev/internal/controller/issuerconfig" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/plog" -) - -// These constants are the default values for the kube-controller-manager flags. If the flags are -// not properly set on the kube-controller-manager process, then we will fallback to using these. -const ( - k8sAPIServerCACertPEMDefaultPath = "/etc/kubernetes/ca/ca.pem" - k8sAPIServerCAKeyPEMDefaultPath = "/etc/kubernetes/ca/ca.key" -) - -type annotaterController struct { - agentPodConfig *AgentPodConfig - credentialIssuerLocationConfig *CredentialIssuerLocationConfig - credentialIssuerLabels map[string]string - clock clock.Clock - k8sClient kubernetes.Interface - pinnipedAPIClient pinnipedclientset.Interface - kubeSystemPodInformer corev1informers.PodInformer - agentPodInformer corev1informers.PodInformer -} - -// NewAnnotaterController returns a controller that updates agent pods with the path to the kube -// API's certificate and key. -// -// This controller will add annotations to agent pods with the best-guess paths to the kube API's -// certificate and key. -// -// It also is tasked with updating the CredentialIssuer, located via the provided -// credentialIssuerLocationConfig, with any errors that it encounters. -func NewAnnotaterController( - agentPodConfig *AgentPodConfig, - credentialIssuerLocationConfig *CredentialIssuerLocationConfig, - credentialIssuerLabels map[string]string, - clock clock.Clock, - k8sClient kubernetes.Interface, - pinnipedAPIClient pinnipedclientset.Interface, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - withInformer pinnipedcontroller.WithInformerOptionFunc, -) controllerlib.Controller { - return controllerlib.New( - controllerlib.Config{ - Name: "kube-cert-agent-annotater-controller", - Syncer: &annotaterController{ - agentPodConfig: agentPodConfig, - credentialIssuerLocationConfig: credentialIssuerLocationConfig, - credentialIssuerLabels: credentialIssuerLabels, - clock: clock, - k8sClient: k8sClient, - pinnipedAPIClient: pinnipedAPIClient, - kubeSystemPodInformer: kubeSystemPodInformer, - agentPodInformer: agentPodInformer, - }, - }, - withInformer( - kubeSystemPodInformer, - pinnipedcontroller.SimpleFilterWithSingletonQueue(isControllerManagerPod), - controllerlib.InformerOption{}, - ), - withInformer( - agentPodInformer, - pinnipedcontroller.SimpleFilterWithSingletonQueue(isAgentPod), - controllerlib.InformerOption{}, - ), - ) -} - -// Sync implements controllerlib.Syncer. -func (c *annotaterController) Sync(ctx controllerlib.Context) error { - agentPods, err := c.agentPodInformer. - Lister(). - Pods(c.agentPodConfig.Namespace). - List(c.agentPodConfig.AgentSelector()) - if err != nil { - return fmt.Errorf("informer cannot list agent pods: %w", err) - } - - for _, agentPod := range agentPods { - controllerManagerPod, err := findControllerManagerPodForSpecificAgentPod(agentPod, c.kubeSystemPodInformer) - if err != nil { - return err - } - if controllerManagerPod == nil { - // The deleter will clean this orphaned agent. - continue - } - - certPath := getContainerArgByName( - controllerManagerPod, - "cluster-signing-cert-file", - k8sAPIServerCACertPEMDefaultPath, - ) - keyPath := getContainerArgByName( - controllerManagerPod, - "cluster-signing-key-file", - k8sAPIServerCAKeyPEMDefaultPath, - ) - if err := c.maybeUpdateAgentPod( - ctx.Context, - agentPod.Name, - agentPod.Namespace, - certPath, - keyPath, - ); err != nil { - err = fmt.Errorf("cannot update agent pod: %w", err) - strategyResultUpdateErr := issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - strategyError(c.clock, err), - ) - if strategyResultUpdateErr != nil { - // If the CI update fails, then we probably want to try again. This controller will get - // called again because of the pod create failure, so just try the CI update again then. - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer") - } - - return err - } - } - - return nil -} - -func (c *annotaterController) maybeUpdateAgentPod( - ctx context.Context, - name string, - namespace string, - certPath string, - keyPath string, -) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - agentPod, err := c.k8sClient.CoreV1().Pods(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return err - } - - if agentPod.Annotations[agentPodCertPathAnnotationKey] != certPath || - agentPod.Annotations[agentPodKeyPathAnnotationKey] != keyPath { - if err := c.reallyUpdateAgentPod( - ctx, - agentPod, - certPath, - keyPath, - ); err != nil { - return err - } - } - - return nil - }) -} - -func (c *annotaterController) reallyUpdateAgentPod( - ctx context.Context, - agentPod *corev1.Pod, - certPath string, - keyPath string, -) error { - // Create a deep copy of the agent pod since it is coming straight from the cache. - updatedAgentPod := agentPod.DeepCopy() - if updatedAgentPod.Annotations == nil { - updatedAgentPod.Annotations = make(map[string]string) - } - updatedAgentPod.Annotations[agentPodCertPathAnnotationKey] = certPath - updatedAgentPod.Annotations[agentPodKeyPathAnnotationKey] = keyPath - - plog.Debug( - "updating agent pod annotations", - "pod", - klog.KObj(updatedAgentPod), - "certPath", - certPath, - "keyPath", - keyPath, - ) - _, err := c.k8sClient. - CoreV1(). - Pods(agentPod.Namespace). - Update(ctx, updatedAgentPod, metav1.UpdateOptions{}) - return err -} - -func getContainerArgByName(pod *corev1.Pod, name, fallbackValue string) string { - for _, container := range pod.Spec.Containers { - flagset := pflag.NewFlagSet("", pflag.ContinueOnError) - flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true} - var val string - flagset.StringVar(&val, name, "", "") - _ = flagset.Parse(append(container.Command, container.Args...)) - if val != "" { - return val - } - } - return fallbackValue -} diff --git a/internal/controller/kubecertagent/annotater_test.go b/internal/controller/kubecertagent/annotater_test.go deleted file mode 100644 index 379133fa..00000000 --- a/internal/controller/kubecertagent/annotater_test.go +++ /dev/null @@ -1,727 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/sclevine/spec" - "github.com/sclevine/spec/report" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/clock" - kubeinformers "k8s.io/client-go/informers" - corev1informers "k8s.io/client-go/informers/core/v1" - kubernetesfake "k8s.io/client-go/kubernetes/fake" - coretesting "k8s.io/client-go/testing" - - configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" - pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/testutil" -) - -func TestAnnotaterControllerFilter(t *testing.T) { - defineSharedKubecertagentFilterSpecs( - t, - "AnnotaterControllerFilter", - func( - agentPodConfig *AgentPodConfig, - _ *CredentialIssuerLocationConfig, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - observableWithInformerOption *testutil.ObservableWithInformerOption, - ) { - _ = NewAnnotaterController( - agentPodConfig, - nil, // credentialIssuerLabels, shouldn't matter - nil, // credentialIssuerLocationConfig, shouldn't matter - nil, // clock, shouldn't matter - nil, // k8sClient, shouldn't matter - nil, // pinnipedClient, shouldn't matter - kubeSystemPodInformer, - agentPodInformer, - observableWithInformerOption.WithInformer, - ) - }, - ) -} - -func TestAnnotaterControllerSync(t *testing.T) { - spec.Run(t, "AnnotaterControllerSync", func(t *testing.T, when spec.G, it spec.S) { - const kubeSystemNamespace = "kube-system" - const agentPodNamespace = "agent-pod-namespace" - const defaultKubeControllerManagerClusterSigningCertFileFlagValue = "/etc/kubernetes/ca/ca.pem" - const defaultKubeControllerManagerClusterSigningKeyFileFlagValue = "/etc/kubernetes/ca/ca.key" - const credentialIssuerResourceName = "ci-resource-name" - - const ( - certPath = "some-cert-path" - certPathAnnotation = "kube-cert-agent.pinniped.dev/cert-path" - - keyPath = "some-key-path" - keyPathAnnotation = "kube-cert-agent.pinniped.dev/key-path" - ) - - var r *require.Assertions - - var subject controllerlib.Controller - var kubeAPIClient *kubernetesfake.Clientset - var kubeSystemInformerClient *kubernetesfake.Clientset - var kubeSystemInformers kubeinformers.SharedInformerFactory - var agentInformerClient *kubernetesfake.Clientset - var agentInformers kubeinformers.SharedInformerFactory - var pinnipedAPIClient *pinnipedfake.Clientset - var cancelContext context.Context - var cancelContextCancelFunc context.CancelFunc - var syncContext *controllerlib.Context - var controllerManagerPod, agentPod *corev1.Pod - var podsGVR schema.GroupVersionResource - var credentialIssuerGVR schema.GroupVersionResource - var frozenNow time.Time - var credentialIssuerLabels map[string]string - - // Defer starting the informers until the last possible moment so that the - // nested Before's can keep adding things to the informer caches. - var startInformersAndController = func() { - // Set this at the last second to allow for injection of server override. - subject = NewAnnotaterController( - &AgentPodConfig{ - Namespace: agentPodNamespace, - ContainerImage: "some-agent-image", - PodNamePrefix: "some-agent-name-", - AdditionalLabels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - &CredentialIssuerLocationConfig{ - Name: credentialIssuerResourceName, - }, - credentialIssuerLabels, - clock.NewFakeClock(frozenNow), - kubeAPIClient, - pinnipedAPIClient, - kubeSystemInformers.Core().V1().Pods(), - agentInformers.Core().V1().Pods(), - controllerlib.WithInformer, - ) - - // Set this at the last second to support calling subject.Name(). - syncContext = &controllerlib.Context{ - Context: cancelContext, - Name: subject.Name(), - Key: controllerlib.Key{ - Namespace: kubeSystemNamespace, - Name: "should-not-matter", - }, - } - - // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(cancelContext.Done()) - agentInformers.Start(cancelContext.Done()) - controllerlib.TestRunSynchronously(t, subject) - } - - it.Before(func() { - r = require.New(t) - - kubeAPIClient = kubernetesfake.NewSimpleClientset() - - kubeSystemInformerClient = kubernetesfake.NewSimpleClientset() - kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0) - - agentInformerClient = kubernetesfake.NewSimpleClientset() - agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0) - - pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - - cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) - - controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( - kubeSystemNamespace, agentPodNamespace, certPath, keyPath, - ) - - podsGVR = schema.GroupVersionResource{ - Group: corev1.SchemeGroupVersion.Group, - Version: corev1.SchemeGroupVersion.Version, - Resource: "pods", - } - - credentialIssuerGVR = schema.GroupVersionResource{ - Group: configv1alpha1.GroupName, - Version: configv1alpha1.SchemeGroupVersion.Version, - Resource: "credentialissuers", - } - - frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - - // Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any - // logic on this thing. - ignorablePod := corev1.Pod{} - ignorablePod.Name = "some-ignorable-pod" - r.NoError(kubeSystemInformerClient.Tracker().Add(&ignorablePod)) - r.NoError(agentInformerClient.Tracker().Add(&ignorablePod)) - r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) - }) - - it.After(func() { - cancelContextCancelFunc() - }) - - when("there is an agent pod without annotations set", func() { - it.Before(func() { - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - when("there is a matching controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the annotations according to the controller manager pod", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = certPath - updatedAgentPod.Annotations[keyPathAnnotation] = keyPath - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - - when("updating the agent pod fails", func() { - it.Before(func() { - kubeAPIClient.PrependReactor( - "update", - "pods", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some update error") - }, - ) - }) - - it("returns the error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "cannot update agent pod: some update error") - }) - - when("there is already a CredentialIssuer", func() { - var initialCredentialIssuer *configv1alpha1.CredentialIssuer - - it.Before(func() { - initialCredentialIssuer = &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{}, - }, - } - r.NoError(pinnipedAPIClient.Tracker().Add(initialCredentialIssuer)) - }) - - it("updates the CredentialIssuer status with the error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - expectedCredentialIssuer := initialCredentialIssuer.DeepCopy() - expectedCredentialIssuer.Status.Strategies = []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: "cannot update agent pod: some update error", - LastUpdateTime: metav1.NewTime(frozenNow), - }, - } - expectedGetAction := coretesting.NewRootGetAction( - credentialIssuerGVR, - credentialIssuerResourceName, - ) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction( - credentialIssuerGVR, - "status", - expectedCredentialIssuer, - ) - - r.EqualError(err, "cannot update agent pod: some update error") - r.Equal( - []coretesting.Action{ - expectedGetAction, - expectedUpdateAction, - }, - pinnipedAPIClient.Actions(), - ) - }) - - when("updating the CredentialIssuer fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "update", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some update error") - }, - ) - }) - - it("returns the original pod update error so the controller gets scheduled again", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "cannot update agent pod: some update error") - }) - }) - }) - - when("there is not already a CredentialIssuer", func() { - it.Before(func() { - credentialIssuerLabels = map[string]string{"foo": "bar"} - }) - - it("creates the CredentialIssuer status with the error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{"foo": "bar"}, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{"foo": "bar"}, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: "cannot update agent pod: some update error", - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction( - credentialIssuerGVR, - credentialIssuerResourceName, - ) - expectedCreateAction := coretesting.NewRootCreateAction( - credentialIssuerGVR, - expectedCreateCredentialIssuer, - ) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction( - credentialIssuerGVR, - "status", - expectedCredentialIssuer, - ) - - r.EqualError(err, "cannot update agent pod: some update error") - r.Equal( - []coretesting.Action{ - expectedGetAction, - expectedCreateAction, - expectedUpdateAction, - }, - pinnipedAPIClient.Actions(), - ) - }) - }) - }) - }) - - when("there is a controller manager pod with CLI flag values separated by spaces", func() { - it.Before(func() { - controllerManagerPod.Spec.Containers[0].Command = []string{ - "kube-controller-manager", - "--cluster-signing-cert-file", certPath, - "--cluster-signing-key-file", keyPath, - } - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the annotations according to the controller manager pod", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = certPath - updatedAgentPod.Annotations[keyPathAnnotation] = keyPath - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is a controller manager pod with no CLI flags", func() { - it.Before(func() { - controllerManagerPod.Spec.Containers[0].Command = []string{ - "kube-controller-manager", - } - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the annotations with the default values", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = defaultKubeControllerManagerClusterSigningCertFileFlagValue - updatedAgentPod.Annotations[keyPathAnnotation] = defaultKubeControllerManagerClusterSigningKeyFileFlagValue - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is a controller manager pod with unparsable CLI flags", func() { - it.Before(func() { - controllerManagerPod.Spec.Containers[0].Command = []string{ - "kube-controller-manager", - "--cluster-signing-cert-file-blah", certPath, - "--cluster-signing-key-file-blah", keyPath, - } - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the annotations with the default values", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = defaultKubeControllerManagerClusterSigningCertFileFlagValue - updatedAgentPod.Annotations[keyPathAnnotation] = defaultKubeControllerManagerClusterSigningKeyFileFlagValue - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is a controller manager pod with unparsable cert CLI flag", func() { - it.Before(func() { - controllerManagerPod.Spec.Containers[0].Command = []string{ - "kube-controller-manager", - "--cluster-signing-cert-file-blah", certPath, - "--cluster-signing-key-file", keyPath, - } - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the key annotation with the default cert flag value", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = defaultKubeControllerManagerClusterSigningCertFileFlagValue - updatedAgentPod.Annotations[keyPathAnnotation] = keyPath - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is a controller manager pod with unparsable key CLI flag", func() { - it.Before(func() { - controllerManagerPod.Spec.Containers[0].Command = []string{ - "kube-controller-manager", - "--cluster-signing-cert-file", certPath, - "--cluster-signing-key-file-blah", keyPath, - } - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the cert annotation with the default key flag value", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = certPath - updatedAgentPod.Annotations[keyPathAnnotation] = defaultKubeControllerManagerClusterSigningKeyFileFlagValue - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is a non-matching controller manager pod via uid", func() { - it.Before(func() { - controllerManagerPod.UID = "some-other-controller-manager-uid" - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("does nothing; the deleter will delete this pod to trigger resync", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal( - []coretesting.Action{}, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is a non-matching controller manager pod via name", func() { - it.Before(func() { - controllerManagerPod.Name = "some-other-controller-manager-name" - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("does nothing; the deleter will delete this pod to trigger resync", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal( - []coretesting.Action{}, - kubeAPIClient.Actions(), - ) - }) - }) - }) - - when("there is an agent pod without annotations set which does not have the configured additional labels", func() { - it.Before(func() { - delete(agentPod.ObjectMeta.Labels, "myLabelKey1") - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - when("there is a matching controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the annotations according to the controller manager pod", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = certPath - updatedAgentPod.Annotations[keyPathAnnotation] = keyPath - - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - }) - - when("there is an agent pod with correct annotations set", func() { - it.Before(func() { - agentPod.Annotations = make(map[string]string) - agentPod.Annotations[certPathAnnotation] = certPath - agentPod.Annotations[keyPathAnnotation] = keyPath - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - when("there is a matching controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("does nothing since the pod is up to date", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal( - []coretesting.Action{}, - kubeAPIClient.Actions(), - ) - }) - }) - }) - - when("there is an agent pod with the wrong cert annotation", func() { - it.Before(func() { - agentPod.Annotations[certPathAnnotation] = "wrong" - agentPod.Annotations[keyPathAnnotation] = keyPath - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - when("there is a matching controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the agent with the correct cert annotation", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[certPathAnnotation] = certPath - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - }) - - when("there is an agent pod with the wrong key annotation", func() { - it.Before(func() { - agentPod.Annotations[certPathAnnotation] = certPath - agentPod.Annotations[keyPathAnnotation] = "key" - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - when("there is a matching controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("updates the agent with the correct key annotation", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Annotations[keyPathAnnotation] = keyPath - r.Equal( - []coretesting.Action{ - coretesting.NewGetAction( - podsGVR, - agentPodNamespace, - updatedAgentPod.Name, - ), - coretesting.NewUpdateAction( - podsGVR, - agentPodNamespace, - updatedAgentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - }) - }, spec.Parallel(), spec.Report(report.Terminal{})) -} diff --git a/internal/controller/kubecertagent/creater.go b/internal/controller/kubecertagent/creater.go deleted file mode 100644 index 6cb37934..00000000 --- a/internal/controller/kubecertagent/creater.go +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "fmt" - - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/util/clock" - corev1informers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - - pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" - "go.pinniped.dev/internal/constable" - pinnipedcontroller "go.pinniped.dev/internal/controller" - "go.pinniped.dev/internal/controller/issuerconfig" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/plog" -) - -type createrController struct { - agentPodConfig *AgentPodConfig - credentialIssuerLocationConfig *CredentialIssuerLocationConfig - credentialIssuerLabels map[string]string - clock clock.Clock - k8sClient kubernetes.Interface - pinnipedAPIClient pinnipedclientset.Interface - kubeSystemPodInformer corev1informers.PodInformer - agentPodInformer corev1informers.PodInformer -} - -// NewCreaterController returns a controller that creates new kube-cert-agent pods for every known -// kube-controller-manager pod. -// -// It also is tasked with updating the CredentialIssuer, located via the provided -// credentialIssuerLocationConfig, with any errors that it encounters. -func NewCreaterController( - agentPodConfig *AgentPodConfig, - credentialIssuerLocationConfig *CredentialIssuerLocationConfig, - credentialIssuerLabels map[string]string, - clock clock.Clock, - k8sClient kubernetes.Interface, - pinnipedAPIClient pinnipedclientset.Interface, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - withInformer pinnipedcontroller.WithInformerOptionFunc, - withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc, -) controllerlib.Controller { - return controllerlib.New( - controllerlib.Config{ - //nolint: misspell - Name: "kube-cert-agent-creater-controller", - Syncer: &createrController{ - agentPodConfig: agentPodConfig, - credentialIssuerLocationConfig: credentialIssuerLocationConfig, - credentialIssuerLabels: credentialIssuerLabels, - clock: clock, - k8sClient: k8sClient, - pinnipedAPIClient: pinnipedAPIClient, - kubeSystemPodInformer: kubeSystemPodInformer, - agentPodInformer: agentPodInformer, - }, - }, - withInformer( - kubeSystemPodInformer, - pinnipedcontroller.SimpleFilterWithSingletonQueue(isControllerManagerPod), - controllerlib.InformerOption{}, - ), - withInformer( - agentPodInformer, - pinnipedcontroller.SimpleFilterWithSingletonQueue(isAgentPod), - controllerlib.InformerOption{}, - ), - // Be sure to run once even to make sure the CI is updated if there are no controller manager - // pods. We should be able to pass an empty key since we don't use the key in the sync (we sync - // the world). - withInitialEvent(controllerlib.Key{}), - ) -} - -// Sync implements controllerlib.Syncer. -func (c *createrController) Sync(ctx controllerlib.Context) error { - controllerManagerSelector, err := labels.Parse("component=kube-controller-manager") - if err != nil { - return fmt.Errorf("cannot create controller manager selector: %w", err) - } - - controllerManagerPods, err := c.kubeSystemPodInformer.Lister().List(controllerManagerSelector) - if err != nil { - return fmt.Errorf("informer cannot list controller manager pods: %w", err) - } - - if len(controllerManagerPods) == 0 { - // If there are no controller manager pods, we alert the user that we can't find the keypair via - // the CredentialIssuer. - return issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - strategyError(c.clock, constable.Error("did not find kube-controller-manager pod(s)")), - ) - } - - for _, controllerManagerPod := range controllerManagerPods { - agentPod, err := findAgentPodForSpecificControllerManagerPod( - controllerManagerPod, - c.kubeSystemPodInformer, - c.agentPodInformer, - c.agentPodConfig.AgentSelector(), - ) - if err != nil { - return err - } - if agentPod == nil { - agentPod = c.agentPodConfig.newAgentPod(controllerManagerPod) - - plog.Debug( - "creating agent pod", - "pod", - klog.KObj(agentPod), - "controller", - klog.KObj(controllerManagerPod), - ) - _, err := c.k8sClient.CoreV1(). - Pods(c.agentPodConfig.Namespace). - Create(ctx.Context, agentPod, metav1.CreateOptions{}) - if err != nil { - err = fmt.Errorf("cannot create agent pod: %w", err) - strategyResultUpdateErr := issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - strategyError(c.clock, err), - ) - if strategyResultUpdateErr != nil { - // If the CI update fails, then we probably want to try again. This controller will get - // called again because of the pod create failure, so just try the CI update again then. - klog.ErrorS(strategyResultUpdateErr, "could not create or update CredentialIssuer") - } - - return err - } - } - - // The deleter controller handles the case where the expected fields do not match in the agent pod. - } - - return nil -} - -func findAgentPodForSpecificControllerManagerPod( - controllerManagerPod *corev1.Pod, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - agentSelector labels.Selector, -) (*corev1.Pod, error) { - agentPods, err := agentPodInformer. - Lister(). - List(agentSelector) - if err != nil { - return nil, fmt.Errorf("informer cannot list agent pods: %w", err) - } - - for _, maybeAgentPod := range agentPods { - maybeControllerManagerPod, err := findControllerManagerPodForSpecificAgentPod( - maybeAgentPod, - kubeSystemPodInformer, - ) - if err != nil { - return nil, err - } - if maybeControllerManagerPod != nil && - maybeControllerManagerPod.UID == controllerManagerPod.UID { - return maybeAgentPod, nil - } - } - - return nil, nil -} diff --git a/internal/controller/kubecertagent/creater_test.go b/internal/controller/kubecertagent/creater_test.go deleted file mode 100644 index 13dd9943..00000000 --- a/internal/controller/kubecertagent/creater_test.go +++ /dev/null @@ -1,623 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "context" - "errors" - "testing" - "time" - - "github.com/sclevine/spec" - "github.com/sclevine/spec/report" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/clock" - kubeinformers "k8s.io/client-go/informers" - corev1informers "k8s.io/client-go/informers/core/v1" - kubernetesfake "k8s.io/client-go/kubernetes/fake" - coretesting "k8s.io/client-go/testing" - - configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" - pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/testutil" -) - -func TestCreaterControllerFilter(t *testing.T) { - defineSharedKubecertagentFilterSpecs( - t, - "CreaterControllerFilter", - func( - agentPodConfig *AgentPodConfig, - credentialIssuerLocationConfig *CredentialIssuerLocationConfig, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - observableWithInformerOption *testutil.ObservableWithInformerOption, - ) { - _ = NewCreaterController( - agentPodConfig, - credentialIssuerLocationConfig, - map[string]string{}, - nil, // clock, shouldn't matter - nil, // k8sClient, shouldn't matter - nil, // pinnipedAPIClient, shouldn't matter - kubeSystemPodInformer, - agentPodInformer, - observableWithInformerOption.WithInformer, - controllerlib.WithInitialEvent, - ) - }, - ) -} - -func TestCreaterControllerInitialEvent(t *testing.T) { - kubeSystemInformerClient := kubernetesfake.NewSimpleClientset() - kubeSystemInformers := kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0) - - agentInformerClient := kubernetesfake.NewSimpleClientset() - agentInformers := kubeinformers.NewSharedInformerFactory(agentInformerClient, 0) - - observableWithInitialEventOption := testutil.NewObservableWithInitialEventOption() - - _ = NewCreaterController( - nil, // agentPodConfig, shouldn't matter - nil, // credentialIssuerLocationConfig, shouldn't matter - map[string]string{}, - nil, // clock, shouldn't matter - nil, // k8sClient, shouldn't matter - nil, // pinnipedAPIClient, shouldn't matter - kubeSystemInformers.Core().V1().Pods(), - agentInformers.Core().V1().Pods(), - controllerlib.WithInformer, - observableWithInitialEventOption.WithInitialEvent, - ) - require.Equal(t, &controllerlib.Key{}, observableWithInitialEventOption.GetInitialEventKey()) -} - -func TestCreaterControllerSync(t *testing.T) { - spec.Run(t, "CreaterControllerSync", func(t *testing.T, when spec.G, it spec.S) { - const kubeSystemNamespace = "kube-system" - const agentPodNamespace = "agent-pod-namespace" - const credentialIssuerResourceName = "ci-resource-name" - - var r *require.Assertions - - var subject controllerlib.Controller - var kubeAPIClient *kubernetesfake.Clientset - var kubeSystemInformerClient *kubernetesfake.Clientset - var kubeSystemInformers kubeinformers.SharedInformerFactory - var agentInformerClient *kubernetesfake.Clientset - var agentInformers kubeinformers.SharedInformerFactory - var pinnipedAPIClient *pinnipedfake.Clientset - var cancelContext context.Context - var cancelContextCancelFunc context.CancelFunc - var syncContext *controllerlib.Context - var controllerManagerPod, agentPod *corev1.Pod - var podsGVR schema.GroupVersionResource - var credentialIssuerGVR schema.GroupVersionResource - var frozenNow time.Time - - // Defer starting the informers until the last possible moment so that the - // nested Before's can keep adding things to the informer caches. - var startInformersAndController = func() { - // Set this at the last second to allow for injection of server override. - subject = NewCreaterController( - &AgentPodConfig{ - Namespace: agentPodNamespace, - ContainerImage: "some-agent-image", - PodNamePrefix: "some-agent-name-", - ContainerImagePullSecrets: []string{"some-image-pull-secret"}, - AdditionalLabels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - &CredentialIssuerLocationConfig{ - Name: credentialIssuerResourceName, - }, - map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - clock.NewFakeClock(frozenNow), - kubeAPIClient, - pinnipedAPIClient, - kubeSystemInformers.Core().V1().Pods(), - agentInformers.Core().V1().Pods(), - controllerlib.WithInformer, - controllerlib.WithInitialEvent, - ) - - // Set this at the last second to support calling subject.Name(). - syncContext = &controllerlib.Context{ - Context: cancelContext, - Name: subject.Name(), - Key: controllerlib.Key{ - Namespace: kubeSystemNamespace, - Name: "should-not-matter", - }, - } - - // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(cancelContext.Done()) - agentInformers.Start(cancelContext.Done()) - controllerlib.TestRunSynchronously(t, subject) - } - - it.Before(func() { - r = require.New(t) - - kubeAPIClient = kubernetesfake.NewSimpleClientset() - - kubeSystemInformerClient = kubernetesfake.NewSimpleClientset() - kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0) - - agentInformerClient = kubernetesfake.NewSimpleClientset() - agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0) - - pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - - cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) - - controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( - kubeSystemNamespace, agentPodNamespace, "ignored for this test", "ignored for this test", - ) - - podsGVR = schema.GroupVersionResource{ - Group: corev1.SchemeGroupVersion.Group, - Version: corev1.SchemeGroupVersion.Version, - Resource: "pods", - } - - credentialIssuerGVR = schema.GroupVersionResource{ - Group: configv1alpha1.GroupName, - Version: configv1alpha1.SchemeGroupVersion.Version, - Resource: "credentialissuers", - } - - frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - - // Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any - // logic on this thing. - ignorablePod := corev1.Pod{} - ignorablePod.Name = "some-ignorable-pod" - r.NoError(kubeSystemInformerClient.Tracker().Add(&ignorablePod)) - r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) - - // Add another valid agent pod to make sure our logic works for just the pod we care about. - otherAgentPod := agentPod.DeepCopy() - otherAgentPod.Name = "some-other-agent" - otherAgentPod.Annotations = map[string]string{ - "kube-cert-agent.pinniped.dev/controller-manager-name": "some-other-controller-manager-name", - "kube-cert-agent.pinniped.dev/controller-manager-uid": "some-other-controller-manager-uid", - } - r.NoError(agentInformerClient.Tracker().Add(otherAgentPod)) - r.NoError(kubeAPIClient.Tracker().Add(otherAgentPod)) - }) - - it.After(func() { - cancelContextCancelFunc() - }) - - when("there is a controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - when("there is a matching agent pod", func() { - it.Before(func() { - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - it("does nothing", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Empty(kubeAPIClient.Actions()) - }) - }) - - when("there is a matching agent pod that is missing some of the configured additional labels", func() { - it.Before(func() { - nonMatchingAgentPod := agentPod.DeepCopy() - delete(nonMatchingAgentPod.ObjectMeta.Labels, "myLabelKey1") - r.NoError(agentInformerClient.Tracker().Add(nonMatchingAgentPod)) - r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod)) - }) - - it("does nothing because the deleter controller is responsible for deleting it", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Empty(kubeAPIClient.Actions()) - }) - }) - - when("there is a non-matching agent pod", func() { - it.Before(func() { - nonMatchingAgentPod := agentPod.DeepCopy() - nonMatchingAgentPod.Name = "some-agent-name-85da432e" - nonMatchingAgentPod.Annotations[controllerManagerUIDAnnotationKey] = "some-non-matching-uid" - r.NoError(agentInformerClient.Tracker().Add(nonMatchingAgentPod)) - r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod)) - }) - - it("creates a matching agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Equal( - []coretesting.Action{ - coretesting.NewCreateAction( - podsGVR, - agentPodNamespace, - agentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - }) - - when("there is no matching agent pod", func() { - it("creates a matching agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Equal( - []coretesting.Action{ - coretesting.NewCreateAction( - podsGVR, - agentPodNamespace, - agentPod, - ), - }, - kubeAPIClient.Actions(), - ) - }) - - when("creating the matching agent pod fails", func() { - it.Before(func() { - kubeAPIClient.PrependReactor( - "create", - "pods", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some create error") - }, - ) - }) - - when("there is already a CredentialIssuer", func() { - var initialCredentialIssuer *configv1alpha1.CredentialIssuer - - it.Before(func() { - initialCredentialIssuer = &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{}, - }, - } - r.NoError(pinnipedAPIClient.Tracker().Add(initialCredentialIssuer)) - }) - - it("updates the CredentialIssuer status saying that controller manager pods couldn't be found", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - expectedCredentialIssuer := initialCredentialIssuer.DeepCopy() - expectedCredentialIssuer.Status.Strategies = []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: "cannot create agent pod: some create error", - LastUpdateTime: metav1.NewTime(frozenNow), - }, - } - expectedGetAction := coretesting.NewRootGetAction( - credentialIssuerGVR, - credentialIssuerResourceName, - ) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction( - credentialIssuerGVR, - "status", - expectedCredentialIssuer, - ) - - r.EqualError(err, "cannot create agent pod: some create error") - r.Equal( - []coretesting.Action{ - expectedGetAction, - expectedUpdateAction, - }, - pinnipedAPIClient.Actions(), - ) - }) - - when("the CredentialIssuer operation fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "update", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some update error") - }, - ) - - it("still returns the pod create error, since the controller will get rescheduled", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "cannot create agent pod: some create error") - }) - }) - }) - }) - - when("there is not already a CredentialIssuer", func() { - it("returns an error and updates the CredentialIssuer status", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: "cannot create agent pod: some create error", - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction( - credentialIssuerGVR, - credentialIssuerResourceName, - ) - expectedCreateAction := coretesting.NewRootCreateAction( - credentialIssuerGVR, - expectedCreateCredentialIssuer, - ) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction( - credentialIssuerGVR, - "status", - expectedCredentialIssuer, - ) - - r.EqualError(err, "cannot create agent pod: some create error") - r.Equal( - []coretesting.Action{ - expectedGetAction, - expectedCreateAction, - expectedUpdateAction, - }, - pinnipedAPIClient.Actions(), - ) - }) - }) - }) - }) - }) - - when("there is no controller manager pod", func() { - when("there is already a CredentialIssuer", func() { - var initialCredentialIssuer *configv1alpha1.CredentialIssuer - - it.Before(func() { - initialCredentialIssuer = &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{}, - }, - } - r.NoError(pinnipedAPIClient.Tracker().Add(initialCredentialIssuer)) - }) - - it("updates the CredentialIssuer status saying that controller manager pods couldn't be found", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - expectedCredentialIssuer := initialCredentialIssuer.DeepCopy() - expectedCredentialIssuer.Status.Strategies = []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: "did not find kube-controller-manager pod(s)", - LastUpdateTime: metav1.NewTime(frozenNow), - }, - } - expectedGetAction := coretesting.NewRootGetAction( - credentialIssuerGVR, - credentialIssuerResourceName, - ) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction( - credentialIssuerGVR, - "status", - expectedCredentialIssuer, - ) - - r.Equal( - []coretesting.Action{ - expectedGetAction, - expectedUpdateAction, - }, - pinnipedAPIClient.Actions(), - ) - }) - - when("when updating the CredentialIssuer fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "update", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some update error") - }, - ) - }) - - it("returns an error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "could not create or update credentialissuer: some update error") - }) - }) - - when("when getting the CredentialIssuer fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "get", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some get error") - }, - ) - }) - - it("returns an error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "could not create or update credentialissuer: get failed: some get error") - }) - }) - }) - - when("there is not already a CredentialIssuer", func() { - it("creates the CredentialIssuer status saying that controller manager pods couldn't be found", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: "did not find kube-controller-manager pod(s)", - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction( - credentialIssuerGVR, - credentialIssuerResourceName, - ) - expectedCreateAction := coretesting.NewRootCreateAction( - credentialIssuerGVR, - expectedCreateCredentialIssuer, - ) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction( - credentialIssuerGVR, - "status", - expectedCredentialIssuer, - ) - - r.NoError(err) - r.Equal( - []coretesting.Action{ - expectedGetAction, - expectedCreateAction, - expectedUpdateAction, - }, - pinnipedAPIClient.Actions(), - ) - }) - - when("when creating the CredentialIssuer fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "create", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some create error") - }, - ) - }) - - it("returns an error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "could not create or update credentialissuer: create failed: some create error") - }) - }) - - when("when getting the CredentialIssuer fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "get", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some get error") - }, - ) - }) - - it("returns an error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "could not create or update credentialissuer: get failed: some get error") - }) - }) - }) - }) - }, spec.Parallel(), spec.Report(report.Terminal{})) -} diff --git a/internal/controller/kubecertagent/deleter.go b/internal/controller/kubecertagent/deleter.go deleted file mode 100644 index dfb66aed..00000000 --- a/internal/controller/kubecertagent/deleter.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "fmt" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corev1informers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/klog/v2" - - pinnipedcontroller "go.pinniped.dev/internal/controller" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/plog" -) - -type deleterController struct { - agentPodConfig *AgentPodConfig - k8sClient kubernetes.Interface - kubeSystemPodInformer corev1informers.PodInformer - agentPodInformer corev1informers.PodInformer -} - -// NewDeleterController returns a controller that deletes any kube-cert-agent pods that are out of -// sync with the known kube-controller-manager pods. -func NewDeleterController( - agentPodConfig *AgentPodConfig, - k8sClient kubernetes.Interface, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - withInformer pinnipedcontroller.WithInformerOptionFunc, -) controllerlib.Controller { - return controllerlib.New( - controllerlib.Config{ - Name: "kube-cert-agent-deleter-controller", - Syncer: &deleterController{ - agentPodConfig: agentPodConfig, - k8sClient: k8sClient, - kubeSystemPodInformer: kubeSystemPodInformer, - agentPodInformer: agentPodInformer, - }, - }, - withInformer( - kubeSystemPodInformer, - pinnipedcontroller.SimpleFilterWithSingletonQueue(isControllerManagerPod), - controllerlib.InformerOption{}, - ), - withInformer( - agentPodInformer, - pinnipedcontroller.SimpleFilterWithSingletonQueue(isAgentPod), - controllerlib.InformerOption{}, - ), - ) -} - -// Sync implements controllerlib.Syncer. -func (c *deleterController) Sync(ctx controllerlib.Context) error { - agentPods, err := c.agentPodInformer. - Lister(). - Pods(c.agentPodConfig.Namespace). - List(c.agentPodConfig.AgentSelector()) - if err != nil { - return fmt.Errorf("informer cannot list agent pods: %w", err) - } - - for _, agentPod := range agentPods { - controllerManagerPod, err := findControllerManagerPodForSpecificAgentPod(agentPod, c.kubeSystemPodInformer) - if err != nil { - return err - } - if controllerManagerPod == nil || - !isAgentPodUpToDate(agentPod, c.agentPodConfig.newAgentPod(controllerManagerPod)) { - plog.Debug("deleting agent pod", "pod", klog.KObj(agentPod)) - err := c.k8sClient. - CoreV1(). - Pods(agentPod.Namespace). - Delete(ctx.Context, agentPod.Name, metav1.DeleteOptions{}) - if err != nil { - return fmt.Errorf("cannot delete agent pod: %w", err) - } - } - } - - return nil -} diff --git a/internal/controller/kubecertagent/deleter_test.go b/internal/controller/kubecertagent/deleter_test.go deleted file mode 100644 index ba5240af..00000000 --- a/internal/controller/kubecertagent/deleter_test.go +++ /dev/null @@ -1,506 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "context" - "testing" - - "github.com/sclevine/spec" - "github.com/sclevine/spec/report" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - kubeinformers "k8s.io/client-go/informers" - corev1informers "k8s.io/client-go/informers/core/v1" - kubernetesfake "k8s.io/client-go/kubernetes/fake" - coretesting "k8s.io/client-go/testing" - - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/testutil" -) - -func TestDeleterControllerFilter(t *testing.T) { - defineSharedKubecertagentFilterSpecs( - t, - "DeleterControllerFilter", - func( - agentPodConfig *AgentPodConfig, - _ *CredentialIssuerLocationConfig, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - observableWithInformerOption *testutil.ObservableWithInformerOption, - ) { - _ = NewDeleterController( - agentPodConfig, - nil, // k8sClient, shouldn't matter - kubeSystemPodInformer, - agentPodInformer, - observableWithInformerOption.WithInformer, - ) - }, - ) -} - -func TestDeleterControllerSync(t *testing.T) { - spec.Run(t, "DeleterControllerSync", func(t *testing.T, when spec.G, it spec.S) { - const kubeSystemNamespace = "kube-system" - const agentPodNamespace = "agent-pod-namespace" - - var r *require.Assertions - - var subject controllerlib.Controller - var kubeAPIClient *kubernetesfake.Clientset - var kubeSystemInformerClient *kubernetesfake.Clientset - var kubeSystemInformers kubeinformers.SharedInformerFactory - var agentInformerClient *kubernetesfake.Clientset - var agentInformers kubeinformers.SharedInformerFactory - var cancelContext context.Context - var cancelContextCancelFunc context.CancelFunc - var syncContext *controllerlib.Context - var controllerManagerPod, agentPod *corev1.Pod - var podsGVR schema.GroupVersionResource - - // Defer starting the informers until the last possible moment so that the - // nested Before's can keep adding things to the informer caches. - var startInformersAndController = func() { - // Set this at the last second to allow for injection of server override. - subject = NewDeleterController( - &AgentPodConfig{ - Namespace: agentPodNamespace, - ContainerImage: "some-agent-image", - PodNamePrefix: "some-agent-name-", - AdditionalLabels: map[string]string{ - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - }, - kubeAPIClient, - kubeSystemInformers.Core().V1().Pods(), - agentInformers.Core().V1().Pods(), - controllerlib.WithInformer, - ) - - // Set this at the last second to support calling subject.Name(). - syncContext = &controllerlib.Context{ - Context: cancelContext, - Name: subject.Name(), - Key: controllerlib.Key{ - Namespace: kubeSystemNamespace, - Name: "should-not-matter", - }, - } - - // Must start informers before calling TestRunSynchronously() - kubeSystemInformers.Start(cancelContext.Done()) - agentInformers.Start(cancelContext.Done()) - controllerlib.TestRunSynchronously(t, subject) - } - - var requireAgentPodWasDeleted = func() { - r.Equal( - []coretesting.Action{coretesting.NewDeleteAction(podsGVR, agentPodNamespace, agentPod.Name)}, - kubeAPIClient.Actions(), - ) - } - - it.Before(func() { - r = require.New(t) - - cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) - - kubeAPIClient = kubernetesfake.NewSimpleClientset() - - kubeSystemInformerClient = kubernetesfake.NewSimpleClientset() - kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0) - - agentInformerClient = kubernetesfake.NewSimpleClientset() - agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0) - - controllerManagerPod, agentPod = exampleControllerManagerAndAgentPods( - kubeSystemNamespace, agentPodNamespace, "ignored for this test", "ignored for this test", - ) - - podsGVR = schema.GroupVersionResource{ - Group: corev1.SchemeGroupVersion.Group, - Version: corev1.SchemeGroupVersion.Version, - Resource: "pods", - } - - // Add an pod into the test that doesn't matter to make sure we don't accidentally - // trigger any logic on this thing. - ignorablePod := corev1.Pod{} - ignorablePod.Name = "some-ignorable-pod" - r.NoError(kubeSystemInformerClient.Tracker().Add(&ignorablePod)) - r.NoError(agentInformerClient.Tracker().Add(&ignorablePod)) - r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) - }) - - it.After(func() { - cancelContextCancelFunc() - }) - - when("there is an agent pod", func() { - it.Before(func() { - r.NoError(agentInformerClient.Tracker().Add(agentPod)) - r.NoError(kubeAPIClient.Tracker().Add(agentPod)) - }) - - when("there is a matching controller manager pod", func() { - it.Before(func() { - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("does nothing", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Empty(kubeAPIClient.Actions()) - }) - - when("the agent pod is out of sync with the controller manager via volume mounts", func() { - it.Before(func() { - controllerManagerPod.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "some-other-volume-mount"}} - r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the controller manager via volumes", func() { - it.Before(func() { - controllerManagerPod.Spec.Volumes = []corev1.Volume{{Name: "some-other-volume"}} - r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the controller manager via node selector", func() { - it.Before(func() { - controllerManagerPod.Spec.NodeSelector = map[string]string{ - "some-other-node-selector-key": "some-other-node-selector-value", - } - r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the controller manager via node name", func() { - it.Before(func() { - controllerManagerPod.Spec.NodeName = "some-other-node-name" - r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the controller manager via tolerations", func() { - it.Before(func() { - controllerManagerPod.Spec.Tolerations = []corev1.Toleration{{Key: "some-other-toleration-key"}} - r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, controllerManagerPod, controllerManagerPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync via restart policy", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Spec.RestartPolicy = corev1.RestartPolicyAlways - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync via automount service account token", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - t := true - updatedAgentPod.Spec.AutomountServiceAccountToken = &t - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the template via name", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Spec.Containers[0].Name = "some-new-name" - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the template via image", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Spec.Containers[0].Image = "new-image" - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the template via runAsUser", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - notRoot := int64(1234) - updatedAgentPod.Spec.SecurityContext.RunAsUser = ¬Root - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the template via runAsGroup", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - notRoot := int64(1234) - updatedAgentPod.Spec.SecurityContext.RunAsGroup = ¬Root - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the template via having a nil SecurityContext", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Spec.SecurityContext = nil - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod is out of sync with the template via labels", func() { - when("an additional label's value was changed", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.ObjectMeta.Labels = map[string]string{ - "kube-cert-agent.pinniped.dev": "true", - // the value of a label is wrong so the pod should be deleted so it can get recreated with the new labels - "myLabelKey1": "myLabelValue1-outdated-value", - "myLabelKey2": "myLabelValue2-outdated-value", - } - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("an additional custom label was added since the agent pod was created", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.ObjectMeta.Labels = map[string]string{ - "kube-cert-agent.pinniped.dev": "true", - "myLabelKey1": "myLabelValue1", - // "myLabelKey2" is missing so the pod should be deleted so it can get recreated with the new labels - } - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("the agent pod has extra labels that seem unrelated to the additional labels", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.ObjectMeta.Labels = map[string]string{ - "kube-cert-agent.pinniped.dev": "true", - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - "extra-label": "not-related-to-the-sepcified-additional-labels", - } - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("does not delete the agent pod because someone else might have put those labels on it", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Empty(kubeAPIClient.Actions()) - }) - }) - }) - - when("the agent pod is out of sync with the template via command", func() { - it.Before(func() { - updatedAgentPod := agentPod.DeepCopy() - updatedAgentPod.Spec.Containers[0].Command = []string{"some", "new", "command"} - r.NoError(agentInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - }) - - when("there is a non-matching controller manager pod via uid", func() { - it.Before(func() { - controllerManagerPod.UID = "some-other-controller-manager-uid" - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("there is a non-matching controller manager pod via name", func() { - it.Before(func() { - controllerManagerPod.Name = "some-other-controller-manager-name" - r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) - r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) - }) - - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - - when("there is no matching controller manager pod", func() { - it("deletes the agent pod", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - requireAgentPodWasDeleted() - }) - }) - }) - - when("there is no agent pod", func() { - it("does nothing", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - - r.NoError(err) - r.Empty(kubeAPIClient.Actions()) - }) - }) - }, spec.Parallel(), spec.Report(report.Terminal{})) -} diff --git a/internal/controller/kubecertagent/execer.go b/internal/controller/kubecertagent/execer.go deleted file mode 100644 index e021ff26..00000000 --- a/internal/controller/kubecertagent/execer.go +++ /dev/null @@ -1,232 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "encoding/base64" - "fmt" - - v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/clock" - "k8s.io/apimachinery/pkg/util/errors" - corev1informers "k8s.io/client-go/informers/core/v1" - "k8s.io/client-go/tools/clientcmd" - - configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" - pinnipedclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" - pinnipedcontroller "go.pinniped.dev/internal/controller" - "go.pinniped.dev/internal/controller/issuerconfig" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/dynamiccert" -) - -const ( - ClusterInfoNamespace = "kube-public" - clusterInfoName = "cluster-info" - clusterInfoConfigMapKey = "kubeconfig" -) - -type execerController struct { - credentialIssuerLocationConfig *CredentialIssuerLocationConfig - credentialIssuerLabels map[string]string - discoveryURLOverride *string - dynamicCertProvider dynamiccert.Private - podCommandExecutor PodCommandExecutor - clock clock.Clock - pinnipedAPIClient pinnipedclientset.Interface - agentPodInformer corev1informers.PodInformer - configMapInformer corev1informers.ConfigMapInformer -} - -// NewExecerController returns a controllerlib.Controller that listens for agent pods with proper -// cert/key path annotations and execs into them to get the cert/key material. It sets the retrieved -// key material in a provided dynamicCertProvider. -// -// It also is tasked with updating the CredentialIssuer, located via the provided -// credentialIssuerLocationConfig, with any errors that it encounters. -func NewExecerController( - credentialIssuerLocationConfig *CredentialIssuerLocationConfig, - credentialIssuerLabels map[string]string, - discoveryURLOverride *string, - dynamicCertProvider dynamiccert.Private, - podCommandExecutor PodCommandExecutor, - pinnipedAPIClient pinnipedclientset.Interface, - clock clock.Clock, - agentPodInformer corev1informers.PodInformer, - configMapInformer corev1informers.ConfigMapInformer, - withInformer pinnipedcontroller.WithInformerOptionFunc, -) controllerlib.Controller { - return controllerlib.New( - controllerlib.Config{ - Name: "kube-cert-agent-execer-controller", - Syncer: &execerController{ - credentialIssuerLocationConfig: credentialIssuerLocationConfig, - credentialIssuerLabels: credentialIssuerLabels, - discoveryURLOverride: discoveryURLOverride, - dynamicCertProvider: dynamicCertProvider, - podCommandExecutor: podCommandExecutor, - pinnipedAPIClient: pinnipedAPIClient, - clock: clock, - agentPodInformer: agentPodInformer, - configMapInformer: configMapInformer, - }, - }, - withInformer( - agentPodInformer, - pinnipedcontroller.SimpleFilter(isAgentPod, nil), // nil parent func is fine because each event is distinct - controllerlib.InformerOption{}, - ), - withInformer( - configMapInformer, - pinnipedcontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace), - controllerlib.InformerOption{}, - ), - ) -} - -func (c *execerController) Sync(ctx controllerlib.Context) error { - maybeAgentPod, err := c.agentPodInformer.Lister().Pods(ctx.Key.Namespace).Get(ctx.Key.Name) - notFound := k8serrors.IsNotFound(err) - if err != nil && !notFound { - return fmt.Errorf("failed to get %s/%s pod: %w", ctx.Key.Namespace, ctx.Key.Name, err) - } - if notFound { - // The pod in question does not exist, so it was probably deleted - return nil - } - - certPath, keyPath := c.getKeypairFilePaths(maybeAgentPod) - if certPath == "" || keyPath == "" { - // The annotator controller has not annotated this agent pod yet, or it is not an agent pod at all - return nil - } - agentPod := maybeAgentPod - - if agentPod.Status.Phase != v1.PodRunning { - // Seems to be an agent pod, but it is not ready yet - return nil - } - - certPEM, err := c.podCommandExecutor.Exec(agentPod.Namespace, agentPod.Name, "cat", certPath) - if err != nil { - strategyResultUpdateErr := issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - strategyError(c.clock, err), - ) - return newAggregate(err, strategyResultUpdateErr) - } - - keyPEM, err := c.podCommandExecutor.Exec(agentPod.Namespace, agentPod.Name, "cat", keyPath) - if err != nil { - strategyResultUpdateErr := issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - strategyError(c.clock, err), - ) - return newAggregate(err, strategyResultUpdateErr) - } - - if err := c.dynamicCertProvider.SetCertKeyContent([]byte(certPEM), []byte(keyPEM)); err != nil { - err = fmt.Errorf("failed to set signing cert/key content from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err) - strategyResultUpdateErr := issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - strategyError(c.clock, err), - ) - return newAggregate(err, strategyResultUpdateErr) - } - - apiInfo, err := c.getTokenCredentialRequestAPIInfo() - if err != nil { - strategyResultUpdateErr := issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - configv1alpha1.CredentialIssuerStrategy{ - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotGetClusterInfoStrategyReason, - Message: err.Error(), - LastUpdateTime: metav1.NewTime(c.clock.Now()), - }, - ) - return newAggregate(err, strategyResultUpdateErr) - } - - return issuerconfig.UpdateStrategy( - ctx.Context, - c.credentialIssuerLocationConfig.Name, - c.credentialIssuerLabels, - c.pinnipedAPIClient, - configv1alpha1.CredentialIssuerStrategy{ - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.SuccessStrategyStatus, - Reason: configv1alpha1.FetchedKeyStrategyReason, - Message: "Key was fetched successfully", - LastUpdateTime: metav1.NewTime(c.clock.Now()), - Frontend: &configv1alpha1.CredentialIssuerFrontend{ - Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, - TokenCredentialRequestAPIInfo: apiInfo, - }, - }, - ) -} - -func (c *execerController) getTokenCredentialRequestAPIInfo() (*configv1alpha1.TokenCredentialRequestAPIInfo, error) { - configMap, err := c.configMapInformer. - Lister(). - ConfigMaps(ClusterInfoNamespace). - Get(clusterInfoName) - if err != nil { - return nil, fmt.Errorf("failed to get %s configmap: %w", clusterInfoName, err) - } - - kubeConfigYAML, kubeConfigPresent := configMap.Data[clusterInfoConfigMapKey] - if !kubeConfigPresent { - return nil, fmt.Errorf("failed to get %s key from %s configmap", clusterInfoConfigMapKey, clusterInfoName) - } - - kubeconfig, err := clientcmd.Load([]byte(kubeConfigYAML)) - if err != nil { - return nil, fmt.Errorf("failed to load data from %s key in %s configmap", clusterInfoConfigMapKey, clusterInfoName) - } - - for _, v := range kubeconfig.Clusters { - result := &configv1alpha1.TokenCredentialRequestAPIInfo{ - Server: v.Server, - CertificateAuthorityData: base64.StdEncoding.EncodeToString(v.CertificateAuthorityData), - } - if c.discoveryURLOverride != nil { - result.Server = *c.discoveryURLOverride - } - return result, nil - } - return nil, fmt.Errorf("kubeconfig in %s key in %s configmap did not contain any clusters", clusterInfoConfigMapKey, clusterInfoName) -} - -func (c *execerController) getKeypairFilePaths(pod *v1.Pod) (string, string) { - annotations := pod.Annotations - if annotations == nil { - annotations = make(map[string]string) - } - - certPath := annotations[agentPodCertPathAnnotationKey] - keyPath := annotations[agentPodKeyPathAnnotationKey] - - return certPath, keyPath -} - -func newAggregate(errs ...error) error { - return errors.NewAggregate(errs) -} diff --git a/internal/controller/kubecertagent/execer_test.go b/internal/controller/kubecertagent/execer_test.go deleted file mode 100644 index c9441010..00000000 --- a/internal/controller/kubecertagent/execer_test.go +++ /dev/null @@ -1,733 +0,0 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. -// SPDX-License-Identifier: Apache-2.0 - -package kubecertagent - -import ( - "context" - "errors" - "fmt" - "io/ioutil" - "testing" - "time" - - "github.com/sclevine/spec" - "github.com/sclevine/spec/report" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/clock" - kubeinformers "k8s.io/client-go/informers" - kubernetesfake "k8s.io/client-go/kubernetes/fake" - coretesting "k8s.io/client-go/testing" - - configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" - pinnipedfake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" - "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/dynamiccert" - "go.pinniped.dev/internal/here" - "go.pinniped.dev/internal/testutil" -) - -func TestExecerControllerOptions(t *testing.T) { - spec.Run(t, "options", func(t *testing.T, when spec.G, it spec.S) { - var r *require.Assertions - var observableWithInformerOption *testutil.ObservableWithInformerOption - var agentPodInformerFilter controllerlib.Filter - - whateverPod := &corev1.Pod{} - - it.Before(func() { - r = require.New(t) - observableWithInformerOption = testutil.NewObservableWithInformerOption() - informerFactory := kubeinformers.NewSharedInformerFactory(nil, 0) - agentPodsInformer := informerFactory.Core().V1().Pods() - configMapsInformer := informerFactory.Core().V1().ConfigMaps() - _ = NewExecerController( - &CredentialIssuerLocationConfig{ - Name: "ignored by this test", - }, - nil, // credentialIssuerLabels, not needed for this test - nil, // discoveryURLOverride, not needed for this test - nil, // dynamicCertProvider, not needed for this test - nil, // podCommandExecutor, not needed for this test - nil, // pinnipedAPIClient, not needed for this test - nil, // clock, not needed for this test - agentPodsInformer, - configMapsInformer, - observableWithInformerOption.WithInformer, - ) - agentPodInformerFilter = observableWithInformerOption.GetFilterForInformer(agentPodsInformer) - }) - - when("the change is happening in the agent's namespace", func() { - when("a pod with all agent labels is added/updated/deleted", func() { - it("returns true", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "kube-cert-agent.pinniped.dev": "true", - }, - }, - } - - r.True(agentPodInformerFilter.Add(pod)) - r.True(agentPodInformerFilter.Update(whateverPod, pod)) - r.True(agentPodInformerFilter.Update(pod, whateverPod)) - r.True(agentPodInformerFilter.Delete(pod)) - }) - }) - - when("a pod missing the agent label is added/updated/deleted", func() { - it("returns false", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "some-other-label-key": "some-other-label-value", - }, - }, - } - - r.False(agentPodInformerFilter.Add(pod)) - r.False(agentPodInformerFilter.Update(whateverPod, pod)) - r.False(agentPodInformerFilter.Update(pod, whateverPod)) - r.False(agentPodInformerFilter.Delete(pod)) - }) - }) - }) - }, spec.Parallel(), spec.Report(report.Terminal{})) -} - -type fakePodExecutor struct { - r *require.Assertions - - resultsToReturn []string - errorsToReturn []error - - calledWithPodName []string - calledWithPodNamespace []string - calledWithCommandAndArgs [][]string - - callCount int -} - -func (s *fakePodExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) { - s.calledWithPodNamespace = append(s.calledWithPodNamespace, podNamespace) - s.calledWithPodName = append(s.calledWithPodName, podName) - s.calledWithCommandAndArgs = append(s.calledWithCommandAndArgs, commandAndArgs) - s.r.Less(s.callCount, len(s.resultsToReturn), "unexpected extra invocation of fakePodExecutor") - result := s.resultsToReturn[s.callCount] - var err error = nil - if s.errorsToReturn != nil { - s.r.Less(s.callCount, len(s.errorsToReturn), "unexpected extra invocation of fakePodExecutor") - err = s.errorsToReturn[s.callCount] - } - s.callCount++ - if err != nil { - return "", err - } - return result, nil -} - -func TestManagerControllerSync(t *testing.T) { - name := t.Name() - spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { - const agentPodNamespace = "some-namespace" - const agentPodName = "some-agent-pod-name-123" - const certPathAnnotationName = "kube-cert-agent.pinniped.dev/cert-path" - const keyPathAnnotationName = "kube-cert-agent.pinniped.dev/key-path" - const fakeCertPath = "/some/cert/path" - const fakeKeyPath = "/some/key/path" - const credentialIssuerResourceName = "ci-resource-name" - - var r *require.Assertions - - var subject controllerlib.Controller - var cancelContext context.Context - var cancelContextCancelFunc context.CancelFunc - var syncContext *controllerlib.Context - var pinnipedAPIClient *pinnipedfake.Clientset - var kubeInformerFactory kubeinformers.SharedInformerFactory - var kubeClientset *kubernetesfake.Clientset - var fakeExecutor *fakePodExecutor - var credentialIssuerLabels map[string]string - var discoveryURLOverride *string - var dynamicCertProvider dynamiccert.Provider - var fakeCertPEM, fakeKeyPEM string - var credentialIssuerGVR schema.GroupVersionResource - var frozenNow time.Time - var defaultDynamicCertProviderCert string - var defaultDynamicCertProviderKey string - - // Defer starting the informers until the last possible moment so that the - // nested Before's can keep adding things to the informer caches. - var startInformersAndController = func() { - // Set this at the last second to allow for injection of server override. - subject = NewExecerController( - &CredentialIssuerLocationConfig{ - Name: credentialIssuerResourceName, - }, - credentialIssuerLabels, - discoveryURLOverride, - dynamicCertProvider, - fakeExecutor, - pinnipedAPIClient, - clock.NewFakeClock(frozenNow), - kubeInformerFactory.Core().V1().Pods(), - kubeInformerFactory.Core().V1().ConfigMaps(), - controllerlib.WithInformer, - ) - - // Set this at the last second to support calling subject.Name(). - syncContext = &controllerlib.Context{ - Context: cancelContext, - Name: subject.Name(), - Key: controllerlib.Key{ - Namespace: agentPodNamespace, - Name: agentPodName, - }, - } - - // Must start informers before calling TestRunSynchronously() - kubeInformerFactory.Start(cancelContext.Done()) - controllerlib.TestRunSynchronously(t, subject) - } - - var newAgentPod = func(agentPodName string, hasCertPathAnnotations bool) *corev1.Pod { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: agentPodName, - Namespace: agentPodNamespace, - Labels: map[string]string{ - "some-label-key": "some-label-value", - }, - }, - } - if hasCertPathAnnotations { - pod.Annotations = map[string]string{ - certPathAnnotationName: fakeCertPath, - keyPathAnnotationName: fakeKeyPath, - } - } - return pod - } - - var requireDynamicCertProviderHasDefaultValues = func() { - actualCertPEM, actualKeyPEM := dynamicCertProvider.CurrentCertKeyContent() - r.Equal(defaultDynamicCertProviderCert, string(actualCertPEM)) - r.Equal(defaultDynamicCertProviderKey, string(actualKeyPEM)) - } - - var requireNoExternalActionsTaken = func() { - r.Empty(pinnipedAPIClient.Actions()) - r.Zero(fakeExecutor.callCount) - requireDynamicCertProviderHasDefaultValues() - } - - it.Before(func() { - r = require.New(t) - - crt, key, err := testutil.CreateCertificate( - time.Now().Add(-time.Hour), - time.Now().Add(time.Hour), - ) - require.NoError(t, err) - defaultDynamicCertProviderCert = string(crt) - defaultDynamicCertProviderKey = string(key) - - cancelContext, cancelContextCancelFunc = context.WithCancel(context.Background()) - pinnipedAPIClient = pinnipedfake.NewSimpleClientset() - kubeClientset = kubernetesfake.NewSimpleClientset() - kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClientset, 0) - fakeExecutor = &fakePodExecutor{r: r} - frozenNow = time.Date(2020, time.September, 23, 7, 42, 0, 0, time.Local) - dynamicCertProvider = dynamiccert.NewCA(name) - err = dynamicCertProvider.SetCertKeyContent([]byte(defaultDynamicCertProviderCert), []byte(defaultDynamicCertProviderKey)) - r.NoError(err) - - loadFile := func(filename string) string { - bytes, err := ioutil.ReadFile(filename) - r.NoError(err) - return string(bytes) - } - fakeCertPEM = loadFile("./testdata/test.crt") - fakeKeyPEM = loadFile("./testdata/test.key") - - credentialIssuerGVR = schema.GroupVersionResource{ - Group: configv1alpha1.GroupName, - Version: configv1alpha1.SchemeGroupVersion.Version, - Resource: "credentialissuers", - } - }) - - it.After(func() { - cancelContextCancelFunc() - }) - - when("there is not yet any agent pods or they were deleted", func() { - it.Before(func() { - unrelatedPod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: "some other pod", - Namespace: agentPodNamespace, - }, - } - r.NoError(kubeClientset.Tracker().Add(unrelatedPod)) - startInformersAndController() - }) - - it("does nothing", func() { - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireNoExternalActionsTaken() - }) - }) - - when("there is an agent pod, as determined by its labels matching the agent pod template labels, which is not yet annotated by the annotater controller", func() { - it.Before(func() { - agentPod := newAgentPod(agentPodName, false) - r.NoError(kubeClientset.Tracker().Add(agentPod)) - startInformersAndController() - }) - - it("does nothing", func() { - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireNoExternalActionsTaken() - }) - }) - - when("there is an agent pod, as determined by its labels matching the agent pod template labels, and it was annotated by the annotater controller, but it is not Running", func() { - it.Before(func() { - agentPod := newAgentPod(agentPodName, true) - agentPod.Status.Phase = corev1.PodPending // not Running - r.NoError(kubeClientset.Tracker().Add(agentPod)) - startInformersAndController() - }) - - it("does nothing", func() { - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - requireNoExternalActionsTaken() - }) - }) - - when("there is an agent pod, as determined by its labels matching the agent pod template labels, which is already annotated by the annotater controller, and it is Running", func() { - it.Before(func() { - targetAgentPod := newAgentPod(agentPodName, true) - targetAgentPod.Status.Phase = corev1.PodRunning - anotherAgentPod := newAgentPod("some-other-agent-pod-which-is-not-the-context-of-this-sync", true) - r.NoError(kubeClientset.Tracker().Add(targetAgentPod)) - r.NoError(kubeClientset.Tracker().Add(anotherAgentPod)) - }) - - when("the resulting pod execs will succeed", func() { - it.Before(func() { - fakeExecutor.resultsToReturn = []string{fakeCertPEM, fakeKeyPEM} - }) - - when("the cluster-info ConfigMap is not found", func() { - it("returns an error and updates the strategy with an error", func() { - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), `failed to get cluster-info configmap: configmap "cluster-info" not found`) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotGetClusterInfoStrategyReason, - Message: `failed to get cluster-info configmap: configmap "cluster-info" not found`, - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) - expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) - r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) - }) - }) - - when("the cluster-info ConfigMap is missing a key", func() { - it.Before(func() { - r.NoError(kubeClientset.Tracker().Add(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ClusterInfoNamespace, - Name: clusterInfoName, - }, - Data: map[string]string{"uninteresting-key": "uninteresting-value"}, - })) - }) - it("returns an error", func() { - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), `failed to get kubeconfig key from cluster-info configmap`) - }) - }) - - when("the cluster-info ConfigMap is contains invalid YAML", func() { - it.Before(func() { - r.NoError(kubeClientset.Tracker().Add(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ClusterInfoNamespace, - Name: clusterInfoName, - }, - Data: map[string]string{"kubeconfig": "invalid-yaml"}, - })) - }) - it("returns an error", func() { - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), `failed to load data from kubeconfig key in cluster-info configmap`) - }) - }) - - when("the cluster-info ConfigMap is contains an empty list of clusters", func() { - it.Before(func() { - r.NoError(kubeClientset.Tracker().Add(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ClusterInfoNamespace, - Name: clusterInfoName, - }, - Data: map[string]string{ - "kubeconfig": here.Doc(` - kind: Config - apiVersion: v1 - clusters: [] - `), - "uninteresting-key": "uninteresting-value", - }, - })) - }) - it("returns an error", func() { - startInformersAndController() - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), `kubeconfig in kubeconfig key in cluster-info configmap did not contain any clusters`) - }) - }) - - when("the cluster-info ConfigMap is valid", func() { - it.Before(func() { - const caData = "c29tZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YQo=" // "some-certificate-authority-data" base64 encoded - const kubeServerURL = "https://some-server" - r.NoError(kubeClientset.Tracker().Add(&corev1.ConfigMap{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: ClusterInfoNamespace, - Name: clusterInfoName, - }, - Data: map[string]string{ - "kubeconfig": here.Docf(` - kind: Config - apiVersion: v1 - clusters: - - name: "" - cluster: - certificate-authority-data: "%s" - server: "%s"`, - caData, kubeServerURL), - "uninteresting-key": "uninteresting-value", - }, - })) - }) - - it("execs to the agent pod to get the keys and updates the dynamic certificates provider with the new certs", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - r.Equal(2, fakeExecutor.callCount) - - r.Equal(agentPodNamespace, fakeExecutor.calledWithPodNamespace[0]) - r.Equal(agentPodName, fakeExecutor.calledWithPodName[0]) - r.Equal([]string{"cat", fakeCertPath}, fakeExecutor.calledWithCommandAndArgs[0]) - - r.Equal(agentPodNamespace, fakeExecutor.calledWithPodNamespace[1]) - r.Equal(agentPodName, fakeExecutor.calledWithPodName[1]) - r.Equal([]string{"cat", fakeKeyPath}, fakeExecutor.calledWithCommandAndArgs[1]) - - actualCertPEM, actualKeyPEM := dynamicCertProvider.CurrentCertKeyContent() - r.Equal(fakeCertPEM, string(actualCertPEM)) - r.Equal(fakeKeyPEM, string(actualKeyPEM)) - }) - - when("there is already a CredentialIssuer", func() { - var initialCredentialIssuer *configv1alpha1.CredentialIssuer - - it.Before(func() { - initialCredentialIssuer = &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{}, - }, - } - r.NoError(pinnipedAPIClient.Tracker().Add(initialCredentialIssuer)) - }) - - it("also updates the the existing CredentialIssuer status field", func() { - startInformersAndController() - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - // The first update to the CredentialIssuer will set the strategy entry - expectedCredentialIssuer := initialCredentialIssuer.DeepCopy() - expectedCredentialIssuer.Status.Strategies = []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.SuccessStrategyStatus, - Reason: configv1alpha1.FetchedKeyStrategyReason, - Message: "Key was fetched successfully", - LastUpdateTime: metav1.NewTime(frozenNow), - Frontend: &configv1alpha1.CredentialIssuerFrontend{ - Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, - TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ - Server: "https://some-server", - CertificateAuthorityData: "c29tZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YQo=", - }, - }, - }, - } - expectedCredentialIssuer.Status.KubeConfigInfo = &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://some-server", - CertificateAuthorityData: "c29tZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YQo=", - } - expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) - expectedCreateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) - r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction}, pinnipedAPIClient.Actions()) - }) - - when("updating the CredentialIssuer fails", func() { - it.Before(func() { - pinnipedAPIClient.PrependReactor( - "update", - "credentialissuers", - func(_ coretesting.Action) (bool, runtime.Object, error) { - return true, nil, errors.New("some update error") - }, - ) - }) - - it("returns an error", func() { - startInformersAndController() - err := controllerlib.TestSync(t, subject, *syncContext) - r.EqualError(err, "could not create or update credentialissuer: some update error") - }) - }) - }) - - when("there is not already a CredentialIssuer", func() { - it.Before(func() { - server := "https://overridden-server-url.example.com" - discoveryURLOverride = &server - credentialIssuerLabels = map[string]string{"foo": "bar"} - startInformersAndController() - }) - - it("also creates the the CredentialIssuer with the appropriate status field and labels", func() { - r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{"foo": "bar"}, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - Labels: map[string]string{"foo": "bar"}, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.SuccessStrategyStatus, - Reason: configv1alpha1.FetchedKeyStrategyReason, - Message: "Key was fetched successfully", - LastUpdateTime: metav1.NewTime(frozenNow), - Frontend: &configv1alpha1.CredentialIssuerFrontend{ - Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, - TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ - Server: "https://overridden-server-url.example.com", - CertificateAuthorityData: "c29tZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YQo=", - }, - }, - }, - }, - KubeConfigInfo: &configv1alpha1.CredentialIssuerKubeConfigInfo{ - Server: "https://overridden-server-url.example.com", - CertificateAuthorityData: "c29tZS1jZXJ0aWZpY2F0ZS1hdXRob3JpdHktZGF0YQo=", - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) - expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) - r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) - }) - }) - }) - }) - - when("the first resulting pod exec will fail", func() { - var podExecErrorMessage string - - it.Before(func() { - podExecErrorMessage = "some pod exec error message" - fakeExecutor.errorsToReturn = []error{fmt.Errorf(podExecErrorMessage), nil} - fakeExecutor.resultsToReturn = []string{"", fakeKeyPEM} - startInformersAndController() - }) - - it("does not update the dynamic certificates provider", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), podExecErrorMessage) - requireDynamicCertProviderHasDefaultValues() - }) - - it("creates or updates the the CredentialIssuer status field with an error", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), podExecErrorMessage) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: podExecErrorMessage, - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) - expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) - r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) - }) - }) - - when("the second resulting pod exec will fail", func() { - var podExecErrorMessage string - - it.Before(func() { - podExecErrorMessage = "some pod exec error message" - fakeExecutor.errorsToReturn = []error{nil, fmt.Errorf(podExecErrorMessage)} - fakeExecutor.resultsToReturn = []string{fakeCertPEM, ""} - startInformersAndController() - }) - - it("does not update the dynamic certificates provider", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), podExecErrorMessage) - requireDynamicCertProviderHasDefaultValues() - }) - - it("creates or updates the the CredentialIssuer status field with an error", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), podExecErrorMessage) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: podExecErrorMessage, - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) - expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) - r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) - }) - }) - - when("the third resulting pod exec has invalid key data", func() { - var keyParseErrorMessage string - - it.Before(func() { - keyParseErrorMessage = "failed to set signing cert/key content from agent pod some-namespace/some-agent-pod-name-123: TestManagerControllerSync: attempt to set invalid key pair: tls: failed to find any PEM data in key input" - fakeExecutor.errorsToReturn = []error{nil, nil} - fakeExecutor.resultsToReturn = []string{fakeCertPEM, ""} - startInformersAndController() - }) - - it("does not update the dynamic certificates provider", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), keyParseErrorMessage) - requireDynamicCertProviderHasDefaultValues() - }) - - it("creates or updates the the CredentialIssuer status field with an error", func() { - r.EqualError(controllerlib.TestSync(t, subject, *syncContext), keyParseErrorMessage) - - expectedCreateCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - } - - expectedCredentialIssuer := &configv1alpha1.CredentialIssuer{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: credentialIssuerResourceName, - }, - Status: configv1alpha1.CredentialIssuerStatus{ - Strategies: []configv1alpha1.CredentialIssuerStrategy{ - { - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: keyParseErrorMessage, - LastUpdateTime: metav1.NewTime(frozenNow), - }, - }, - }, - } - expectedGetAction := coretesting.NewRootGetAction(credentialIssuerGVR, credentialIssuerResourceName) - expectedCreateAction := coretesting.NewRootCreateAction(credentialIssuerGVR, expectedCreateCredentialIssuer) - expectedUpdateAction := coretesting.NewRootUpdateSubresourceAction(credentialIssuerGVR, "status", expectedCredentialIssuer) - r.Equal([]coretesting.Action{expectedGetAction, expectedCreateAction, expectedUpdateAction}, pinnipedAPIClient.Actions()) - }) - }) - }) - }, spec.Parallel(), spec.Report(report.Terminal{})) -} diff --git a/internal/controller/kubecertagent/kubecertagent.go b/internal/controller/kubecertagent/kubecertagent.go index 553765fc..478d27fe 100644 --- a/internal/controller/kubecertagent/kubecertagent.go +++ b/internal/controller/kubecertagent/kubecertagent.go @@ -1,296 +1,528 @@ -// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// Copyright 2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 -// Package kubecertagent provides controllers that ensure a set of pods (the kube-cert-agent), is -// colocated with the Kubernetes controller manager so that Pinniped can access its signing keys. -// -// Note: the controllers use a filter that accepts all pods that look like the controller manager or -// an agent pod, across any add/update/delete event. Each of the controllers only care about a -// subset of these events in reality, but the liberal filter implementation serves as an MVP. +// Package kubecertagent provides controllers that ensure a pod (the kube-cert-agent), is +// co-located with the Kubernetes controller manager so that Pinniped can access its signing keys. package kubecertagent import ( - "encoding/hex" + "context" + "encoding/base64" "fmt" - "hash/fnv" + "strings" + "time" + "github.com/go-logr/logr" + "github.com/spf13/pflag" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" + apiequality "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/cache" "k8s.io/apimachinery/pkg/util/clock" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + appsv1informers "k8s.io/client-go/informers/apps/v1" corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog/v2" + "k8s.io/klog/v2/klogr" + "k8s.io/utils/pointer" configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" - "go.pinniped.dev/internal/plog" + pinnipedcontroller "go.pinniped.dev/internal/controller" + "go.pinniped.dev/internal/controller/issuerconfig" + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/dynamiccert" + "go.pinniped.dev/internal/kubeclient" ) const ( // ControllerManagerNamespace is the assumed namespace of the kube-controller-manager pod(s). ControllerManagerNamespace = "kube-system" - // controllerManagerNameAnnotationKey is used to store an agent pod's parent's name, i.e., the - // name of the controller manager pod with which it is supposed to be in sync. - controllerManagerNameAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-name" - // controllerManagerUIDAnnotationKey is used to store an agent pod's parent's UID, i.e., the UID - // of the controller manager pod with which it is supposed to be in sync. - controllerManagerUIDAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-uid" - // agentPodLabelKey is used to identify which pods are created by the kube-cert-agent // controllers. agentPodLabelKey = "kube-cert-agent.pinniped.dev" - agentPodLabelValue = "true" + agentPodLabelValue = "v2" - // agentPodCertPathAnnotationKey is the annotation that the kube-cert-agent pod will use - // to communicate the in-pod path to the kube API's certificate. - agentPodCertPathAnnotationKey = "kube-cert-agent.pinniped.dev/cert-path" - - // agentPodKeyPathAnnotationKey is the annotation that the kube-cert-agent pod will use - // to communicate the in-pod path to the kube API's key. - agentPodKeyPathAnnotationKey = "kube-cert-agent.pinniped.dev/key-path" + ClusterInfoNamespace = "kube-public" + clusterInfoName = "cluster-info" + clusterInfoConfigMapKey = "kubeconfig" ) -type AgentPodConfig struct { - // The namespace in which agent pods will be created. +// AgentConfig is the configuration for the kube-cert-agent controller. +type AgentConfig struct { + // Namespace in which agent pods will be created. Namespace string - // The container image used for the agent pods. + // ContainerImage specifies the container image used for the agent pods. ContainerImage string - // The name prefix for each of the agent pods. - PodNamePrefix string + // NamePrefix will be prefixed to all agent pod names. + NamePrefix string // ContainerImagePullSecrets is a list of names of Kubernetes Secret objects that will be used as // ImagePullSecrets on the kube-cert-agent pods. ContainerImagePullSecrets []string - // Additional labels that should be added to every agent pod during creation. - AdditionalLabels map[string]string + // CredentialIssuerName specifies the CredentialIssuer to be created/updated. + CredentialIssuerName string + + // Labels to be applied to the CredentialIssuer and agent pods. + Labels map[string]string + + // DiscoveryURLOverride is the Kubernetes server endpoint to report in the CredentialIssuer, overriding any + // value discovered in the kube-public/cluster-info ConfigMap. + DiscoveryURLOverride *string } -type CredentialIssuerLocationConfig struct { - // The resource name for the CredentialIssuer to be created/updated. - Name string -} - -func (c *AgentPodConfig) Labels() map[string]string { - allLabels := map[string]string{ - agentPodLabelKey: agentPodLabelValue, - } - for k, v := range c.AdditionalLabels { +func (a *AgentConfig) agentLabels() map[string]string { + allLabels := map[string]string{agentPodLabelKey: agentPodLabelValue} + for k, v := range a.Labels { allLabels[k] = v } return allLabels } -func (c *AgentPodConfig) AgentSelector() labels.Selector { - return labels.SelectorFromSet(map[string]string{agentPodLabelKey: agentPodLabelValue}) +func (a *AgentConfig) deploymentName() string { + return strings.TrimSuffix(a.NamePrefix, "-") } -func (c *AgentPodConfig) newAgentPod(controllerManagerPod *corev1.Pod) *corev1.Pod { - terminateImmediately := int64(0) - rootID := int64(0) - f := false - falsePtr := &f - - imagePullSecrets := []corev1.LocalObjectReference{} - for _, imagePullSecret := range c.ContainerImagePullSecrets { - imagePullSecrets = append( - imagePullSecrets, - corev1.LocalObjectReference{ - Name: imagePullSecret, - }, - ) - } - - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: fmt.Sprintf("%s%s", c.PodNamePrefix, hash(controllerManagerPod)), - Namespace: c.Namespace, - Labels: c.Labels(), - Annotations: map[string]string{ - controllerManagerNameAnnotationKey: controllerManagerPod.Name, - controllerManagerUIDAnnotationKey: string(controllerManagerPod.UID), - }, - }, - Spec: corev1.PodSpec{ - TerminationGracePeriodSeconds: &terminateImmediately, - ImagePullSecrets: imagePullSecrets, - Containers: []corev1.Container{ - { - Name: "sleeper", - Image: c.ContainerImage, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: []string{"/bin/sleep", "infinity"}, - VolumeMounts: controllerManagerPod.Spec.Containers[0].VolumeMounts, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("16Mi"), - corev1.ResourceCPU: resource.MustParse("10m"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("16Mi"), - corev1.ResourceCPU: resource.MustParse("10m"), - }, - }, - }, - }, - Volumes: controllerManagerPod.Spec.Volumes, - RestartPolicy: corev1.RestartPolicyNever, - NodeSelector: controllerManagerPod.Spec.NodeSelector, - AutomountServiceAccountToken: falsePtr, - NodeName: controllerManagerPod.Spec.NodeName, - Tolerations: controllerManagerPod.Spec.Tolerations, - // We need to run the agent pod as root since the file permissions - // on the cluster keypair usually restricts access to only root. - SecurityContext: &corev1.PodSecurityContext{ - RunAsUser: &rootID, - RunAsGroup: &rootID, - }, - }, - } +type agentController struct { + cfg AgentConfig + client *kubeclient.Client + kubeSystemPods corev1informers.PodInformer + agentDeployments appsv1informers.DeploymentInformer + agentPods corev1informers.PodInformer + kubePublicConfigMaps corev1informers.ConfigMapInformer + executor PodCommandExecutor + dynamicCertProvider dynamiccert.Private + clock clock.Clock + log logr.Logger + execCache *cache.Expiring } -func isAgentPodUpToDate(actualAgentPod, expectedAgentPod *corev1.Pod) bool { - requiredLabelsAllPresentWithCorrectValues := true - actualLabels := actualAgentPod.ObjectMeta.Labels - for expectedLabelKey, expectedLabelValue := range expectedAgentPod.ObjectMeta.Labels { - if actualLabels[expectedLabelKey] != expectedLabelValue { - requiredLabelsAllPresentWithCorrectValues = false - break +var ( + // controllerManagerLabels are the Kubernetes labels we expect on the kube-controller-manager Pod. + controllerManagerLabels = labels.SelectorFromSet(map[string]string{ //nolint: gochecknoglobals + "component": "kube-controller-manager", + }) + + // agentLabels are the Kubernetes labels we always expect on the kube-controller-manager Pod. + agentLabels = labels.SelectorFromSet(map[string]string{ //nolint: gochecknoglobals + agentPodLabelKey: agentPodLabelValue, + }) +) + +// NewAgentController returns a controller that manages the kube-cert-agent Deployment. It also is tasked with updating +// the CredentialIssuer with any errors that it encounters. +func NewAgentController( + cfg AgentConfig, + client *kubeclient.Client, + kubeSystemPods corev1informers.PodInformer, + agentDeployments appsv1informers.DeploymentInformer, + agentPods corev1informers.PodInformer, + kubePublicConfigMaps corev1informers.ConfigMapInformer, + dynamicCertProvider dynamiccert.Private, +) controllerlib.Controller { + return newAgentController( + cfg, + client, + kubeSystemPods, + agentDeployments, + agentPods, + kubePublicConfigMaps, + NewPodCommandExecutor(client.JSONConfig, client.Kubernetes), + dynamicCertProvider, + &clock.RealClock{}, + cache.NewExpiring(), + klogr.New(), + ) +} + +func newAgentController( + cfg AgentConfig, + client *kubeclient.Client, + kubeSystemPods corev1informers.PodInformer, + agentDeployments appsv1informers.DeploymentInformer, + agentPods corev1informers.PodInformer, + kubePublicConfigMaps corev1informers.ConfigMapInformer, + podCommandExecutor PodCommandExecutor, + dynamicCertProvider dynamiccert.Private, + clock clock.Clock, + execCache *cache.Expiring, + log logr.Logger, + options ...controllerlib.Option, +) controllerlib.Controller { + return controllerlib.New( + controllerlib.Config{ + Name: "kube-cert-agent-controller", + Syncer: &agentController{ + cfg: cfg, + client: client, + kubeSystemPods: kubeSystemPods, + agentDeployments: agentDeployments, + agentPods: agentPods, + kubePublicConfigMaps: kubePublicConfigMaps, + executor: podCommandExecutor, + dynamicCertProvider: dynamicCertProvider, + clock: clock, + log: log.WithName("kube-cert-agent-controller"), + execCache: execCache, + }, + }, + append([]controllerlib.Option{ + controllerlib.WithInformer( + kubeSystemPods, + pinnipedcontroller.SimpleFilterWithSingletonQueue(func(obj metav1.Object) bool { + return controllerManagerLabels.Matches(labels.Set(obj.GetLabels())) + }), + controllerlib.InformerOption{}, + ), + controllerlib.WithInformer( + agentDeployments, + pinnipedcontroller.SimpleFilterWithSingletonQueue(func(obj metav1.Object) bool { + return obj.GetNamespace() == cfg.Namespace && obj.GetName() == cfg.deploymentName() + }), + controllerlib.InformerOption{}, + ), + controllerlib.WithInformer( + agentPods, + pinnipedcontroller.SimpleFilterWithSingletonQueue(func(obj metav1.Object) bool { + return agentLabels.Matches(labels.Set(obj.GetLabels())) + }), + controllerlib.InformerOption{}, + ), + controllerlib.WithInformer( + kubePublicConfigMaps, + pinnipedcontroller.SimpleFilterWithSingletonQueue(func(obj metav1.Object) bool { + return obj.GetNamespace() == ClusterInfoNamespace && obj.GetName() == clusterInfoName + }), + controllerlib.InformerOption{}, + ), + // Be sure to run once even to make sure the CredentialIssuer is updated if there are no controller manager + // pods. We should be able to pass an empty key since we don't use the key in the sync (we sync + // the world). + controllerlib.WithInitialEvent(controllerlib.Key{}), + }, options...)..., + ) +} + +// Sync implements controllerlib.Syncer. +func (c *agentController) Sync(ctx controllerlib.Context) error { + // Find the latest healthy kube-controller-manager Pod in kube-system.. + controllerManagerPods, err := c.kubeSystemPods.Lister().Pods(ControllerManagerNamespace).List(controllerManagerLabels) + if err != nil { + err := fmt.Errorf("could not list controller manager pods: %w", err) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotFetchKeyStrategyReason) + } + newestControllerManager := newestRunningPod(controllerManagerPods) + + // If there are no healthy controller manager pods, we alert the user that we can't find the keypair via + // the CredentialIssuer. + if newestControllerManager == nil { + err := fmt.Errorf("could not find a healthy kube-controller-manager pod (%s)", pluralize(controllerManagerPods)) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotFetchKeyStrategyReason) + } + + if err := c.createOrUpdateDeployment(ctx, newestControllerManager); err != nil { + err := fmt.Errorf("could not ensure agent deployment: %w", err) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotFetchKeyStrategyReason) + } + + // Find the latest healthy agent Pod in our namespace. + agentPods, err := c.agentPods.Lister().Pods(c.cfg.Namespace).List(agentLabels) + if err != nil { + err := fmt.Errorf("could not list agent pods: %w", err) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotFetchKeyStrategyReason) + } + newestAgentPod := newestRunningPod(agentPods) + + // If there are no healthy controller agent pods, we alert the user that we can't find the keypair via + // the CredentialIssuer. + if newestAgentPod == nil { + err := fmt.Errorf("could not find a healthy agent pod (%s)", pluralize(agentPods)) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotFetchKeyStrategyReason) + } + + // Load the Kubernetes API info from the kube-public/cluster-info ConfigMap. + configMap, err := c.kubePublicConfigMaps.Lister().ConfigMaps(ClusterInfoNamespace).Get(clusterInfoName) + if err != nil { + err := fmt.Errorf("failed to get %s/%s configmap: %w", ClusterInfoNamespace, clusterInfoName, err) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotGetClusterInfoStrategyReason) + } + + apiInfo, err := c.extractAPIInfo(configMap) + if err != nil { + err := fmt.Errorf("could not extract Kubernetes API endpoint info from %s/%s configmap: %w", ClusterInfoNamespace, clusterInfoName, err) + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotGetClusterInfoStrategyReason) + } + + // Load the certificate and key from the agent pod into our in-memory signer. + if err := c.loadSigningKey(newestAgentPod); err != nil { + return c.failStrategyAndErr(ctx.Context, err, configv1alpha1.CouldNotFetchKeyStrategyReason) + } + + // Set the CredentialIssuer strategy to successful. + return issuerconfig.UpdateStrategy( + ctx.Context, + c.cfg.CredentialIssuerName, + c.cfg.Labels, + c.client.PinnipedConcierge, + configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Message: "key was fetched successfully", + LastUpdateTime: metav1.NewTime(c.clock.Now()), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: apiInfo, + }, + }, + ) +} + +func (c *agentController) loadSigningKey(agentPod *corev1.Pod) error { + // If we remember successfully loading the key from this pod recently, we can skip this step and return immediately. + if _, exists := c.execCache.Get(agentPod.UID); exists { + return nil + } + + // Exec into the agent pod and cat out the certificate and the key. + combinedPEM, err := c.executor.Exec( + agentPod.Namespace, agentPod.Name, + "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}", + ) + if err != nil { + return fmt.Errorf("could not exec into agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err) + } + + // Split up the output by looking for the block of newlines. + var certPEM, keyPEM string + if parts := strings.Split(combinedPEM, "\n\n\n"); len(parts) == 2 { + certPEM, keyPEM = parts[0], parts[1] + } + + // Load the certificate and key into the dynamic signer. + if err := c.dynamicCertProvider.SetCertKeyContent([]byte(certPEM), []byte(keyPEM)); err != nil { + return fmt.Errorf("failed to set signing cert/key content from agent pod %s/%s: %w", agentPod.Namespace, agentPod.Name, err) + } + + // Remember that we've successfully loaded the key from this pod so we can skip the exec+load if nothing has changed. + c.execCache.Set(agentPod.UID, struct{}{}, 15*time.Minute) + return nil +} + +func (c *agentController) createOrUpdateDeployment(ctx controllerlib.Context, newestControllerManager *corev1.Pod) error { + // Build the expected Deployment based on the kube-controller-manager Pod as a template. + expectedDeployment := c.newAgentDeployment(newestControllerManager) + + // Try to get the existing Deployment, if it exists. + existingDeployment, err := c.agentDeployments.Lister().Deployments(expectedDeployment.Namespace).Get(expectedDeployment.Name) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return fmt.Errorf("could not get deployments: %w", err) + } + + log := c.log.WithValues( + "deployment", klog.KObj(expectedDeployment), + "templatePod", klog.KObj(newestControllerManager), + ) + + // If the Deployment did not exist, create it and be done. + if notFound { + log.Info("creating new deployment") + _, err := c.client.Kubernetes.AppsV1().Deployments(expectedDeployment.Namespace).Create(ctx.Context, expectedDeployment, metav1.CreateOptions{}) + return err + } + + // Otherwise update the spec of the Deployment to match our desired state. + updatedDeployment := existingDeployment.DeepCopy() + updatedDeployment.Spec = expectedDeployment.Spec + updatedDeployment.ObjectMeta = mergeLabelsAndAnnotations(updatedDeployment.ObjectMeta, expectedDeployment.ObjectMeta) + + // If the existing Deployment already matches our desired spec, we're done. + if apiequality.Semantic.DeepDerivative(updatedDeployment, existingDeployment) { + return nil + } + + log.Info("updating existing deployment") + _, err = c.client.Kubernetes.AppsV1().Deployments(updatedDeployment.Namespace).Update(ctx.Context, updatedDeployment, metav1.UpdateOptions{}) + return err +} + +func (c *agentController) failStrategyAndErr(ctx context.Context, err error, reason configv1alpha1.StrategyReason) error { + return utilerrors.NewAggregate([]error{err, issuerconfig.UpdateStrategy( + ctx, + c.cfg.CredentialIssuerName, + c.cfg.Labels, + c.client.PinnipedConcierge, + configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: reason, + Message: err.Error(), + LastUpdateTime: metav1.NewTime(c.clock.Now()), + }, + )}) +} + +func (c *agentController) extractAPIInfo(configMap *corev1.ConfigMap) (*configv1alpha1.TokenCredentialRequestAPIInfo, error) { + kubeConfigYAML, kubeConfigPresent := configMap.Data[clusterInfoConfigMapKey] + if !kubeConfigPresent { + return nil, fmt.Errorf("missing %q key", clusterInfoConfigMapKey) + } + + kubeconfig, err := clientcmd.Load([]byte(kubeConfigYAML)) + if err != nil { + // We purposefully don't wrap "err" here because it's very verbose. + return nil, fmt.Errorf("key %q does not contain a valid kubeconfig", clusterInfoConfigMapKey) + } + + for _, v := range kubeconfig.Clusters { + result := &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: v.Server, + CertificateAuthorityData: base64.StdEncoding.EncodeToString(v.CertificateAuthorityData), + } + if c.cfg.DiscoveryURLOverride != nil { + result.Server = *c.cfg.DiscoveryURLOverride + } + return result, nil + } + return nil, fmt.Errorf("kubeconfig in key %q does not contain any clusters", clusterInfoConfigMapKey) +} + +// newestRunningPod takes a list of pods and returns the newest one with status.phase == "Running". +func newestRunningPod(pods []*corev1.Pod) *corev1.Pod { + // Compare two pods based on creation timestamp, breaking ties by name + newer := func(a, b *corev1.Pod) bool { + if a.CreationTimestamp.Time.Equal(b.CreationTimestamp.Time) { + return a.Name < b.Name + } + return a.CreationTimestamp.After(b.CreationTimestamp.Time) + } + + var result *corev1.Pod + for _, pod := range pods { + if pod.Status.Phase == corev1.PodRunning && (result == nil || newer(pod, result)) { + result = pod + } + } + return result +} + +func (c *agentController) newAgentDeployment(controllerManagerPod *corev1.Pod) *appsv1.Deployment { + var volumeMounts []corev1.VolumeMount + if len(controllerManagerPod.Spec.Containers) > 0 { + volumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts + } + + var imagePullSecrets []corev1.LocalObjectReference + if len(c.cfg.ContainerImagePullSecrets) > 0 { + imagePullSecrets = make([]corev1.LocalObjectReference, 0, len(c.cfg.ContainerImagePullSecrets)) + for _, name := range c.cfg.ContainerImagePullSecrets { + imagePullSecrets = append(imagePullSecrets, corev1.LocalObjectReference{Name: name}) } } - if actualAgentPod.Spec.SecurityContext == nil { - return false - } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.cfg.deploymentName(), + Namespace: c.cfg.Namespace, + Labels: c.cfg.Labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32Ptr(1), + Selector: metav1.SetAsLabelSelector(c.cfg.agentLabels()), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.cfg.agentLabels(), + }, + Spec: corev1.PodSpec{ + TerminationGracePeriodSeconds: pointer.Int64Ptr(0), + ImagePullSecrets: imagePullSecrets, + Containers: []corev1.Container{ + { + Name: "sleeper", + Image: c.cfg.ContainerImage, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: []string{"/bin/sleep", "infinity"}, + VolumeMounts: volumeMounts, + Env: []corev1.EnvVar{ + {Name: "CERT_PATH", Value: getContainerArgByName(controllerManagerPod, "cluster-signing-cert-file", "/etc/kubernetes/ca/ca.pem")}, + {Name: "KEY_PATH", Value: getContainerArgByName(controllerManagerPod, "cluster-signing-key-file", "/etc/kubernetes/ca/ca.key")}, + }, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("16Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("16Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + }, + }, + }, + }, + Volumes: controllerManagerPod.Spec.Volumes, + RestartPolicy: corev1.RestartPolicyAlways, + NodeSelector: controllerManagerPod.Spec.NodeSelector, + AutomountServiceAccountToken: pointer.BoolPtr(false), + NodeName: controllerManagerPod.Spec.NodeName, + Tolerations: controllerManagerPod.Spec.Tolerations, + // We need to run the agent pod as root since the file permissions + // on the cluster keypair usually restricts access to only root. + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64Ptr(0), + RunAsGroup: pointer.Int64Ptr(0), + }, + }, + }, - return requiredLabelsAllPresentWithCorrectValues && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.Containers[0].VolumeMounts, - expectedAgentPod.Spec.Containers[0].VolumeMounts, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.Containers[0].Name, - expectedAgentPod.Spec.Containers[0].Name, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.Containers[0].Image, - expectedAgentPod.Spec.Containers[0].Image, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.Containers[0].Command, - expectedAgentPod.Spec.Containers[0].Command, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.Volumes, - expectedAgentPod.Spec.Volumes, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.RestartPolicy, - expectedAgentPod.Spec.RestartPolicy, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.NodeSelector, - expectedAgentPod.Spec.NodeSelector, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.AutomountServiceAccountToken, - expectedAgentPod.Spec.AutomountServiceAccountToken, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.NodeName, - expectedAgentPod.Spec.NodeName, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.Tolerations, - expectedAgentPod.Spec.Tolerations, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.SecurityContext.RunAsUser, - expectedAgentPod.Spec.SecurityContext.RunAsUser, - ) && - equality.Semantic.DeepEqual( - actualAgentPod.Spec.SecurityContext.RunAsGroup, - expectedAgentPod.Spec.SecurityContext.RunAsGroup, - ) -} - -func isControllerManagerPod(obj metav1.Object) bool { - pod, ok := obj.(*corev1.Pod) - if !ok { - return false - } - - if pod.Labels == nil { - return false - } - - component, ok := pod.Labels["component"] - if !ok || component != "kube-controller-manager" { - return false - } - - if pod.Status.Phase != corev1.PodRunning { - return false - } - - return true -} - -func isAgentPod(obj metav1.Object) bool { - value, foundLabel := obj.GetLabels()[agentPodLabelKey] - return foundLabel && value == agentPodLabelValue -} - -func findControllerManagerPodForSpecificAgentPod( - agentPod *corev1.Pod, - kubeSystemPodInformer corev1informers.PodInformer, -) (*corev1.Pod, error) { - name, ok := agentPod.Annotations[controllerManagerNameAnnotationKey] - if !ok { - plog.Debug("agent pod missing parent name annotation", "pod", agentPod.Name) - return nil, nil - } - - uid, ok := agentPod.Annotations[controllerManagerUIDAnnotationKey] - if !ok { - plog.Debug("agent pod missing parent uid annotation", "pod", agentPod.Name) - return nil, nil - } - - maybeControllerManagerPod, err := kubeSystemPodInformer. - Lister(). - Pods(ControllerManagerNamespace). - Get(name) - notFound := k8serrors.IsNotFound(err) - if err != nil && !notFound { - return nil, fmt.Errorf("cannot get controller pod: %w", err) - } else if notFound || - maybeControllerManagerPod == nil || - string(maybeControllerManagerPod.UID) != uid { - return nil, nil - } - - return maybeControllerManagerPod, nil -} - -func strategyError(clock clock.Clock, err error) configv1alpha1.CredentialIssuerStrategy { - return configv1alpha1.CredentialIssuerStrategy{ - Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, - Status: configv1alpha1.ErrorStrategyStatus, - Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, - Message: err.Error(), - LastUpdateTime: metav1.NewTime(clock.Now()), + // Setting MinReadySeconds prevents the agent pods from being churned too quickly by the deployments controller. + MinReadySeconds: 10, + }, } } -func hash(controllerManagerPod *corev1.Pod) string { - // FNV should be faster than SHA, and we don't care about hash-reversibility here, and Kubernetes - // uses FNV for their pod templates, so should be good enough for us? - h := fnv.New32a() - _, _ = h.Write([]byte(controllerManagerPod.UID)) // Never returns an error, per godoc. - return hex.EncodeToString(h.Sum([]byte{})) +func mergeLabelsAndAnnotations(existing metav1.ObjectMeta, desired metav1.ObjectMeta) metav1.ObjectMeta { + result := existing.DeepCopy() + for k, v := range desired.Labels { + if result.Labels == nil { + result.Labels = map[string]string{} + } + result.Labels[k] = v + } + for k, v := range desired.Annotations { + if result.Annotations == nil { + result.Annotations = map[string]string{} + } + result.Annotations[k] = v + } + return *result +} + +func getContainerArgByName(pod *corev1.Pod, name, fallbackValue string) string { + for _, container := range pod.Spec.Containers { + flagset := pflag.NewFlagSet("", pflag.ContinueOnError) + flagset.ParseErrorsWhitelist = pflag.ParseErrorsWhitelist{UnknownFlags: true} + var val string + flagset.StringVar(&val, name, "", "") + _ = flagset.Parse(append(container.Command, container.Args...)) + if val != "" { + return val + } + } + return fallbackValue +} + +func pluralize(pods []*corev1.Pod) string { + if len(pods) == 1 { + return "1 candidate" + } + return fmt.Sprintf("%d candidates", len(pods)) } diff --git a/internal/controller/kubecertagent/kubecertagent_test.go b/internal/controller/kubecertagent/kubecertagent_test.go index 29bb5955..4a1a17f9 100644 --- a/internal/controller/kubecertagent/kubecertagent_test.go +++ b/internal/controller/kubecertagent/kubecertagent_test.go @@ -1,249 +1,831 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package kubecertagent import ( + "context" + "fmt" "testing" + "time" - "github.com/sclevine/spec" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - kubeinformers "k8s.io/client-go/informers" - corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/apimachinery/pkg/util/cache" + "k8s.io/apimachinery/pkg/util/clock" + "k8s.io/client-go/informers" + kubefake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + "k8s.io/utils/pointer" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" + conciergefake "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned/fake" + "go.pinniped.dev/internal/controller/kubecertagent/mocks" "go.pinniped.dev/internal/controllerlib" - "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/internal/here" + "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/testutil/testlogger" ) -func exampleControllerManagerAndAgentPods( - kubeSystemNamespace, - agentPodNamespace, - certPath, - keyPath string, -) (*corev1.Pod, *corev1.Pod) { - controllerManagerPod := &corev1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: corev1.SchemeGroupVersion.String(), - Kind: "Pod", - }, +func TestAgentController(t *testing.T) { + t.Parallel() + now := time.Date(2021, 4, 13, 9, 57, 0, 0, time.UTC) + + healthyKubeControllerManagerPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Namespace: kubeSystemNamespace, - Name: "some-controller-manager-name", - Labels: map[string]string{ - "component": "kube-controller-manager", - }, - UID: types.UID("some-controller-manager-uid"), + Namespace: "kube-system", + Name: "kube-controller-manager-1", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-1 * time.Hour)), }, Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Image: "some-controller-manager-image", - Command: []string{ - "kube-controller-manager", - "--cluster-signing-cert-file=" + certPath, - "--cluster-signing-key-file=" + keyPath, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "some-volume-mount-name", - }, + Containers: []corev1.Container{{ + Name: "kube-controller-manager", + Image: "kubernetes/kube-controller-manager", + Command: []string{ + "kube-controller-manager", + "--some-flag", + "--some-other-flag", + "--cluster-signing-cert-file", "/path/to/signing.crt", + "--cluster-signing-key-file=/path/to/signing.key", + "some arguments here", + "--and-another-flag", + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "test-volume", + ReadOnly: true, + MountPath: "/path", + }}, + }}, + Volumes: []corev1.Volume{{ + Name: "test-volume", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/host/path", }, }, - }, - NodeName: "some-node-name", - NodeSelector: map[string]string{ - "some-node-selector-key": "some-node-selector-value", - }, - Tolerations: []corev1.Toleration{ - { - Key: "some-toleration", - }, - }, + }}, }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + healthyAgentDeployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "concierge", + Name: "pinniped-concierge-kube-cert-agent", + Labels: map[string]string{"extralabel": "labelvalue"}, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32Ptr(1), + Selector: metav1.SetAsLabelSelector(map[string]string{ + "extralabel": "labelvalue", + "kube-cert-agent.pinniped.dev": "v2", + }), + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "extralabel": "labelvalue", + "kube-cert-agent.pinniped.dev": "v2", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "sleeper", + Image: "pinniped-server-image", + Command: []string{"/bin/sleep", "infinity"}, + Env: []corev1.EnvVar{ + {Name: "CERT_PATH", Value: "/path/to/signing.crt"}, + {Name: "KEY_PATH", Value: "/path/to/signing.key"}, + }, + VolumeMounts: []corev1.VolumeMount{{ + Name: "test-volume", + ReadOnly: true, + MountPath: "/path", + }}, + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("16Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + }, + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("16Mi"), + corev1.ResourceCPU: resource.MustParse("10m"), + }, + }, + ImagePullPolicy: corev1.PullIfNotPresent, + }}, + RestartPolicy: corev1.RestartPolicyAlways, + TerminationGracePeriodSeconds: pointer.Int64Ptr(0), + AutomountServiceAccountToken: pointer.BoolPtr(false), + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: pointer.Int64Ptr(0), + RunAsGroup: pointer.Int64Ptr(0), + }, + ImagePullSecrets: []corev1.LocalObjectReference{{ + Name: "pinniped-image-pull-secret", + }}, + Volumes: []corev1.Volume{{ + Name: "test-volume", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/host/path", + }, + }, + }}, + }, + }, + MinReadySeconds: 10, }, } - zero := int64(0) - f := false + // Make another kube-controller-manager pod that's similar, but does not have the CLI flags we're expecting. + // We should handle this by falling back to default values for the cert and key paths. + healthyKubeControllerManagerPodWithoutArgs := healthyKubeControllerManagerPod.DeepCopy() + healthyKubeControllerManagerPodWithoutArgs.Spec.Containers[0].Command = []string{"kube-controller-manager"} + healthyAgentDeploymentWithDefaultedPaths := healthyAgentDeployment.DeepCopy() + healthyAgentDeploymentWithDefaultedPaths.Spec.Template.Spec.Containers[0].Env = []corev1.EnvVar{ + {Name: "CERT_PATH", Value: "/etc/kubernetes/ca/ca.pem"}, + {Name: "KEY_PATH", Value: "/etc/kubernetes/ca/ca.key"}, + } - // fnv 32a hash of controller-manager uid - controllerManagerPodHash := "fbb0addd" - agentPod := &corev1.Pod{ + // If an admission controller sets extra labels or annotations, that's okay. + // We test this by ensuring that if a Deployment exists with extra labels, we don't try to delete them. + healthyAgentDeploymentWithExtraLabels := healthyAgentDeployment.DeepCopy() + healthyAgentDeploymentWithExtraLabels.Labels["some-additional-label"] = "some-additional-value" + healthyAgentDeploymentWithExtraLabels.Annotations = map[string]string{"some-additional-annotation": "some-additional-value"} + + // If a Deployment with the wrong image exists, we want to change that. + agentDeploymentWithExtraLabelsAndWrongImage := healthyAgentDeploymentWithExtraLabels.DeepCopy() + agentDeploymentWithExtraLabelsAndWrongImage.Spec.Template.Spec.Containers[0].Image = "wrong-image" + + healthyAgentPod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "some-agent-name-" + controllerManagerPodHash, - Namespace: agentPodNamespace, - Labels: map[string]string{ - "kube-cert-agent.pinniped.dev": "true", - "myLabelKey1": "myLabelValue1", - "myLabelKey2": "myLabelValue2", - }, - Annotations: map[string]string{ - "kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name, - "kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID), - }, + Namespace: "concierge", + Name: "pinniped-concierge-kube-cert-agent-xyz-1234", + UID: types.UID("pinniped-concierge-kube-cert-agent-xyz-1234-test-uid"), + Labels: map[string]string{"kube-cert-agent.pinniped.dev": "v2"}, + CreationTimestamp: metav1.NewTime(now.Add(-2 * time.Hour)), }, - Spec: corev1.PodSpec{ - TerminationGracePeriodSeconds: &zero, - ImagePullSecrets: []corev1.LocalObjectReference{ - { - Name: "some-image-pull-secret", - }, - }, - Containers: []corev1.Container{ - { - Name: "sleeper", - Image: "some-agent-image", - ImagePullPolicy: corev1.PullIfNotPresent, - VolumeMounts: controllerManagerPod.Spec.Containers[0].VolumeMounts, - Command: []string{"/bin/sleep", "infinity"}, - Resources: corev1.ResourceRequirements{ - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("16Mi"), - corev1.ResourceCPU: resource.MustParse("10m"), - }, - Requests: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("16Mi"), - corev1.ResourceCPU: resource.MustParse("10m"), - }, - }, - }, - }, - RestartPolicy: corev1.RestartPolicyNever, - AutomountServiceAccountToken: &f, - NodeName: controllerManagerPod.Spec.NodeName, - NodeSelector: controllerManagerPod.Spec.NodeSelector, - Tolerations: controllerManagerPod.Spec.Tolerations, - SecurityContext: &corev1.PodSecurityContext{RunAsUser: &zero, RunAsGroup: &zero}, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + pendingAgentPod := healthyAgentPod.DeepCopy() + pendingAgentPod.Status.Phase = corev1.PodPending + + validClusterInfoConfigMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "kube-public", Name: "cluster-info"}, + Data: map[string]string{"kubeconfig": here.Docf(` + kind: Config + apiVersion: v1 + clusters: + - name: "" + cluster: + certificate-authority-data: dGVzdC1rdWJlcm5ldGVzLWNh # "test-kubernetes-ca" + server: https://test-kubernetes-endpoint.example.com + `), }, } - return controllerManagerPod, agentPod + mockExecSucceeds := func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) { + executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}"). + Return("test-cert\n\n\ntest-key", nil) + dynamicCert.SetCertKeyContent([]byte("test-cert"), []byte("test-key")). + Return(nil) + } + + tests := []struct { + name string + discoveryURLOverride *string + kubeObjects []runtime.Object + addKubeReactions func(*kubefake.Clientset) + mocks func(*testing.T, *mocks.MockPodCommandExecutorMockRecorder, *mocks.MockDynamicCertPrivateMockRecorder, *cache.Expiring) + wantDistinctErrors []string + wantDistinctLogs []string + wantAgentDeployment *appsv1.Deployment + wantStrategy *configv1alpha1.CredentialIssuerStrategy + }{ + { + name: "no kube-controller-manager pods", + kubeObjects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-proxy", + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + }, + wantDistinctErrors: []string{ + "could not find a healthy kube-controller-manager pod (0 candidates)", + }, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not find a healthy kube-controller-manager pod (0 candidates)", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "only unhealthy kube-controller-manager pods", + kubeObjects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-proxy", + }, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-1", + Labels: map[string]string{"component": "kube-controller-manager"}, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodPending}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-2", + Labels: map[string]string{"component": "kube-controller-manager"}, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodUnknown}, + }, + }, + wantDistinctErrors: []string{ + "could not find a healthy kube-controller-manager pod (2 candidates)", + }, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not find a healthy kube-controller-manager pod (2 candidates)", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "failed to created new deployment", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + }, + addKubeReactions: func(clientset *kubefake.Clientset) { + clientset.PrependReactor("create", "deployments", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("some creation error") + }) + }, + wantDistinctErrors: []string{ + "could not ensure agent deployment: some creation error", + }, + wantDistinctLogs: []string{ + `kube-cert-agent-controller "level"=0 "msg"="creating new deployment" "deployment"={"name":"pinniped-concierge-kube-cert-agent","namespace":"concierge"} "templatePod"={"name":"kube-controller-manager-1","namespace":"kube-system"}`, + }, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not ensure agent deployment: some creation error", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "created new deployment, no agent pods running yet", + kubeObjects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-3", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-1 * time.Hour)), + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + healthyKubeControllerManagerPod, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-2", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-2 * time.Hour)), + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + pendingAgentPod, + }, + wantDistinctErrors: []string{ + "could not find a healthy agent pod (1 candidate)", + }, + wantDistinctLogs: []string{ + `kube-cert-agent-controller "level"=0 "msg"="creating new deployment" "deployment"={"name":"pinniped-concierge-kube-cert-agent","namespace":"concierge"} "templatePod"={"name":"kube-controller-manager-1","namespace":"kube-system"}`, + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not find a healthy agent pod (1 candidate)", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "created new deployment with defaulted paths, no agent pods running yet", + kubeObjects: []runtime.Object{ + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-3", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-1 * time.Hour)), + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + healthyKubeControllerManagerPodWithoutArgs, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-2", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-2 * time.Hour)), + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + pendingAgentPod, + }, + wantDistinctErrors: []string{ + "could not find a healthy agent pod (1 candidate)", + }, + wantDistinctLogs: []string{ + `kube-cert-agent-controller "level"=0 "msg"="creating new deployment" "deployment"={"name":"pinniped-concierge-kube-cert-agent","namespace":"concierge"} "templatePod"={"name":"kube-controller-manager-1","namespace":"kube-system"}`, + }, + wantAgentDeployment: healthyAgentDeploymentWithDefaultedPaths, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not find a healthy agent pod (1 candidate)", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "update to existing deployment, no running agent pods yet", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-3", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-1 * time.Hour)), + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "kube-system", + Name: "kube-controller-manager-2", + Labels: map[string]string{"component": "kube-controller-manager"}, + CreationTimestamp: metav1.NewTime(now.Add(-2 * time.Hour)), + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + }, + agentDeploymentWithExtraLabelsAndWrongImage, + pendingAgentPod, + }, + wantDistinctErrors: []string{ + "could not find a healthy agent pod (1 candidate)", + }, + wantDistinctLogs: []string{ + `kube-cert-agent-controller "level"=0 "msg"="updating existing deployment" "deployment"={"name":"pinniped-concierge-kube-cert-agent","namespace":"concierge"} "templatePod"={"name":"kube-controller-manager-1","namespace":"kube-system"}`, + }, + wantAgentDeployment: healthyAgentDeploymentWithExtraLabels, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not find a healthy agent pod (1 candidate)", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap missing", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + }, + wantDistinctErrors: []string{ + "failed to get kube-public/cluster-info configmap: configmap \"cluster-info\" not found", + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotGetClusterInfoStrategyReason, + Message: "failed to get kube-public/cluster-info configmap: configmap \"cluster-info\" not found", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap missing key", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "kube-public", Name: "cluster-info"}, + Data: map[string]string{}, + }, + }, + wantDistinctErrors: []string{ + "could not extract Kubernetes API endpoint info from kube-public/cluster-info configmap: missing \"kubeconfig\" key", + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotGetClusterInfoStrategyReason, + Message: "could not extract Kubernetes API endpoint info from kube-public/cluster-info configmap: missing \"kubeconfig\" key", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap key has invalid data", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "kube-public", Name: "cluster-info"}, + Data: map[string]string{"kubeconfig": "'"}, + }, + }, + wantDistinctErrors: []string{ + "could not extract Kubernetes API endpoint info from kube-public/cluster-info configmap: key \"kubeconfig\" does not contain a valid kubeconfig", + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotGetClusterInfoStrategyReason, + Message: "could not extract Kubernetes API endpoint info from kube-public/cluster-info configmap: key \"kubeconfig\" does not contain a valid kubeconfig", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap kubeconfig has no clusters", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{Namespace: "kube-public", Name: "cluster-info"}, + Data: map[string]string{"kubeconfig": "{}"}, + }, + }, + wantDistinctErrors: []string{ + "could not extract Kubernetes API endpoint info from kube-public/cluster-info configmap: kubeconfig in key \"kubeconfig\" does not contain any clusters", + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotGetClusterInfoStrategyReason, + Message: "could not extract Kubernetes API endpoint info from kube-public/cluster-info configmap: kubeconfig in key \"kubeconfig\" does not contain any clusters", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap is valid,, exec into agent pod fails", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + validClusterInfoConfigMap, + }, + mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) { + executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}"). + Return("", fmt.Errorf("some exec error")). + AnyTimes() + }, + wantDistinctErrors: []string{ + "could not exec into agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: some exec error", + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "could not exec into agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: some exec error", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap is valid, exec into agent pod returns bogus certs", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + validClusterInfoConfigMap, + }, + mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) { + executor.Exec("concierge", "pinniped-concierge-kube-cert-agent-xyz-1234", "sh", "-c", "cat ${CERT_PATH}; echo; echo; cat ${KEY_PATH}"). + Return("bogus-data", nil). + AnyTimes() + dynamicCert.SetCertKeyContent([]byte(""), []byte("")). + Return(fmt.Errorf("some dynamic cert error")). + AnyTimes() + }, + wantDistinctErrors: []string{ + "failed to set signing cert/key content from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: some dynamic cert error", + }, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.ErrorStrategyStatus, + Reason: configv1alpha1.CouldNotFetchKeyStrategyReason, + Message: "failed to set signing cert/key content from agent pod concierge/pinniped-concierge-kube-cert-agent-xyz-1234: some dynamic cert error", + LastUpdateTime: metav1.NewTime(now), + }, + }, + { + name: "deployment exists, configmap is valid, exec is cached", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + validClusterInfoConfigMap, + }, + mocks: func(t *testing.T, executor *mocks.MockPodCommandExecutorMockRecorder, dynamicCert *mocks.MockDynamicCertPrivateMockRecorder, execCache *cache.Expiring) { + // If we pre-fill the cache here, we should never see any calls to the executor or dynamicCert mocks. + execCache.Set(healthyAgentPod.UID, struct{}{}, 1*time.Hour) + }, + wantDistinctErrors: []string{""}, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Message: "key was fetched successfully", + LastUpdateTime: metav1.NewTime(now), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://test-kubernetes-endpoint.example.com", + CertificateAuthorityData: "dGVzdC1rdWJlcm5ldGVzLWNh", + }, + }, + }, + }, + { + name: "deployment exists, configmap is valid, exec succeeds", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + validClusterInfoConfigMap, + }, + mocks: mockExecSucceeds, + wantDistinctErrors: []string{""}, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Message: "key was fetched successfully", + LastUpdateTime: metav1.NewTime(now), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://test-kubernetes-endpoint.example.com", + CertificateAuthorityData: "dGVzdC1rdWJlcm5ldGVzLWNh", + }, + }, + }, + }, + { + name: "deployment exists, configmap is valid, exec succeeds, overridden discovery URL", + kubeObjects: []runtime.Object{ + healthyKubeControllerManagerPod, + healthyAgentDeployment, + healthyAgentPod, + validClusterInfoConfigMap, + }, + discoveryURLOverride: pointer.StringPtr("https://overridden-server.example.com/some/path"), + mocks: mockExecSucceeds, + wantDistinctErrors: []string{""}, + wantAgentDeployment: healthyAgentDeployment, + wantStrategy: &configv1alpha1.CredentialIssuerStrategy{ + Type: configv1alpha1.KubeClusterSigningCertificateStrategyType, + Status: configv1alpha1.SuccessStrategyStatus, + Reason: configv1alpha1.FetchedKeyStrategyReason, + Message: "key was fetched successfully", + LastUpdateTime: metav1.NewTime(now), + Frontend: &configv1alpha1.CredentialIssuerFrontend{ + Type: configv1alpha1.TokenCredentialRequestAPIFrontendType, + TokenCredentialRequestAPIInfo: &configv1alpha1.TokenCredentialRequestAPIInfo{ + Server: "https://overridden-server.example.com/some/path", + CertificateAuthorityData: "dGVzdC1rdWJlcm5ldGVzLWNh", + }, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + kubeClientset := kubefake.NewSimpleClientset(tt.kubeObjects...) + if tt.addKubeReactions != nil { + tt.addKubeReactions(kubeClientset) + } + + conciergeClientset := conciergefake.NewSimpleClientset() + kubeInformers := informers.NewSharedInformerFactory(kubeClientset, 0) + log := testlogger.New(t) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockExecutor := mocks.NewMockPodCommandExecutor(ctrl) + mockDynamicCert := mocks.NewMockDynamicCertPrivate(ctrl) + fakeClock := clock.NewFakeClock(now) + execCache := cache.NewExpiringWithClock(fakeClock) + if tt.mocks != nil { + tt.mocks(t, mockExecutor.EXPECT(), mockDynamicCert.EXPECT(), execCache) + } + controller := newAgentController( + AgentConfig{ + Namespace: "concierge", + ContainerImage: "pinniped-server-image", + NamePrefix: "pinniped-concierge-kube-cert-agent-", + ContainerImagePullSecrets: []string{"pinniped-image-pull-secret"}, + CredentialIssuerName: "pinniped-concierge-config", + Labels: map[string]string{"extralabel": "labelvalue"}, + DiscoveryURLOverride: tt.discoveryURLOverride, + }, + &kubeclient.Client{Kubernetes: kubeClientset, PinnipedConcierge: conciergeClientset}, + kubeInformers.Core().V1().Pods(), + kubeInformers.Apps().V1().Deployments(), + kubeInformers.Core().V1().Pods(), + kubeInformers.Core().V1().ConfigMaps(), + mockExecutor, + mockDynamicCert, + fakeClock, + execCache, + log, + controllerlib.WithMaxRetries(1), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + errorMessages := runControllerUntilQuiet(ctx, t, controller, kubeInformers) + assert.Equal(t, tt.wantDistinctErrors, deduplicate(errorMessages), "unexpected errors") + assert.Equal(t, tt.wantDistinctLogs, deduplicate(log.Lines()), "unexpected logs") + + // Assert that the agent deployment is in the expected final state. + deployments, err := kubeClientset.AppsV1().Deployments("concierge").List(ctx, metav1.ListOptions{}) + require.NoError(t, err) + if tt.wantAgentDeployment == nil { + require.Empty(t, deployments.Items, "did not expect an agent deployment") + } else { + require.Len(t, deployments.Items, 1, "expected a single agent deployment") + require.Equal(t, tt.wantAgentDeployment, &deployments.Items[0]) + } + + // Assert that the CredentialIssuer is in the expected final state + credIssuer, err := conciergeClientset.ConfigV1alpha1().CredentialIssuers().Get(ctx, "pinniped-concierge-config", metav1.GetOptions{}) + require.NoError(t, err) + require.Len(t, credIssuer.Status.Strategies, 1, "expected a single strategy in the CredentialIssuer") + require.Equal(t, tt.wantStrategy, &credIssuer.Status.Strategies[0]) + }) + } } -func defineSharedKubecertagentFilterSpecs( - t *testing.T, - name string, - newFunc func( - agentPodConfig *AgentPodConfig, - credentialIssuerLocationConfig *CredentialIssuerLocationConfig, - kubeSystemPodInformer corev1informers.PodInformer, - agentPodInformer corev1informers.PodInformer, - observableWithInformerOption *testutil.ObservableWithInformerOption, - ), -) { - spec.Run(t, name, func(t *testing.T, when spec.G, it spec.S) { - var r *require.Assertions - var kubeSystemPodInformerFilter, agentPodInformerFilter controllerlib.Filter +func TestMergeLabelsAndAnnotations(t *testing.T) { + t.Parallel() - whateverPod := &corev1.Pod{} - - it.Before(func() { - r = require.New(t) - - kubeSystemPodInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Pods() - agentPodInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Pods() - observableWithInformerOption := testutil.NewObservableWithInformerOption() - newFunc(&AgentPodConfig{}, &CredentialIssuerLocationConfig{}, kubeSystemPodInformer, agentPodInformer, observableWithInformerOption) - - kubeSystemPodInformerFilter = observableWithInformerOption.GetFilterForInformer(kubeSystemPodInformer) - agentPodInformerFilter = observableWithInformerOption.GetFilterForInformer(agentPodInformer) + tests := []struct { + name string + existing metav1.ObjectMeta + desired metav1.ObjectMeta + expected metav1.ObjectMeta + }{ + { + name: "empty", + existing: metav1.ObjectMeta{}, + desired: metav1.ObjectMeta{}, + expected: metav1.ObjectMeta{}, + }, + { + name: "new labels and annotations", + existing: metav1.ObjectMeta{}, + desired: metav1.ObjectMeta{ + Labels: map[string]string{"k1": "v1"}, + Annotations: map[string]string{"k2": "v2"}, + }, + expected: metav1.ObjectMeta{ + Labels: map[string]string{"k1": "v1"}, + Annotations: map[string]string{"k2": "v2"}, + }, + }, + { + name: "merged labels and annotations", + existing: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-name", + Labels: map[string]string{"k1": "old-v1", "extra-1": "v3"}, + Annotations: map[string]string{"k2": "old-v2", "extra-2": "v4"}, + }, + desired: metav1.ObjectMeta{ + Labels: map[string]string{"k1": "v1"}, + Annotations: map[string]string{"k2": "v2"}, + }, + expected: metav1.ObjectMeta{ + Namespace: "test-namespace", + Name: "test-name", + Labels: map[string]string{"k1": "v1", "extra-1": "v3"}, + Annotations: map[string]string{"k2": "v2", "extra-2": "v4"}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + existingCopy := tt.existing.DeepCopy() + desiredCopy := tt.desired.DeepCopy() + got := mergeLabelsAndAnnotations(tt.existing, tt.desired) + require.Equal(t, tt.expected, got) + require.Equal(t, existingCopy, &tt.existing, "input was modified!") + require.Equal(t, desiredCopy, &tt.desired, "input was modified!") }) + } +} - when("the event is happening in the kube system namespace", func() { - when("a pod with the proper controller manager labels and phase is added/updated/deleted", func() { - it("returns true", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "component": "kube-controller-manager", - }, - }, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - }, - } +func deduplicate(strings []string) []string { + if strings == nil { + return nil + } + seen, result := map[string]bool{}, make([]string, 0, len(strings)) + for _, s := range strings { + if !seen[s] { + seen[s] = true + result = append(result, s) + } + } + return result +} - r.True(kubeSystemPodInformerFilter.Add(pod)) - r.True(kubeSystemPodInformerFilter.Update(whateverPod, pod)) - r.True(kubeSystemPodInformerFilter.Update(pod, whateverPod)) - r.True(kubeSystemPodInformerFilter.Delete(pod)) - }) - }) +func runControllerUntilQuiet(ctx context.Context, t *testing.T, controller controllerlib.Controller, informers ...informers.SharedInformerFactory) []string { + ctx, cancel := context.WithCancel(ctx) + defer cancel() - when("a pod without the proper controller manager label is added/updated/deleted", func() { - it("returns false", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{}, - Status: corev1.PodStatus{ - Phase: corev1.PodRunning, - }, - } - - r.False(kubeSystemPodInformerFilter.Add(pod)) - r.False(kubeSystemPodInformerFilter.Update(whateverPod, pod)) - r.False(kubeSystemPodInformerFilter.Update(pod, whateverPod)) - r.False(kubeSystemPodInformerFilter.Delete(pod)) - }) - }) - - when("a pod without the proper controller manager phase is added/updated/deleted", func() { - it("returns false", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "component": "kube-controller-manager", - }, - }, - } - - r.False(kubeSystemPodInformerFilter.Add(pod)) - r.False(kubeSystemPodInformerFilter.Update(whateverPod, pod)) - r.False(kubeSystemPodInformerFilter.Update(pod, whateverPod)) - r.False(kubeSystemPodInformerFilter.Delete(pod)) - }) - }) - }) - - when("the change is happening in the agent's informer", func() { - when("a pod with the agent label is added/updated/deleted", func() { - it("returns true", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "kube-cert-agent.pinniped.dev": "true", - }, - }, - } - - r.True(agentPodInformerFilter.Add(pod)) - r.True(agentPodInformerFilter.Update(whateverPod, pod)) - r.True(agentPodInformerFilter.Update(pod, whateverPod)) - r.True(agentPodInformerFilter.Delete(pod)) - }) - }) - - when("a pod missing the agent label is added/updated/deleted", func() { - it("returns false", func() { - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - "some-other-label-key": "some-other-label-value", - }, - }, - } - - r.False(agentPodInformerFilter.Add(pod)) - r.False(agentPodInformerFilter.Update(whateverPod, pod)) - r.False(agentPodInformerFilter.Update(pod, whateverPod)) - r.False(agentPodInformerFilter.Delete(pod)) - }) - }) + errorStream := make(chan error) + controllerlib.TestWrap(t, controller, func(syncer controllerlib.Syncer) controllerlib.Syncer { + controller.Name() + return controllerlib.SyncFunc(func(ctx controllerlib.Context) error { + err := syncer.Sync(ctx) + errorStream <- err + return err }) }) + + for _, informer := range informers { + informer.Start(ctx.Done()) + } + + go controller.Run(ctx, 1) + + // Wait until the controller is quiet for two seconds. + var errorMessages []string +done: + for { + select { + case err := <-errorStream: + if err == nil { + errorMessages = append(errorMessages, "") + } else { + errorMessages = append(errorMessages, err.Error()) + } + case <-time.After(2 * time.Second): + break done + } + } + return errorMessages } diff --git a/internal/controller/kubecertagent/pod_command_executor.go b/internal/controller/kubecertagent/pod_command_executor.go index 7b982cf5..ff9aeb31 100644 --- a/internal/controller/kubecertagent/pod_command_executor.go +++ b/internal/controller/kubecertagent/pod_command_executor.go @@ -30,6 +30,7 @@ func NewPodCommandExecutor(kubeConfig *restclient.Config, kubeClient kubernetes. } func (s *kubeClientPodCommandExecutor) Exec(podNamespace string, podName string, commandAndArgs ...string) (string, error) { + // TODO: see if we can add a timeout or make this cancelable somehow request := s.kubeClient. CoreV1(). RESTClient(). diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 6fe0b212..2383fabc 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -119,16 +119,14 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { // Create informers. Don't forget to make sure they get started in the function returned below. informers := createInformers(c.ServerInstallationInfo.Namespace, client.Kubernetes, client.PinnipedConcierge) - // Configuration for the kubecertagent controllers created below. - agentPodConfig := &kubecertagent.AgentPodConfig{ + agentConfig := kubecertagent.AgentConfig{ Namespace: c.ServerInstallationInfo.Namespace, ContainerImage: *c.KubeCertAgentConfig.Image, - PodNamePrefix: *c.KubeCertAgentConfig.NamePrefix, + NamePrefix: *c.KubeCertAgentConfig.NamePrefix, ContainerImagePullSecrets: c.KubeCertAgentConfig.ImagePullSecrets, - AdditionalLabels: c.Labels, - } - credentialIssuerLocationConfig := &kubecertagent.CredentialIssuerLocationConfig{ - Name: c.NamesConfig.CredentialIssuer, + Labels: c.Labels, + CredentialIssuerName: c.NamesConfig.CredentialIssuer, + DiscoveryURLOverride: c.DiscoveryURLOverride, } // Create controller manager. @@ -195,64 +193,20 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { ), singletonWorker, ). - - // Kube cert agent controllers are responsible for finding the cluster's signing keys and keeping them + // The kube-cert-agent controller is responsible for finding the cluster's signing keys and keeping them // up to date in memory, as well as reporting status on this cluster integration strategy. WithController( - kubecertagent.NewCreaterController( - agentPodConfig, - credentialIssuerLocationConfig, - c.Labels, - clock.RealClock{}, - client.Kubernetes, - client.PinnipedConcierge, + kubecertagent.NewAgentController( + agentConfig, + client, informers.kubeSystemNamespaceK8s.Core().V1().Pods(), - informers.installationNamespaceK8s.Core().V1().Pods(), - controllerlib.WithInformer, - controllerlib.WithInitialEvent, - ), - singletonWorker, - ). - WithController( - kubecertagent.NewAnnotaterController( - agentPodConfig, - credentialIssuerLocationConfig, - c.Labels, - clock.RealClock{}, - client.Kubernetes, - client.PinnipedConcierge, - informers.kubeSystemNamespaceK8s.Core().V1().Pods(), - informers.installationNamespaceK8s.Core().V1().Pods(), - controllerlib.WithInformer, - ), - singletonWorker, - ). - WithController( - kubecertagent.NewExecerController( - credentialIssuerLocationConfig, - c.Labels, - c.DiscoveryURLOverride, - c.DynamicSigningCertProvider, - kubecertagent.NewPodCommandExecutor(client.JSONConfig, client.Kubernetes), - client.PinnipedConcierge, - clock.RealClock{}, + informers.installationNamespaceK8s.Apps().V1().Deployments(), informers.installationNamespaceK8s.Core().V1().Pods(), informers.kubePublicNamespaceK8s.Core().V1().ConfigMaps(), - controllerlib.WithInformer, + c.DynamicSigningCertProvider, ), singletonWorker, ). - WithController( - kubecertagent.NewDeleterController( - agentPodConfig, - client.Kubernetes, - informers.kubeSystemNamespaceK8s.Core().V1().Pods(), - informers.installationNamespaceK8s.Core().V1().Pods(), - controllerlib.WithInformer, - ), - singletonWorker, - ). - // The cache filler/cleaner controllers are responsible for keep an in-memory representation of active // authenticators up to date. WithController( diff --git a/test/integration/concierge_credentialissuer_test.go b/test/integration/concierge_credentialissuer_test.go index c146dcc2..3ba01a52 100644 --- a/test/integration/concierge_credentialissuer_test.go +++ b/test/integration/concierge_credentialissuer_test.go @@ -80,7 +80,7 @@ func TestCredentialIssuer(t *testing.T) { if env.HasCapability(library.ClusterSigningKeyIsAvailable) { require.Equal(t, configv1alpha1.SuccessStrategyStatus, actualStatusStrategy.Status) require.Equal(t, configv1alpha1.FetchedKeyStrategyReason, actualStatusStrategy.Reason) - require.Equal(t, "Key was fetched successfully", actualStatusStrategy.Message) + require.Equal(t, "key was fetched successfully", actualStatusStrategy.Message) require.NotNil(t, actualStatusStrategy.Frontend) require.Equal(t, configv1alpha1.TokenCredentialRequestAPIFrontendType, actualStatusStrategy.Frontend.Type) expectedTokenRequestAPIInfo := configv1alpha1.TokenCredentialRequestAPIInfo{ @@ -111,10 +111,7 @@ func TestCredentialIssuer(t *testing.T) { } else { require.Equal(t, configv1alpha1.ErrorStrategyStatus, actualStatusStrategy.Status) require.Equal(t, configv1alpha1.CouldNotFetchKeyStrategyReason, actualStatusStrategy.Reason) - require.Contains(t, actualStatusStrategy.Message, "did not find kube-controller-manager pod(s)") - // For now, don't verify the kube config info because its not available on GKE. We'll need to address - // this somehow once we starting supporting those cluster types. - // Require `nil` to remind us to address this later for other types of clusters where it is available. + require.Contains(t, actualStatusStrategy.Message, "could not find a healthy kube-controller-manager pod (0 candidates)") require.Nil(t, actualStatusKubeConfigInfo) } }) diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 0b0ae8ae..8fded736 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -6,169 +6,85 @@ package integration import ( "context" "fmt" - "sort" "testing" "time" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/diff" - "k8s.io/apimachinery/pkg/util/wait" conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" "go.pinniped.dev/test/library" ) -const ( - kubeCertAgentLabelSelector = "kube-cert-agent.pinniped.dev=true" -) - func TestKubeCertAgent(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) - - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - kubeClient := library.NewKubernetesClientset(t) + adminConciergeClient := library.NewConciergeClientset(t) - // Get the current number of kube-cert-agent pods. - // - // We can pretty safely assert there should be more than 1, since there should be a - // kube-cert-agent pod per kube-controller-manager pod, and there should probably be at least - // 1 kube-controller-manager for this to be a working kube API. - originalAgentPods, err := kubeClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: kubeCertAgentLabelSelector, - }) - require.NoError(t, err) - require.NotEmpty(t, originalAgentPods.Items) - sortPods(originalAgentPods) - - for _, agentPod := range originalAgentPods.Items { - // All agent pods should contain all custom labels - for k, v := range env.ConciergeCustomLabels { - require.Equalf(t, v, agentPod.Labels[k], "expected agent pod to have label `%s: %s`", k, v) - } - require.Equal(t, env.ConciergeAppName, agentPod.Labels["app"]) - } - - agentPodsReconciled := func() bool { - var currentAgentPods *corev1.PodList - currentAgentPods, err = kubeClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{ - LabelSelector: kubeCertAgentLabelSelector, - }) - - if err != nil { - return false - } - - if len(originalAgentPods.Items) != len(currentAgentPods.Items) { - err = fmt.Errorf( - "original agent pod len != current agent pod len: %s", - diff.ObjectDiff(originalAgentPods.Items, currentAgentPods.Items), - ) - return false - } - - sortPods(currentAgentPods) - for i := range originalAgentPods.Items { - if !equality.Semantic.DeepEqual( - originalAgentPods.Items[i].Spec, - currentAgentPods.Items[i].Spec, - ) { - err = fmt.Errorf( - "original agent pod != current agent pod: %s", - diff.ObjectDiff(originalAgentPods.Items[i].Spec, currentAgentPods.Items[i].Spec), - ) - return false - } - } - - return true - } - - t.Run("reconcile on update", func(t *testing.T) { - // Ensure that the next test will start from a known state. - defer ensureKubeCertAgentSteadyState(t, agentPodsReconciled) - - // Update the image of the first pod. The controller should see it, and flip it back. - // - // Note that we update the toleration field here because it is the only field, currently, that - // 1) we are allowed to update on a running pod AND 2) the kube-cert-agent controllers care - // about. - updatedAgentPod := originalAgentPods.Items[0].DeepCopy() - updatedAgentPod.Spec.Tolerations = append( - updatedAgentPod.Spec.Tolerations, - corev1.Toleration{Key: "fake-toleration"}, - ) - _, err = kubeClient.CoreV1().Pods(env.ConciergeNamespace).Update(ctx, updatedAgentPod, metav1.UpdateOptions{}) - require.NoError(t, err) - - // Make sure the original pods come back. - assert.Eventually(t, agentPodsReconciled, 10*time.Second, 250*time.Millisecond) - require.NoError(t, err) - }) - - t.Run("reconcile on delete", func(t *testing.T) { - // Ensure that the next test will start from a known state. - defer ensureKubeCertAgentSteadyState(t, agentPodsReconciled) - - // Delete the first pod. The controller should see it, and flip it back. - err = kubeClient. - CoreV1(). - Pods(env.ConciergeNamespace). - Delete(ctx, originalAgentPods.Items[0].Name, metav1.DeleteOptions{}) - require.NoError(t, err) - - // Make sure the original pods come back. - assert.Eventually(t, agentPodsReconciled, 10*time.Second, 250*time.Millisecond) - require.NoError(t, err) - }) - - // Because the above tests have purposefully put the kube cert issuer strategy into a broken - // state, wait for it to become healthy again before moving on to other integration tests, - // otherwise those tests would be polluted by this test and would have to wait for the - // strategy to become successful again. + // Expect there to be at least on healthy kube-cert-agent pod on this cluster. library.RequireEventuallyWithoutError(t, func() (bool, error) { - adminConciergeClient := library.NewConciergeClientset(t) - credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) - if err != nil || credentialIssuer.Status.Strategies == nil { - t.Log("Did not find any CredentialIssuer with any strategies") - return false, nil // didn't find it, but keep trying + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + agentPods, err := kubeClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{ + LabelSelector: "kube-cert-agent.pinniped.dev=v2", + }) + if err != nil { + return false, fmt.Errorf("failed to list pods: %w", err) } - for _, strategy := range credentialIssuer.Status.Strategies { - // There will be other strategy types in the list, so ignore those. - if strategy.Type == conciergev1alpha.KubeClusterSigningCertificateStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif - if strategy.Frontend == nil { - return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test - } - return true, nil // found it, continue the test! + for _, p := range agentPods.Items { + t.Logf("found agent pod %s/%s in phase %s", p.Namespace, p.Name, p.Status.Phase) + } + + for _, p := range agentPods.Items { + if p.Status.Phase == corev1.PodRunning { + return true, nil } } - t.Log("Did not find any successful KubeClusterSigningCertificate strategy on CredentialIssuer") - return false, nil // didn't find it, but keep trying - }, 3*time.Minute, 3*time.Second) -} + return false, nil + }, 1*time.Minute, 2*time.Second, "never saw a healthy kube-cert-agent Pod running") -func ensureKubeCertAgentSteadyState(t *testing.T, agentPodsReconciled func() bool) { - t.Helper() - - const wantSteadyStateSnapshots = 3 - var steadyStateSnapshots int - require.NoError(t, wait.Poll(250*time.Millisecond, 30*time.Second, func() (bool, error) { - if agentPodsReconciled() { - steadyStateSnapshots++ - } else { - steadyStateSnapshots = 0 + // Expect that the CredentialIssuer will have a healthy KubeClusterSigningCertificate strategy. + library.RequireEventuallyWithoutError(t, func() (bool, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) + if err != nil { + t.Logf("could not get the CredentialIssuer: %v", err) + return false, nil } - return steadyStateSnapshots == wantSteadyStateSnapshots, nil - })) + + // If there's no successful strategy yet, wait until there is. + strategy := findSuccessfulStrategy(credentialIssuer, conciergev1alpha.KubeClusterSigningCertificateStrategyType) + if strategy == nil { + t.Log("could not find a successful TokenCredentialRequestAPI strategy in the CredentialIssuer:") + for _, s := range credentialIssuer.Status.Strategies { + t.Logf(" strategy %s has status %s/%s: %s", s.Type, s.Status, s.Reason, s.Message) + } + return false, nil + } + + // The successful strategy must have a frontend of type TokenCredentialRequestAPI. + if strategy.Frontend == nil { + return false, fmt.Errorf("strategy did not find a Frontend") + } + if strategy.Frontend.Type != conciergev1alpha.TokenCredentialRequestAPIFrontendType { + return false, fmt.Errorf("strategy had unexpected frontend type %q", strategy.Frontend.Type) + } + return true, nil + }, 3*time.Minute, 2*time.Second) } -func sortPods(pods *corev1.PodList) { - sort.Slice(pods.Items, func(i, j int) bool { - return pods.Items[i].Name < pods.Items[j].Name - }) +func findSuccessfulStrategy(credentialIssuer *conciergev1alpha.CredentialIssuer, strategyType conciergev1alpha.StrategyType) *conciergev1alpha.CredentialIssuerStrategy { + for _, strategy := range credentialIssuer.Status.Strategies { + if strategy.Type != strategyType { + continue + } + if strategy.Status != conciergev1alpha.SuccessStrategyStatus { + continue + } + return &strategy + } + return nil } From 54a8297cc42aa540866c2484598e61c9e175ae84 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Fri, 16 Apr 2021 18:20:21 -0500 Subject: [PATCH 07/10] Add generated mocks for kubecertagent. Signed-off-by: Matt Moyer --- .../kubecertagent/mocks/generate.go | 7 + .../kubecertagent/mocks/mockdynamiccert.go | 132 ++++++++++++++++++ .../mocks/mockpodcommandexecutor.go | 58 ++++++++ 3 files changed, 197 insertions(+) create mode 100644 internal/controller/kubecertagent/mocks/generate.go create mode 100644 internal/controller/kubecertagent/mocks/mockdynamiccert.go create mode 100644 internal/controller/kubecertagent/mocks/mockpodcommandexecutor.go diff --git a/internal/controller/kubecertagent/mocks/generate.go b/internal/controller/kubecertagent/mocks/generate.go new file mode 100644 index 00000000..0b5128b6 --- /dev/null +++ b/internal/controller/kubecertagent/mocks/generate.go @@ -0,0 +1,7 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package mocks + +//go:generate go run -v github.com/golang/mock/mockgen -destination=mockpodcommandexecutor.go -package=mocks -copyright_file=../../../../hack/header.txt go.pinniped.dev/internal/controller/kubecertagent PodCommandExecutor +//go:generate go run -v github.com/golang/mock/mockgen -destination=mockdynamiccert.go -package=mocks -copyright_file=../../../../hack/header.txt -mock_names Private=MockDynamicCertPrivate go.pinniped.dev/internal/dynamiccert Private diff --git a/internal/controller/kubecertagent/mocks/mockdynamiccert.go b/internal/controller/kubecertagent/mocks/mockdynamiccert.go new file mode 100644 index 00000000..030d3e07 --- /dev/null +++ b/internal/controller/kubecertagent/mocks/mockdynamiccert.go @@ -0,0 +1,132 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: go.pinniped.dev/internal/dynamiccert (interfaces: Private) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + dynamiccertificates "k8s.io/apiserver/pkg/server/dynamiccertificates" +) + +// MockDynamicCertPrivate is a mock of Private interface. +type MockDynamicCertPrivate struct { + ctrl *gomock.Controller + recorder *MockDynamicCertPrivateMockRecorder +} + +// MockDynamicCertPrivateMockRecorder is the mock recorder for MockDynamicCertPrivate. +type MockDynamicCertPrivateMockRecorder struct { + mock *MockDynamicCertPrivate +} + +// NewMockDynamicCertPrivate creates a new mock instance. +func NewMockDynamicCertPrivate(ctrl *gomock.Controller) *MockDynamicCertPrivate { + mock := &MockDynamicCertPrivate{ctrl: ctrl} + mock.recorder = &MockDynamicCertPrivateMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDynamicCertPrivate) EXPECT() *MockDynamicCertPrivateMockRecorder { + return m.recorder +} + +// AddListener mocks base method. +func (m *MockDynamicCertPrivate) AddListener(arg0 dynamiccertificates.Listener) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "AddListener", arg0) +} + +// AddListener indicates an expected call of AddListener. +func (mr *MockDynamicCertPrivateMockRecorder) AddListener(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddListener", reflect.TypeOf((*MockDynamicCertPrivate)(nil).AddListener), arg0) +} + +// CurrentCertKeyContent mocks base method. +func (m *MockDynamicCertPrivate) CurrentCertKeyContent() ([]byte, []byte) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CurrentCertKeyContent") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].([]byte) + return ret0, ret1 +} + +// CurrentCertKeyContent indicates an expected call of CurrentCertKeyContent. +func (mr *MockDynamicCertPrivateMockRecorder) CurrentCertKeyContent() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CurrentCertKeyContent", reflect.TypeOf((*MockDynamicCertPrivate)(nil).CurrentCertKeyContent)) +} + +// Name mocks base method. +func (m *MockDynamicCertPrivate) Name() string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Name") + ret0, _ := ret[0].(string) + return ret0 +} + +// Name indicates an expected call of Name. +func (mr *MockDynamicCertPrivateMockRecorder) Name() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Name", reflect.TypeOf((*MockDynamicCertPrivate)(nil).Name)) +} + +// Run mocks base method. +func (m *MockDynamicCertPrivate) Run(arg0 int, arg1 <-chan struct{}) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Run", arg0, arg1) +} + +// Run indicates an expected call of Run. +func (mr *MockDynamicCertPrivateMockRecorder) Run(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Run", reflect.TypeOf((*MockDynamicCertPrivate)(nil).Run), arg0, arg1) +} + +// RunOnce mocks base method. +func (m *MockDynamicCertPrivate) RunOnce() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RunOnce") + ret0, _ := ret[0].(error) + return ret0 +} + +// RunOnce indicates an expected call of RunOnce. +func (mr *MockDynamicCertPrivateMockRecorder) RunOnce() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RunOnce", reflect.TypeOf((*MockDynamicCertPrivate)(nil).RunOnce)) +} + +// SetCertKeyContent mocks base method. +func (m *MockDynamicCertPrivate) SetCertKeyContent(arg0, arg1 []byte) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetCertKeyContent", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// SetCertKeyContent indicates an expected call of SetCertKeyContent. +func (mr *MockDynamicCertPrivateMockRecorder) SetCertKeyContent(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCertKeyContent", reflect.TypeOf((*MockDynamicCertPrivate)(nil).SetCertKeyContent), arg0, arg1) +} + +// UnsetCertKeyContent mocks base method. +func (m *MockDynamicCertPrivate) UnsetCertKeyContent() { + m.ctrl.T.Helper() + m.ctrl.Call(m, "UnsetCertKeyContent") +} + +// UnsetCertKeyContent indicates an expected call of UnsetCertKeyContent. +func (mr *MockDynamicCertPrivateMockRecorder) UnsetCertKeyContent() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UnsetCertKeyContent", reflect.TypeOf((*MockDynamicCertPrivate)(nil).UnsetCertKeyContent)) +} diff --git a/internal/controller/kubecertagent/mocks/mockpodcommandexecutor.go b/internal/controller/kubecertagent/mocks/mockpodcommandexecutor.go new file mode 100644 index 00000000..637f0927 --- /dev/null +++ b/internal/controller/kubecertagent/mocks/mockpodcommandexecutor.go @@ -0,0 +1,58 @@ +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: go.pinniped.dev/internal/controller/kubecertagent (interfaces: PodCommandExecutor) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockPodCommandExecutor is a mock of PodCommandExecutor interface. +type MockPodCommandExecutor struct { + ctrl *gomock.Controller + recorder *MockPodCommandExecutorMockRecorder +} + +// MockPodCommandExecutorMockRecorder is the mock recorder for MockPodCommandExecutor. +type MockPodCommandExecutorMockRecorder struct { + mock *MockPodCommandExecutor +} + +// NewMockPodCommandExecutor creates a new mock instance. +func NewMockPodCommandExecutor(ctrl *gomock.Controller) *MockPodCommandExecutor { + mock := &MockPodCommandExecutor{ctrl: ctrl} + mock.recorder = &MockPodCommandExecutorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockPodCommandExecutor) EXPECT() *MockPodCommandExecutorMockRecorder { + return m.recorder +} + +// Exec mocks base method. +func (m *MockPodCommandExecutor) Exec(arg0, arg1 string, arg2 ...string) (string, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Exec", varargs...) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Exec indicates an expected call of Exec. +func (mr *MockPodCommandExecutorMockRecorder) Exec(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Exec", reflect.TypeOf((*MockPodCommandExecutor)(nil).Exec), varargs...) +} From e532a88647e00d6cd2f8a19966fced17542baad1 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Tue, 20 Apr 2021 14:56:43 -0500 Subject: [PATCH 08/10] Add a new "legacy pod cleaner" controller. This controller is responsible for cleaning up kube-cert-agent pods that were deployed by previous versions. They are easily identified because they use a different `kube-cert-agent.pinniped.dev` label compared to the new agent pods (`true` vs. `v2`). Signed-off-by: Matt Moyer --- deploy/concierge/rbac.yaml | 4 + .../kubecertagent/legacypodcleaner.go | 63 ++++++++ .../kubecertagent/legacypodcleaner_test.go | 145 ++++++++++++++++++ .../controllermanager/prepare_controllers.go | 11 ++ .../concierge_kubecertagent_test.go | 58 +++++++ 5 files changed, 281 insertions(+) create mode 100644 internal/controller/kubecertagent/legacypodcleaner.go create mode 100644 internal/controller/kubecertagent/legacypodcleaner_test.go diff --git a/deploy/concierge/rbac.yaml b/deploy/concierge/rbac.yaml index 6bd56fe0..74e5653e 100644 --- a/deploy/concierge/rbac.yaml +++ b/deploy/concierge/rbac.yaml @@ -90,6 +90,10 @@ rules: - apiGroups: [ "" ] resources: [ pods/exec ] verbs: [ create ] + #! We need to be able to delete pods in our namespace so we can clean up legacy kube-cert-agent pods. + - apiGroups: [ "" ] + resources: [ pods ] + verbs: [ delete ] #! We need to be able to create and update deployments in our namespace so we can manage the kube-cert-agent Deployment. - apiGroups: [ apps ] resources: [ deployments ] diff --git a/internal/controller/kubecertagent/legacypodcleaner.go b/internal/controller/kubecertagent/legacypodcleaner.go new file mode 100644 index 00000000..1a44477e --- /dev/null +++ b/internal/controller/kubecertagent/legacypodcleaner.go @@ -0,0 +1,63 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package kubecertagent + +import ( + "fmt" + + "github.com/go-logr/logr" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/klog/v2" + + pinnipedcontroller "go.pinniped.dev/internal/controller" + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/kubeclient" +) + +// NewLegacyPodCleanerController returns a controller that cleans up legacy kube-cert-agent Pods created by Pinniped v0.7.0 and below. +func NewLegacyPodCleanerController( + cfg AgentConfig, + client *kubeclient.Client, + agentPods corev1informers.PodInformer, + log logr.Logger, + options ...controllerlib.Option, +) controllerlib.Controller { + // legacyAgentLabels are the Kubernetes labels we previously added to agent pods (the new value is "v2"). + // We also expect these pods to have the "extra" labels configured on the Concierge. + legacyAgentLabels := map[string]string{"kube-cert-agent.pinniped.dev": "true"} + for k, v := range cfg.Labels { + legacyAgentLabels[k] = v + } + legacyAgentSelector := labels.SelectorFromSet(legacyAgentLabels) + + log = log.WithName("legacy-pod-cleaner-controller") + + return controllerlib.New( + controllerlib.Config{ + Name: "legacy-pod-cleaner-controller", + Syncer: controllerlib.SyncFunc(func(ctx controllerlib.Context) error { + if err := client.Kubernetes.CoreV1().Pods(ctx.Key.Namespace).Delete(ctx.Context, ctx.Key.Name, metav1.DeleteOptions{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return fmt.Errorf("could not delete legacy agent pod: %w", err) + } + log.Info("deleted legacy kube-cert-agent pod", "pod", klog.KRef(ctx.Key.Namespace, ctx.Key.Name)) + return nil + }), + }, + append([]controllerlib.Option{ + controllerlib.WithInformer( + agentPods, + pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { + return obj.GetNamespace() == cfg.Namespace && legacyAgentSelector.Matches(labels.Set(obj.GetLabels())) + }, nil), + controllerlib.InformerOption{}, + ), + }, options...)..., + ) +} diff --git a/internal/controller/kubecertagent/legacypodcleaner_test.go b/internal/controller/kubecertagent/legacypodcleaner_test.go new file mode 100644 index 00000000..b2b1a5e7 --- /dev/null +++ b/internal/controller/kubecertagent/legacypodcleaner_test.go @@ -0,0 +1,145 @@ +// Copyright 2021 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package kubecertagent + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/informers" + kubefake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + + "go.pinniped.dev/internal/controllerlib" + "go.pinniped.dev/internal/kubeclient" + "go.pinniped.dev/internal/testutil/testlogger" +) + +func TestLegacyPodCleanerController(t *testing.T) { + t.Parallel() + + legacyAgentPodWithoutExtraLabel := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "concierge", + Name: "pinniped-concierge-kube-cert-agent-without-extra-label", + Labels: map[string]string{"kube-cert-agent.pinniped.dev": "true"}, + }, + Spec: corev1.PodSpec{}, + Status: corev1.PodStatus{Phase: corev1.PodRunning}, + } + + legacyAgentPodWithExtraLabel := legacyAgentPodWithoutExtraLabel.DeepCopy() + legacyAgentPodWithExtraLabel.Name = "pinniped-concierge-kube-cert-agent-with-extra-label" + legacyAgentPodWithExtraLabel.Labels["extralabel"] = "labelvalue" + legacyAgentPodWithExtraLabel.Labels["anotherextralabel"] = "labelvalue" + + nonLegacyAgentPod := legacyAgentPodWithExtraLabel.DeepCopy() + nonLegacyAgentPod.Name = "pinniped-concierge-kube-cert-agent-not-legacy" + nonLegacyAgentPod.Labels["kube-cert-agent.pinniped.dev"] = "v2" + + tests := []struct { + name string + kubeObjects []runtime.Object + addKubeReactions func(*kubefake.Clientset) + wantDistinctErrors []string + wantDistinctLogs []string + wantActions []coretesting.Action + }{ + { + name: "no pods", + wantActions: []coretesting.Action{}, + }, + { + name: "mix of pods", + kubeObjects: []runtime.Object{ + legacyAgentPodWithoutExtraLabel, // should not be delete (missing extra label) + legacyAgentPodWithExtraLabel, // should be deleted + nonLegacyAgentPod, // should not be deleted (missing legacy agent label) + }, + wantDistinctErrors: []string{""}, + wantDistinctLogs: []string{ + `legacy-pod-cleaner-controller "level"=0 "msg"="deleted legacy kube-cert-agent pod" "pod"={"name":"pinniped-concierge-kube-cert-agent-with-extra-label","namespace":"concierge"}`, + }, + wantActions: []coretesting.Action{ // the first delete triggers the informer again, but the second invocation triggers a Not Found + coretesting.NewDeleteAction(corev1.Resource("pods").WithVersion("v1"), "concierge", legacyAgentPodWithExtraLabel.Name), + coretesting.NewDeleteAction(corev1.Resource("pods").WithVersion("v1"), "concierge", legacyAgentPodWithExtraLabel.Name), + }, + }, + { + name: "fail to delete", + kubeObjects: []runtime.Object{ + legacyAgentPodWithoutExtraLabel, // should not be delete (missing extra label) + legacyAgentPodWithExtraLabel, // should be deleted + nonLegacyAgentPod, // should not be deleted (missing legacy agent label) + }, + addKubeReactions: func(clientset *kubefake.Clientset) { + clientset.PrependReactor("delete", "*", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, fmt.Errorf("some delete error") + }) + }, + wantDistinctErrors: []string{ + "could not delete legacy agent pod: some delete error", + }, + wantActions: []coretesting.Action{ + coretesting.NewDeleteAction(corev1.Resource("pods").WithVersion("v1"), "concierge", legacyAgentPodWithExtraLabel.Name), + coretesting.NewDeleteAction(corev1.Resource("pods").WithVersion("v1"), "concierge", legacyAgentPodWithExtraLabel.Name), + }, + }, + { + name: "fail to delete because of not found error", + kubeObjects: []runtime.Object{ + legacyAgentPodWithoutExtraLabel, // should not be delete (missing extra label) + legacyAgentPodWithExtraLabel, // should be deleted + nonLegacyAgentPod, // should not be deleted (missing legacy agent label) + }, + addKubeReactions: func(clientset *kubefake.Clientset) { + clientset.PrependReactor("delete", "*", func(action coretesting.Action) (handled bool, ret runtime.Object, err error) { + return true, nil, k8serrors.NewNotFound(action.GetResource().GroupResource(), "") + }) + }, + wantDistinctErrors: []string{""}, + wantActions: []coretesting.Action{ + coretesting.NewDeleteAction(corev1.Resource("pods").WithVersion("v1"), "concierge", legacyAgentPodWithExtraLabel.Name), + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + kubeClientset := kubefake.NewSimpleClientset(tt.kubeObjects...) + if tt.addKubeReactions != nil { + tt.addKubeReactions(kubeClientset) + } + kubeInformers := informers.NewSharedInformerFactory(kubeClientset, 0) + log := testlogger.New(t) + controller := NewLegacyPodCleanerController( + AgentConfig{ + Namespace: "concierge", + Labels: map[string]string{"extralabel": "labelvalue"}, + }, + &kubeclient.Client{Kubernetes: kubeClientset}, + kubeInformers.Core().V1().Pods(), + log, + controllerlib.WithMaxRetries(1), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + errorMessages := runControllerUntilQuiet(ctx, t, controller, kubeInformers) + assert.Equal(t, tt.wantDistinctErrors, deduplicate(errorMessages), "unexpected errors") + assert.Equal(t, tt.wantDistinctLogs, deduplicate(log.Lines()), "unexpected logs") + assert.Equal(t, tt.wantActions, kubeClientset.Actions()[2:], "unexpected actions") + }) + } +} diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 2383fabc..8990a4ec 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -207,6 +207,17 @@ func PrepareControllers(c *Config) (func(ctx context.Context), error) { ), singletonWorker, ). + // The kube-cert-agent legacy pod cleaner controller is responsible for cleaning up pods that were deployed by + // versions of Pinniped prior to v0.7.0. If we stop supporting upgrades from v0.7.0, we can safely remove this. + WithController( + kubecertagent.NewLegacyPodCleanerController( + agentConfig, + client, + informers.installationNamespaceK8s.Core().V1().Pods(), + klogr.New(), + ), + singletonWorker, + ). // The cache filler/cleaner controllers are responsible for keep an in-memory representation of active // authenticators up to date. WithController( diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 8fded736..4b46ef05 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -9,8 +9,12 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/utils/pointer" conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" "go.pinniped.dev/test/library" @@ -88,3 +92,57 @@ func findSuccessfulStrategy(credentialIssuer *conciergev1alpha.CredentialIssuer, } return nil } + +func TestLegacyPodCleaner(t *testing.T) { + env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + kubeClient := library.NewKubernetesClientset(t) + + // Pick the same labels that the legacy code would have used to run the kube-cert-agent pod. + legacyAgentLabels := map[string]string{} + for k, v := range env.ConciergeCustomLabels { + legacyAgentLabels[k] = v + } + legacyAgentLabels["app"] = env.ConciergeAppName + legacyAgentLabels["kube-cert-agent.pinniped.dev"] = "true" + legacyAgentLabels["pinniped.dev/test"] = "" + + // Deploy a fake legacy agent pod using those labels. + pod, err := kubeClient.CoreV1().Pods(env.ConciergeNamespace).Create(ctx, &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-legacy-kube-cert-agent-", + Labels: legacyAgentLabels, + Annotations: map[string]string{"pinniped.dev/testName": t.Name()}, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "sleeper", + Image: "debian:10.9-slim", + Command: []string{"/bin/sleep", "infinity"}, + }}, + }, + }, metav1.CreateOptions{}) + require.NoError(t, err, "failed to create fake legacy agent pod") + t.Logf("deployed fake legacy agent pod %s/%s with labels %s", pod.Namespace, pod.Name, labels.SelectorFromSet(legacyAgentLabels).String()) + + // No matter what happens, clean up the agent pod at the end of the test (normally it will already have been deleted). + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) + defer cancel() + err := kubeClient.CoreV1().Pods(pod.Namespace).Delete(ctx, pod.Name, metav1.DeleteOptions{GracePeriodSeconds: pointer.Int64Ptr(0)}) + if !k8serrors.IsNotFound(err) { + require.NoError(t, err, "failed to clean up fake legacy agent pod") + } + }) + + // Expect the legacy-pod-cleaner controller to delete the pod. + library.RequireEventuallyWithoutError(t, func() (bool, error) { + _, err := kubeClient.CoreV1().Pods(pod.Namespace).Get(ctx, pod.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + t.Logf("fake legacy agent pod %s/%s was deleted as expected", pod.Namespace, pod.Name) + return true, nil + } + return false, err + }, 60*time.Second, 1*time.Second) +} From a52872cd03a316131830ecfb00bbd670ed96fb47 Mon Sep 17 00:00:00 2001 From: Matt Moyer Date: Mon, 26 Apr 2021 08:17:36 -0600 Subject: [PATCH 09/10] Fix a broken docs link in our README. Signed-off-by: Matt Moyer --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc09ff98..ca7df8dd 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ To learn more, see [architecture](https://pinniped.dev/docs/background/architect ## Getting started with Pinniped -Care to kick the tires? It's easy to [install and try Pinniped](https://pinniped.dev/docs/demo/). +Care to kick the tires? It's easy to [install and try Pinniped](https://pinniped.dev/docs/). ## Community meetings From 67a568811a1b61260c2aad797d87c05c30e48fbb Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 27 Apr 2021 10:10:02 -0700 Subject: [PATCH 10/10] Make prepare-for-integration-tests.sh work on linux too - The linux base64 command is different, so avoid using it at all. On linux the default is to split the output into multiple lines, which messes up the integration-test-env file. The flag used to disable this behavior on linux ("-w0") does not exist on MacOS's base64. - On debian linux, the latest version of Docker from apt-get still requires DOCKER_BUILDKIT=1 or else it barfs. --- hack/prepare-for-integration-tests.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/hack/prepare-for-integration-tests.sh b/hack/prepare-for-integration-tests.sh index e2909d2a..0313b60a 100755 --- a/hack/prepare-for-integration-tests.sh +++ b/hack/prepare-for-integration-tests.sh @@ -187,7 +187,8 @@ registry_repo_tag="${registry_repo}:${tag}" if [[ "$do_build" == "yes" ]]; then # Rebuild the code log_note "Docker building the app..." - docker build . --tag "$registry_repo_tag" + # DOCKER_BUILDKIT=1 is optional on MacOS but required on linux. + DOCKER_BUILDKIT=1 docker build . --tag "$registry_repo_tag" fi # Load it into the cluster @@ -300,8 +301,9 @@ popd >/dev/null # # Download the test CA bundle that was generated in the Dex pod. +# Note that this returns a base64 encoded value. # -test_ca_bundle_pem="$(kubectl get secrets -n tools certs -o go-template='{{index .data "ca.pem" | base64decode}}')" +test_ca_bundle_pem="$(kubectl get secrets -n tools certs -o go-template='{{index .data "ca.pem"}}')" # # Create the environment file. @@ -331,7 +333,7 @@ export PINNIPED_TEST_SUPERVISOR_HTTP_ADDRESS="127.0.0.1:12345" export PINNIPED_TEST_SUPERVISOR_HTTPS_ADDRESS="localhost:12344" export PINNIPED_TEST_PROXY=http://127.0.0.1:12346 export PINNIPED_TEST_LDAP_HOST=ldap.tools.svc.cluster.local -export PINNIPED_TEST_LDAP_LDAPS_CA_BUNDLE=$(echo "${test_ca_bundle_pem}" | base64 ) +export PINNIPED_TEST_LDAP_LDAPS_CA_BUNDLE="${test_ca_bundle_pem}" export PINNIPED_TEST_LDAP_BIND_ACCOUNT_USERNAME="cn=admin,dc=pinniped,dc=dev" export PINNIPED_TEST_LDAP_BIND_ACCOUNT_PASSWORD=password export PINNIPED_TEST_LDAP_USERS_SEARCH_BASE="ou=users,dc=pinniped,dc=dev" @@ -348,13 +350,13 @@ export PINNIPED_TEST_LDAP_EXPECTED_INDIRECT_GROUPS_DN="cn=pinnipeds,ou=groups,dc export PINNIPED_TEST_LDAP_EXPECTED_DIRECT_GROUPS_CN="ball-game-players;seals" export PINNIPED_TEST_LDAP_EXPECTED_INDIRECT_GROUPS_CN="pinnipeds;mammals" export PINNIPED_TEST_CLI_OIDC_ISSUER=https://dex.tools.svc.cluster.local/dex -export PINNIPED_TEST_CLI_OIDC_ISSUER_CA_BUNDLE=$(echo "${test_ca_bundle_pem}" | base64 ) +export PINNIPED_TEST_CLI_OIDC_ISSUER_CA_BUNDLE="${test_ca_bundle_pem}" export PINNIPED_TEST_CLI_OIDC_CLIENT_ID=pinniped-cli export PINNIPED_TEST_CLI_OIDC_CALLBACK_URL=http://127.0.0.1:48095/callback export PINNIPED_TEST_CLI_OIDC_USERNAME=pinny@example.com export PINNIPED_TEST_CLI_OIDC_PASSWORD=${dex_test_password} export PINNIPED_TEST_SUPERVISOR_UPSTREAM_OIDC_ISSUER=https://dex.tools.svc.cluster.local/dex -export PINNIPED_TEST_SUPERVISOR_UPSTREAM_OIDC_ISSUER_CA_BUNDLE=$(echo "${test_ca_bundle_pem}" | base64 ) +export PINNIPED_TEST_SUPERVISOR_UPSTREAM_OIDC_ISSUER_CA_BUNDLE="${test_ca_bundle_pem}" export PINNIPED_TEST_SUPERVISOR_UPSTREAM_OIDC_ADDITIONAL_SCOPES=email export PINNIPED_TEST_SUPERVISOR_UPSTREAM_OIDC_USERNAME_CLAIM=email export PINNIPED_TEST_SUPERVISOR_UPSTREAM_OIDC_GROUPS_CLAIM=groups