// Copyright 2020-2022 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package integration import ( "bytes" "context" "crypto/ecdsa" "crypto/elliptic" "crypto/rand" "crypto/x509" "crypto/x509/pkix" "encoding/base64" "encoding/json" "encoding/pem" "fmt" "io" "net" "net/http" "net/url" "os" "os/exec" "path/filepath" "sort" "strings" "sync" "testing" "time" "github.com/gorilla/websocket" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/net/http2" authenticationv1 "k8s.io/api/authentication/v1" authorizationv1 "k8s.io/api/authorization/v1" certificatesv1 "k8s.io/api/certificates/v1" certificatesv1beta1 "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/watch" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/request/bearertoken" "k8s.io/apiserver/pkg/authentication/serviceaccount" "k8s.io/apiserver/pkg/authentication/user" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/transport" "k8s.io/client-go/util/cert" "k8s.io/client-go/util/certificate/csr" "k8s.io/client-go/util/keyutil" "k8s.io/client-go/util/retry" "k8s.io/utils/pointer" conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1" identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1" loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1" pinnipedconciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned" "go.pinniped.dev/internal/crypto/ptls" "go.pinniped.dev/internal/httputil/roundtripper" "go.pinniped.dev/internal/kubeclient" "go.pinniped.dev/internal/testutil" "go.pinniped.dev/test/testlib" ) // syncBuffer wraps bytes.Buffer with a mutex so we don't have races in our test code. type syncBuffer struct { buf bytes.Buffer mu sync.Mutex } func (sb *syncBuffer) String() string { sb.mu.Lock() defer sb.mu.Unlock() return sb.buf.String() } func (sb *syncBuffer) Read(b []byte) (int, error) { sb.mu.Lock() defer sb.mu.Unlock() return sb.buf.Read(b) } func (sb *syncBuffer) Write(b []byte) (int, error) { sb.mu.Lock() defer sb.mu.Unlock() return sb.buf.Write(b) } // Note that this test supports being run on all of our integration test cluster types: // - TKGS acceptance (long-running) cluster: auto mode will choose disabled, supports LBs, does not have squid. // - GKE acceptance (long-running) cluster: auto will choose enabled, support LBs, does not have squid. // - kind: auto mode will choose disabled, does not support LBs, has squid. // - GKE ephemeral clusters: auto mode will choose enabled, supports LBs, has squid. // - AKS ephemeral clusters: auto mode will choose enabled, supports LBs, has squid. // - EKS ephemeral clusters: auto mode will choose enabled, supports LBs, has squid. func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's complex. env := testlib.IntegrationEnv(t) impersonatorShouldHaveStartedAutomaticallyByDefault := !env.HasCapability(testlib.ClusterSigningKeyIsAvailable) clusterSupportsLoadBalancers := env.HasCapability(testlib.HasExternalLoadBalancerProvider) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) defer cancel() // Create a client using the admin kubeconfig. adminClient := testlib.NewKubernetesClientset(t) adminConciergeClient := testlib.NewConciergeClientset(t) // Create a WebhookAuthenticator and prepare a TokenCredentialRequestSpec using the authenticator for use later. credentialRequestSpecWithWorkingCredentials := loginv1alpha1.TokenCredentialRequestSpec{ Token: env.TestUser.Token, Authenticator: testlib.CreateTestWebhookAuthenticator(ctx, t), } // The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer). proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace) var ( mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest mostRecentTokenCredentialRequestResponseLock sync.Mutex ) refreshCredentialHelper := func(t *testing.T, client pinnipedconciergeclientset.Interface) *loginv1alpha1.ClusterCredential { t.Helper() mostRecentTokenCredentialRequestResponseLock.Lock() defer mostRecentTokenCredentialRequestResponseLock.Unlock() if mostRecentTokenCredentialRequestResponse == nil || credentialAlmostExpired(t, mostRecentTokenCredentialRequestResponse) { // Make a TokenCredentialRequest. This can either return a cert signed by the Kube API server's CA (e.g. on kind) // or a cert signed by the impersonator's signing CA (e.g. on GKE). Either should be accepted by the impersonation // proxy server as a valid authentication. // // However, we issue short-lived certs, so this cert will only be valid for a few minutes. // Cache it until it is almost expired and then refresh it whenever it is close to expired. // testlib.RequireEventually(t, func(requireEventually *require.Assertions) { resp, err := createTokenCredentialRequest(credentialRequestSpecWithWorkingCredentials, client) requireEventually.NoError(err) requireEventually.NotNil(resp) requireEventually.NotNil(resp.Status) requireEventually.NotNil(resp.Status.Credential) requireEventually.Nilf(resp.Status.Message, "expected no error message but got: %s", testlib.Sdump(resp.Status.Message)) requireEventually.NotEmpty(resp.Status.Credential.ClientCertificateData) requireEventually.NotEmpty(resp.Status.Credential.ClientKeyData) // At the moment the credential request should not have returned a token. In the future, if we make it return // tokens, we should revisit this test's rest config below. requireEventually.Empty(resp.Status.Credential.Token) mostRecentTokenCredentialRequestResponse = resp }, 5*time.Minute, 5*time.Second) } return mostRecentTokenCredentialRequestResponse.Status.Credential } refreshCredential := func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential { // Use an anonymous client which goes through the impersonation proxy to make the request because that's // what would normally happen when a user is using a kubeconfig where the server is the impersonation proxy, // so it more closely simulates the normal use case, and also because we want this to work on AKS clusters // which do not allow anonymous requests. client := newAnonymousImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge return refreshCredentialHelper(t, client) } oldCredentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) require.NoError(t, err) // At the end of the test, clean up the CredentialIssuer t.Cleanup(func() { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() // Delete any version that was created by this test. t.Logf("cleaning up credentialissuer at end of test %s", credentialIssuerName(env)) err = retry.RetryOnConflict(retry.DefaultRetry, func() error { newCredentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) if err != nil { return err } oldCredentialIssuer.Spec.DeepCopyInto(&newCredentialIssuer.Spec) _, err = adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Update(ctx, newCredentialIssuer, metav1.UpdateOptions{}) return err }) require.NoError(t, err) // If we are running on an environment that has a load balancer, expect that the // CredentialIssuer will be updated eventually with a successful impersonation proxy frontend. // We do this to ensure that future tests that use the impersonation proxy (e.g., // TestE2EFullIntegration) will start with a known-good state. if clusterSupportsLoadBalancers { performImpersonatorDiscovery(ctx, t, env, adminClient, adminConciergeClient, refreshCredential) } }) // Done with set-up and ready to get started with the test. There are several states that we could be in at // this point depending on the capabilities of the cluster under test. We handle each possible case here. switch { case impersonatorShouldHaveStartedAutomaticallyByDefault && clusterSupportsLoadBalancers: // configure the credential issuer spec to have the impersonation proxy in auto mode updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{ ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{ Mode: conciergev1alpha.ImpersonationProxyModeAuto, Service: conciergev1alpha.ImpersonationProxyServiceSpec{ Type: conciergev1alpha.ImpersonationProxyServiceTypeLoadBalancer, Annotations: map[string]string{ "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "4000", }, }, }, }) // Auto mode should have decided that the impersonator will run and should have started a load balancer, // and we will be able to use the load balancer to access the impersonator. (e.g. GKE, AKS, EKS) // Check that load balancer has been automatically created by the impersonator's "auto" mode. testlib.RequireEventuallyWithoutError(t, func() (bool, error) { return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) }, 30*time.Second, 500*time.Millisecond) case impersonatorShouldHaveStartedAutomaticallyByDefault && !clusterSupportsLoadBalancers: t.Fatal("None of the clusters types that we currently test against should automatically" + "enable the impersonation proxy without also supporting load balancers. If we add such a" + "cluster type in the future then we should enhance this test.") case !impersonatorShouldHaveStartedAutomaticallyByDefault && clusterSupportsLoadBalancers: // Auto mode should have decided that the impersonator will be disabled. We need to manually enable it. // The cluster supports load balancers so we should enable it and let the impersonator create a load balancer // automatically. (e.g. TKGS) // The CredentialIssuer's strategies array should have been updated to include an unsuccessful impersonation // strategy saying that it was automatically disabled. requireDisabledStrategy(ctx, t, env, adminConciergeClient) // Create configuration to make the impersonation proxy turn on with no endpoint (i.e. automatically create a load balancer). updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{ ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{ Mode: conciergev1alpha.ImpersonationProxyModeEnabled, }, }) default: // Auto mode should have decided that the impersonator will be disabled. We need to manually enable it. // However, the cluster does not support load balancers so we should enable it without a load balancer // and use squid to make requests. (e.g. kind) if env.Proxy == "" { t.Skip("test cluster does not support load balancers but also doesn't have a squid proxy... " + "this is not a supported configuration for test clusters") } // Check that no load balancer has been created by the impersonator's "auto" mode. testlib.RequireNeverWithoutError(t, func() (bool, error) { return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) }, 10*time.Second, 500*time.Millisecond) // Check that we can't use the impersonation proxy to execute kubectl commands yet. _, err = impersonationProxyViaSquidKubeClientWithoutCredential(t, proxyServiceEndpoint).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) isErr, message := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) require.Truef(t, isErr, "wanted error %q to be service unavailable via squid error, but: %s", err, message) // Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer). updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{ ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{ Mode: conciergev1alpha.ImpersonationProxyModeEnabled, ExternalEndpoint: proxyServiceEndpoint, }, }) } // At this point the impersonator should be starting/running. When it is ready, the CredentialIssuer's // strategies array should be updated to include a successful impersonation strategy which can be used // to discover the impersonator's URL and CA certificate. Until it has finished starting, it may not be included // in the strategies array or it may be included in an error state. It can be in an error state for // awhile when it is waiting for the load balancer to be assigned an ip/hostname. impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminClient, adminConciergeClient, refreshCredential) if !clusterSupportsLoadBalancers { // In this case, we specified the endpoint in the configmap, so check that it was reported correctly in the CredentialIssuer. require.Equal(t, "https://"+proxyServiceEndpoint, impersonationProxyURL) } else { // If the impersonationProxyURL is a hostname, make sure DNS will resolve before we move on. ensureDNSResolves(t, impersonationProxyURL) } // Because our credentials expire so quickly, we'll always use a new client, to give us a chance to refresh our // credentials before they expire. Create a closure to capture the arguments to newImpersonationProxyClient // so we don't have to keep repeating them. // This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly. impersonationProxyKubeClient := func(t *testing.T) kubernetes.Interface { return newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil, refreshCredential).Kubernetes } t.Run("positive tests", func(t *testing.T) { // Create an RBAC rule to allow this user to read/write everything. testlib.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, ) // Wait for the above RBAC rule to take effect. testlib.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &authorizationv1.ResourceAttributes{ Verb: "get", Group: "", Version: "v1", Resource: "namespaces", }) // Get pods in supervisor namespace and pick one. // this is for tests that require performing actions against a running pod. // We use the supervisor pod because we already have it handy and need to port-forward a running port. // We avoid using the concierge for this because it requires TLS 1.3 which is not support by older versions of curl. supervisorPods, err := adminClient.CoreV1().Pods(env.SupervisorNamespace).List(ctx, metav1.ListOptions{LabelSelector: "deployment.pinniped.dev=supervisor"}) require.NoError(t, err) require.NotEmpty(t, supervisorPods.Items, "could not find supervisor pods") supervisorPod := supervisorPods.Items[0] // Test that the user can perform basic actions through the client with their username and group membership // influencing RBAC checks correctly. t.Run( "access as user", testlib.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyKubeClient(t)), ) for _, group := range env.TestUser.ExpectedGroups { group := group t.Run( "access as group "+group, testlib.AccessAsGroupTest(ctx, group, impersonationProxyKubeClient(t)), ) } if env.KubernetesDistribution == testlib.EKSDistro { t.Log("eks: sleeping for 10 minutes to allow DNS propagation") time.Sleep(10 * time.Minute) } t.Run("kubectl port-forward and keeping the connection open for over a minute (non-idle)", func(t *testing.T) { parallelIfNotEKS(t) kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator) // Run the kubectl port-forward command. timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", supervisorPod.Namespace, supervisorPod.Name, "10443:8443") portForwardCmd.Env = envVarsWithProxy // Start, but don't wait for the command to finish. err := portForwardCmd.Start() require.NoError(t, err, `"kubectl port-forward" failed`) go func() { assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() // The server should recognize this this // is going to be a long-running command and keep the connection open as long as the client stays connected. // curl the endpoint as many times as we can within 70 seconds. // this will ensure that we don't run into idle timeouts. var curlStdOut, curlStdErr bytes.Buffer timeout, cancelFunc = context.WithTimeout(ctx, 75*time.Second) defer cancelFunc() startTime := time.Now() for time.Now().Before(startTime.Add(70 * time.Second)) { curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10443/healthz") // -sS turns off the progressbar but still prints errors curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr curlErr := curlCmd.Run() if curlErr != nil { t.Log("curl error: " + curlErr.Error()) t.Log("curlStdErr: " + curlStdErr.String()) t.Log("stdout: " + curlStdOut.String()) } t.Log("Running curl through the kubectl port-forward port for 70 seconds. Elapsed time:", time.Since(startTime)) time.Sleep(1 * time.Second) } // curl the endpoint once more, once 70 seconds has elapsed, to make sure the connection is still open. timeout, cancelFunc = context.WithTimeout(ctx, 30*time.Second) defer cancelFunc() curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10443/healthz") // -sS turns off the progressbar but still prints errors curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr curlErr := curlCmd.Run() if curlErr != nil { t.Log("curl error: " + curlErr.Error()) t.Log("curlStdErr: " + curlStdErr.String()) t.Log("stdout: " + curlStdOut.String()) } require.NoError(t, curlErr) require.Contains(t, curlStdOut.String(), "okokokokok") // a few successful healthz responses }) t.Run("kubectl port-forward and keeping the connection open for over a minute (idle)", func(t *testing.T) { parallelIfNotEKS(t) kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator) // Run the kubectl port-forward command. timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", supervisorPod.Namespace, supervisorPod.Name, "10444:8443") portForwardCmd.Env = envVarsWithProxy // Start, but don't wait for the command to finish. err := portForwardCmd.Start() require.NoError(t, err, `"kubectl port-forward" failed`) go func() { assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String()) }() // Wait to see if we time out. The default timeout is 60 seconds, but the server should recognize that this // is going to be a long-running command and keep the connection open as long as the client stays connected. time.Sleep(70 * time.Second) timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10444/healthz") // -sS turns off the progressbar but still prints errors var curlStdOut, curlStdErr bytes.Buffer curlCmd.Stdout = &curlStdOut curlCmd.Stderr = &curlStdErr err = curlCmd.Run() if err != nil { t.Log("curl error: " + err.Error()) t.Log("curlStdErr: " + curlStdErr.String()) t.Log("stdout: " + curlStdOut.String()) } require.NoError(t, err) require.Equal(t, curlStdOut.String(), "ok") }) t.Run("using and watching all the basic verbs", func(t *testing.T) { parallelIfNotEKS(t) // Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace. namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name // Create and start informer to exercise the "watch" verb for us. informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions( impersonationProxyKubeClient(t), 0, k8sinformers.WithNamespace(namespaceName)) informer := informerFactory.Core().V1().ConfigMaps() informer.Informer() // makes sure that the informer will cache stopChannel := make(chan struct{}) informerFactory.Start(stopChannel) t.Cleanup(func() { // Shut down the informer. close(stopChannel) }) informerFactory.WaitForCacheSync(ctx.Done()) // Use labels on our created ConfigMaps to avoid accidentally listing other ConfigMaps that might // exist in the namespace. In Kube 1.20+ there is a default ConfigMap in every namespace. configMapLabels := labels.Set{ "pinniped.dev/testConfigMap": testlib.RandHex(t, 8), } // Test "create" verb through the impersonation proxy. _, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) _, err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) _, err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Create(ctx, &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}}, metav1.CreateOptions{}, ) require.NoError(t, err) // Make sure that all of the created ConfigMaps show up in the informer's cache to // demonstrate that the informer's "watch" verb is working through the impersonation proxy. testlib.RequireEventually(t, func(requireEventually *require.Assertions) { _, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-1") requireEventually.NoError(err) _, err = informer.Lister().ConfigMaps(namespaceName).Get("configmap-2") requireEventually.NoError(err) _, err = informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") requireEventually.NoError(err) }, 10*time.Second, 50*time.Millisecond) // Test "get" verb through the impersonation proxy. configMap3, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Get(ctx, "configmap-3", metav1.GetOptions{}) require.NoError(t, err) // Test "list" verb through the impersonation proxy. listResult, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) require.Len(t, listResult.Items, 3) // Test "update" verb through the impersonation proxy. configMap3.Data = map[string]string{"foo": "bar"} updateResult, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Update(ctx, configMap3, metav1.UpdateOptions{}) require.NoError(t, err) require.Equal(t, "bar", updateResult.Data["foo"]) // Make sure that the updated ConfigMap shows up in the informer's cache. testlib.RequireEventually(t, func(requireEventually *require.Assertions) { configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") requireEventually.NoError(err) requireEventually.Equal("bar", configMap.Data["foo"]) }, 10*time.Second, 50*time.Millisecond) // Test "patch" verb through the impersonation proxy. patchResult, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Patch(ctx, "configmap-3", types.MergePatchType, []byte(`{"data":{"baz":"42"}}`), metav1.PatchOptions{}, ) require.NoError(t, err) require.Equal(t, "bar", patchResult.Data["foo"]) require.Equal(t, "42", patchResult.Data["baz"]) // Make sure that the patched ConfigMap shows up in the informer's cache. testlib.RequireEventually(t, func(requireEventually *require.Assertions) { configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") requireEventually.NoError(err) requireEventually.Equal("bar", configMap.Data["foo"]) requireEventually.Equal("42", configMap.Data["baz"]) }, 10*time.Second, 50*time.Millisecond) // Test "delete" verb through the impersonation proxy. err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Delete(ctx, "configmap-3", metav1.DeleteOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMap shows up in the informer's cache. testlib.RequireEventually(t, func(requireEventually *require.Assertions) { _, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3") requireEventually.Truef(k8serrors.IsNotFound(err), "expected a NotFound error from get, got %v", err) list, err := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) requireEventually.NoError(err) requireEventually.Len(list, 2) }, 10*time.Second, 50*time.Millisecond) // Test "deletecollection" verb through the impersonation proxy. err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) require.NoError(t, err) // Make sure that the deleted ConfigMaps shows up in the informer's cache. testlib.RequireEventually(t, func(requireEventually *require.Assertions) { list, err := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector()) requireEventually.NoError(err) requireEventually.Empty(list) }, 10*time.Second, 50*time.Millisecond) // There should be no ConfigMaps left. listResult, err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{ LabelSelector: configMapLabels.String(), }) require.NoError(t, err) require.Len(t, listResult.Items, 0) }) t.Run("nested impersonation as a regular user is allowed if they have enough RBAC permissions", func(t *testing.T) { parallelIfNotEKS(t) // Make a client which will send requests through the impersonation proxy and will also add // impersonate headers to the request. nestedImpersonationClient := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{UserName: "other-user-to-impersonate"}, refreshCredential) // Check that we can get some resource through the impersonation proxy without any impersonation headers on the request. // We could use any resource for this, but we happen to know that this one should exist. _, err := impersonationProxyKubeClient(t).CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.NoError(t, err) // Now we'll see what happens when we add an impersonation header to the request. This should generate a // request similar to the one above, except that it will also have an impersonation header. _, err = nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) // this user is not allowed to impersonate other users require.True(t, k8serrors.IsForbidden(err), err) require.EqualError(t, err, fmt.Sprintf( `users "other-user-to-impersonate" is forbidden: `+ `User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+ `decision made by impersonation-proxy.concierge.pinniped.dev`, env.TestUser.ExpectedUsername)) // impersonate the GC service account instead which can read anything (the binding to edit allows this) nestedImpersonationClientAsSA := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{UserName: "system:serviceaccount:kube-system:generic-garbage-collector"}, refreshCredential) _, err = nestedImpersonationClientAsSA.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.NoError(t, err) expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...) expectedGroups = append(expectedGroups, "system:authenticated") expectedOriginalUserInfo := authenticationv1.UserInfo{ Username: env.TestUser.ExpectedUsername, Groups: expectedGroups, } expectedOriginalUserInfoJSON, err := json.Marshal(expectedOriginalUserInfo) require.NoError(t, err) // check that we impersonated the correct user and that the original user is retained in the extra whoAmI, err := nestedImpersonationClientAsSA.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, expectedWhoAmIRequestResponse( "system:serviceaccount:kube-system:generic-garbage-collector", []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"}, map[string]identityv1alpha1.ExtraValue{ "original-user-info.impersonation-proxy.concierge.pinniped.dev": {string(expectedOriginalUserInfoJSON)}, }, ), whoAmI, ) _, err = newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{ UserName: "system:serviceaccount:kube-system:generic-garbage-collector", Extra: map[string][]string{ "some-fancy-key": {"with a dangerous value"}, }, }, refreshCredential).PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) // this user should not be able to impersonate extra require.True(t, k8serrors.IsForbidden(err), err) require.EqualError(t, err, fmt.Sprintf( `userextras.authentication.k8s.io "with a dangerous value" is forbidden: `+ `User "%s" cannot impersonate resource "userextras/some-fancy-key" in API group "authentication.k8s.io" at the cluster scope: `+ `decision made by impersonation-proxy.concierge.pinniped.dev`, env.TestUser.ExpectedUsername)) }) t.Run("nested impersonation as a cluster admin user is allowed", func(t *testing.T) { parallelIfNotEKS(t) // Copy the admin credentials from the admin kubeconfig. adminClientRestConfig := testlib.NewClientConfig(t) clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig) // figure out who the admin user is whoAmIAdmin, err := newImpersonationProxyClientWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil). PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) // The WhoAmI API is lossy: // - It drops UID // - It lowercases all extra keys // the admin user on EKS has both a UID set and an extra key with uppercase characters // Thus we fallback to the CSR API to grab the UID and Extra to handle this scenario uid, extra := getUIDAndExtraViaCSR(ctx, t, whoAmIAdmin.Status.KubernetesUserInfo.User.UID, newImpersonationProxyClientWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil). Kubernetes, ) expectedExtra := make(map[string]authenticationv1.ExtraValue, len(extra)) for k, v := range extra { expectedExtra[k] = authenticationv1.ExtraValue(v) } expectedOriginalUserInfo := authenticationv1.UserInfo{ Username: whoAmIAdmin.Status.KubernetesUserInfo.User.Username, UID: uid, Groups: whoAmIAdmin.Status.KubernetesUserInfo.User.Groups, Extra: expectedExtra, } expectedOriginalUserInfoJSON, err := json.Marshal(expectedOriginalUserInfo) require.NoError(t, err) // Make a client using the admin credentials which will send requests through the impersonation proxy // and will also add impersonate headers to the request. nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{ UserName: "other-user-to-impersonate", Groups: []string{"other-group-1", "other-group-2"}, Extra: map[string][]string{ "this-key": {"to this value"}, }, }, ) _, err = nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) // the impersonated user lacks the RBAC to perform this call require.True(t, k8serrors.IsForbidden(err), err) require.EqualError(t, err, fmt.Sprintf( `secrets "%s" is forbidden: User "other-user-to-impersonate" cannot get resource "secrets" in API group "" in the namespace "%s": `+ `decision made by impersonation-proxy.concierge.pinniped.dev`, impersonationProxyTLSSecretName(env), env.ConciergeNamespace, )) // check that we impersonated the correct user and that the original user is retained in the extra whoAmI, err := nestedImpersonationClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, expectedWhoAmIRequestResponse( "other-user-to-impersonate", []string{"other-group-1", "other-group-2", "system:authenticated"}, map[string]identityv1alpha1.ExtraValue{ "this-key": {"to this value"}, "original-user-info.impersonation-proxy.concierge.pinniped.dev": {string(expectedOriginalUserInfoJSON)}, }, ), whoAmI, ) }) t.Run("nested impersonation as a cluster admin fails on reserved key", func(t *testing.T) { parallelIfNotEKS(t) adminClientRestConfig := testlib.NewClientConfig(t) clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig) nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{ UserName: "other-user-to-impersonate", Groups: []string{"other-group-1", "other-group-2", "system:masters"}, // impersonate system:masters so we get past authorization checks Extra: map[string][]string{ "this-good-key": {"to this good value"}, "something.impersonation-proxy.concierge.pinniped.dev": {"super sneaky value"}, }, }, ) _, err := nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") require.True(t, k8serrors.IsInternalError(err), err) require.Equal(t, &k8serrors.StatusError{ ErrStatus: metav1.Status{ Status: metav1.StatusFailure, Code: http.StatusInternalServerError, Reason: metav1.StatusReasonInternalError, Details: &metav1.StatusDetails{ Causes: []metav1.StatusCause{ { Message: "unimplemented functionality - unable to act as current user", }, }, }, Message: "Internal error occurred: unimplemented functionality - unable to act as current user", }, }, err) }) t.Run("nested impersonation as a cluster admin fails if UID impersonation is attempted", func(t *testing.T) { parallelIfNotEKS(t) adminClientRestConfig := testlib.NewClientConfig(t) clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig) nestedImpersonationUIDOnly := newImpersonationProxyConfigWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil, ) nestedImpersonationUIDOnly.Wrap(func(rt http.RoundTripper) http.RoundTripper { return roundtripper.WrapFunc(rt, func(r *http.Request) (*http.Response, error) { r.Header.Set("iMperSONATE-uid", "some-awesome-uid") return rt.RoundTrip(r) }) }) _, errUID := testlib.NewKubeclient(t, nestedImpersonationUIDOnly).Kubernetes.CoreV1().Secrets("foo").Get(ctx, "bar", metav1.GetOptions{}) msg := `Internal Server Error: "/api/v1/namespaces/foo/secrets/bar": requested [{UID some-awesome-uid authentication.k8s.io/v1 }] without impersonating a user` full := fmt.Sprintf(`an error on the server (%q) has prevented the request from succeeding (get secrets bar)`, msg) require.EqualError(t, errUID, full) require.True(t, k8serrors.IsInternalError(errUID), errUID) require.Equal(t, &k8serrors.StatusError{ ErrStatus: metav1.Status{ Status: metav1.StatusFailure, Code: http.StatusInternalServerError, Reason: metav1.StatusReasonInternalError, Details: &metav1.StatusDetails{ Name: "bar", Kind: "secrets", Causes: []metav1.StatusCause{ { Type: metav1.CauseTypeUnexpectedServerResponse, Message: msg, }, }, }, Message: full, }, }, errUID) nestedImpersonationUID := newImpersonationProxyConfigWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{ UserName: "other-user-to-impersonate", Groups: []string{"system:masters"}, // impersonate system:masters so we get past authorization checks }, ) nestedImpersonationUID.Wrap(func(rt http.RoundTripper) http.RoundTripper { return roundtripper.WrapFunc(rt, func(r *http.Request) (*http.Response, error) { r.Header.Set("imperSONate-uiD", "some-fancy-uid") return rt.RoundTrip(r) }) }) _, err := testlib.NewKubeclient(t, nestedImpersonationUID).Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user") require.True(t, k8serrors.IsInternalError(err), err) require.Equal(t, &k8serrors.StatusError{ ErrStatus: metav1.Status{ Status: metav1.StatusFailure, Code: http.StatusInternalServerError, Reason: metav1.StatusReasonInternalError, Details: &metav1.StatusDetails{ Causes: []metav1.StatusCause{ { Message: "unimplemented functionality - unable to act as current user", }, }, }, Message: "Internal error occurred: unimplemented functionality - unable to act as current user", }, }, err) }) // this works because impersonation cannot set UID and thus the final user info the proxy sees has no UID t.Run("nested impersonation as a service account is allowed if it has enough RBAC permissions", func(t *testing.T) { parallelIfNotEKS(t) namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name saName, saToken, saUID := createServiceAccountToken(ctx, t, adminClient, namespaceName) nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t, &loginv1alpha1.ClusterCredential{Token: saToken}, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{UserName: "system:serviceaccount:kube-system:root-ca-cert-publisher"}).PinnipedConcierge _, err := nestedImpersonationClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) // this SA is not yet allowed to impersonate SAs require.True(t, k8serrors.IsForbidden(err), err) require.EqualError(t, err, fmt.Sprintf( `serviceaccounts "root-ca-cert-publisher" is forbidden: `+ `User "%s" cannot impersonate resource "serviceaccounts" in API group "" in the namespace "kube-system": `+ `decision made by impersonation-proxy.concierge.pinniped.dev`, serviceaccount.MakeUsername(namespaceName, saName))) // webhook authorizer deny cache TTL is 10 seconds so we need to wait long enough for it to drain time.Sleep(15 * time.Second) // allow the test SA to impersonate any SA testlib.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: saName, Namespace: namespaceName}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"}, ) testlib.WaitForUserToHaveAccess(t, serviceaccount.MakeUsername(namespaceName, saName), []string{}, &authorizationv1.ResourceAttributes{ Verb: "impersonate", Group: "", Version: "v1", Resource: "serviceaccounts", }) whoAmI, err := nestedImpersonationClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, expectedWhoAmIRequestResponse( "system:serviceaccount:kube-system:root-ca-cert-publisher", []string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"}, map[string]identityv1alpha1.ExtraValue{ "original-user-info.impersonation-proxy.concierge.pinniped.dev": { fmt.Sprintf(`{"username":"%s","uid":"%s","groups":["system:serviceaccounts","system:serviceaccounts:%s","system:authenticated"]}`, serviceaccount.MakeUsername(namespaceName, saName), saUID, namespaceName), }, }, ), whoAmI, ) }) t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) { parallelIfNotEKS(t) // Test using the TokenCredentialRequest for authentication. impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil, refreshCredential, ).PinnipedConcierge whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...) expectedGroups = append(expectedGroups, "system:authenticated") require.Equal(t, expectedWhoAmIRequestResponse( env.TestUser.ExpectedUsername, expectedGroups, nil, ), whoAmI, ) // Test an unauthenticated request which does not include any credentials. impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient( t, impersonationProxyURL, impersonationProxyCACertPEM, nil, ).PinnipedConcierge whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) // we expect the impersonation proxy to match the behavior of KAS in regards to anonymous requests if env.HasCapability(testlib.AnonymousAuthenticationSupported) { require.NoError(t, err) require.Equal(t, expectedWhoAmIRequestResponse( "system:anonymous", []string{"system:unauthenticated"}, nil, ), whoAmI, ) } else { require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err)) } // Test using a service account token. namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name saName, saToken, _ := createServiceAccountToken(ctx, t, adminClient, namespaceName) impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials(t, &loginv1alpha1.ClusterCredential{Token: saToken}, impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge whoAmI, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, expectedWhoAmIRequestResponse( serviceaccount.MakeUsername(namespaceName, saName), []string{"system:serviceaccounts", "system:serviceaccounts:" + namespaceName, "system:authenticated"}, nil, ), whoAmI, ) }) t.Run("WhoAmIRequests and SA token request", func(t *testing.T) { namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name kubeClient := adminClient.CoreV1() saName, _, saUID := createServiceAccountToken(ctx, t, adminClient, namespaceName) expectedUsername := serviceaccount.MakeUsername(namespaceName, saName) expectedUID := string(saUID) expectedGroups := []string{"system:serviceaccounts", "system:serviceaccounts:" + namespaceName, "system:authenticated"} _, tokenRequestProbeErr := kubeClient.ServiceAccounts(namespaceName).CreateToken(ctx, saName, &authenticationv1.TokenRequest{}, metav1.CreateOptions{}) if k8serrors.IsNotFound(tokenRequestProbeErr) && tokenRequestProbeErr.Error() == "the server could not find the requested resource" { return // stop test early since the token request API is not enabled on this cluster - other errors are caught below } pod := testlib.CreatePod(ctx, t, "impersonation-proxy", namespaceName, corev1.PodSpec{ Containers: []corev1.Container{ { Name: "sleeper", Image: env.ShellContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"sh", "-c", "sleep 3600"}, }, }, ServiceAccountName: saName, }) tokenRequestBadAudience, err := kubeClient.ServiceAccounts(namespaceName).CreateToken(ctx, saName, &authenticationv1.TokenRequest{ Spec: authenticationv1.TokenRequestSpec{ Audiences: []string{"should-fail-because-wrong-audience"}, // anything that is not an API server audience BoundObjectRef: &authenticationv1.BoundObjectReference{ Kind: "Pod", APIVersion: "", Name: pod.Name, UID: pod.UID, }, }, }, metav1.CreateOptions{}) require.NoError(t, err) impersonationProxySABadAudPinnipedConciergeClient := newImpersonationProxyClientWithCredentials(t, &loginv1alpha1.ClusterCredential{Token: tokenRequestBadAudience.Status.Token}, impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge _, badAudErr := impersonationProxySABadAudPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.True(t, k8serrors.IsUnauthorized(badAudErr), testlib.Sdump(badAudErr)) tokenRequest, err := kubeClient.ServiceAccounts(namespaceName).CreateToken(ctx, saName, &authenticationv1.TokenRequest{ Spec: authenticationv1.TokenRequestSpec{ Audiences: []string{}, BoundObjectRef: &authenticationv1.BoundObjectReference{ Kind: "Pod", APIVersion: "", Name: pod.Name, UID: pod.UID, }, }, }, metav1.CreateOptions{}) require.NoError(t, err) impersonationProxySAClient := newImpersonationProxyClientWithCredentials(t, &loginv1alpha1.ClusterCredential{Token: tokenRequest.Status.Token}, impersonationProxyURL, impersonationProxyCACertPEM, nil) whoAmITokenReq, err := impersonationProxySAClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) // new service account tokens include the pod info in the extra fields require.Equal(t, expectedWhoAmIRequestResponse( expectedUsername, expectedGroups, map[string]identityv1alpha1.ExtraValue{ "authentication.kubernetes.io/pod-name": {pod.Name}, "authentication.kubernetes.io/pod-uid": {string(pod.UID)}, }, ), whoAmITokenReq, ) // allow the test SA to create CSRs testlib.CreateTestClusterRoleBinding(t, rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: saName, Namespace: namespaceName}, rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "system:node-bootstrapper"}, ) testlib.WaitForUserToHaveAccess(t, expectedUsername, []string{}, &authorizationv1.ResourceAttributes{ Verb: "create", Group: certificatesv1.GroupName, Version: "*", Resource: "certificatesigningrequests", }) privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) csrPEM, err := cert.MakeCSR(privateKey, &pkix.Name{ CommonName: "panda-man", Organization: []string{"living-the-dream", "need-more-sleep"}, }, nil, nil) require.NoError(t, err) csrName, _, err := csr.RequestCertificate( impersonationProxySAClient.Kubernetes, csrPEM, "", certificatesv1.KubeAPIServerClientSignerName, nil, []certificatesv1.KeyUsage{certificatesv1.UsageClientAuth}, privateKey, ) require.NoError(t, err) if testutil.KubeServerSupportsCertificatesV1API(t, adminClient.Discovery()) { saCSR, err := impersonationProxySAClient.Kubernetes.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) require.NoError(t, err) err = adminClient.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}) require.NoError(t, err) // make sure the user info that the CSR captured matches the SA, including the UID require.Equal(t, expectedUsername, saCSR.Spec.Username) require.Equal(t, expectedUID, saCSR.Spec.UID) require.Equal(t, expectedGroups, saCSR.Spec.Groups) require.Equal(t, map[string]certificatesv1.ExtraValue{ "authentication.kubernetes.io/pod-name": {pod.Name}, "authentication.kubernetes.io/pod-uid": {string(pod.UID)}, }, saCSR.Spec.Extra) } else { // On old Kubernetes clusters use CertificatesV1beta1 saCSR, err := impersonationProxySAClient.Kubernetes.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) require.NoError(t, err) err = adminClient.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}) require.NoError(t, err) // make sure the user info that the CSR captured matches the SA, including the UID require.Equal(t, expectedUsername, saCSR.Spec.Username) require.Equal(t, expectedUID, saCSR.Spec.UID) require.Equal(t, expectedGroups, saCSR.Spec.Groups) require.Equal(t, map[string]certificatesv1beta1.ExtraValue{ "authentication.kubernetes.io/pod-name": {pod.Name}, "authentication.kubernetes.io/pod-uid": {string(pod.UID)}, }, saCSR.Spec.Extra) } }) t.Run("kubectl as a client", func(t *testing.T) { parallelIfNotEKS(t) kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator) // Run a new test pod so we can interact with it using kubectl. We use a fresh pod here rather than the // existing Concierge pod because we need more tools than we can get from a scratch/distroless base image. runningTestPod := testlib.CreatePod(ctx, t, "impersonation-proxy", env.ConciergeNamespace, corev1.PodSpec{Containers: []corev1.Container{{ Name: "impersonation-proxy-test", Image: env.ShellContainerImage, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"bash", "-c", `while true; do read VAR; echo "VAR: $VAR"; done`}, Stdin: true, Resources: corev1.ResourceRequirements{ Limits: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("16Mi"), corev1.ResourceCPU: resource.MustParse("10m"), }, Requests: corev1.ResourceList{ corev1.ResourceMemory: resource.MustParse("16Mi"), corev1.ResourceCPU: resource.MustParse("10m"), }, }, }}}) // Try "kubectl exec" through the impersonation proxy. echoString := "hello world" remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix()) stdout, err := runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", runningTestPod.Namespace, runningTestPod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile)) require.NoError(t, err, `"kubectl exec" failed`) require.Equal(t, echoString+"\n", stdout) // run the kubectl cp command localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile)) _, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "cp", fmt.Sprintf("%s/%s:%s", runningTestPod.Namespace, runningTestPod.Name, remoteEchoFile), localEchoFile) require.NoError(t, err, `"kubectl cp" failed`) localEchoFileData, err := os.ReadFile(localEchoFile) require.NoError(t, err) require.Equal(t, echoString+"\n", string(localEchoFileData)) // run the kubectl logs command logLinesCount := 10 stdout, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "logs", "--namespace", supervisorPod.Namespace, supervisorPod.Name, fmt.Sprintf("--tail=%d", logLinesCount)) require.NoError(t, err, `"kubectl logs" failed`) // Expect _approximately_ logLinesCount lines in the output // (we can't match 100% exactly due to https://github.com/kubernetes/kubernetes/issues/72628). require.InDeltaf(t, logLinesCount, strings.Count(stdout, "\n"), 1, "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout) // run the kubectl attach command timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute) defer cancelFunc() attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", runningTestPod.Namespace, runningTestPod.Name, "-v=10") attachCmd.Env = envVarsWithProxy attachStdin, err := attachCmd.StdinPipe() require.NoError(t, err) // start but don't wait for the attach command err = attachCmd.Start() require.NoError(t, err) attachExitCh := make(chan struct{}) go func() { assert.NoError(t, attachCmd.Wait()) close(attachExitCh) }() // write to stdin on the attach process _, err = attachStdin.Write([]byte(echoString + "\n")) require.NoError(t, err) // see that we can read stdout and it spits out stdin output back to us wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString) testlib.RequireEventually(t, func(requireEventually *require.Assertions) { requireEventually.Equal( wantAttachStdout, attachStdout.String(), `got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`, attachStdout.String(), wantAttachStdout, attachStderr.String(), ) }, time.Second*60, time.Millisecond*250) // close stdin and attach process should exit err = attachStdin.Close() require.NoError(t, err) requireClose(t, attachExitCh, time.Second*20) }) t.Run("websocket client", func(t *testing.T) { parallelIfNotEKS(t) namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name impersonationRestConfig := impersonationProxyRestConfig( refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM), impersonationProxyURL, impersonationProxyCACertPEM, nil, ) tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig) require.NoError(t, err) wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" dest, _ := url.Parse(impersonationProxyURL) dest.Scheme = "wss" dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" dest.RawQuery = url.Values{ "watch": {"1"}, "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, "resourceVersion": {"0"}, }.Encode() dialer := websocket.Dialer{ TLSClientConfig: tlsConfig, } if !clusterSupportsLoadBalancers { dialer.Proxy = func(req *http.Request) (*url.URL, error) { proxyURL, err := url.Parse(env.Proxy) require.NoError(t, err) t.Logf("passing request for %s through proxy %s", testlib.RedactURLParams(req.URL), proxyURL.String()) return proxyURL, nil } } var ( resp *http.Response conn *websocket.Conn ) testlib.RequireEventually(t, func(requireEventually *require.Assertions) { var err error conn, resp, err = dialer.Dial(dest.String(), http.Header{"Origin": {dest.String()}}) if resp != nil { defer func() { requireEventually.NoError(resp.Body.Close()) }() } if err != nil && resp != nil { body, _ := io.ReadAll(resp.Body) t.Logf("websocket dial failed: %d:%s", resp.StatusCode, body) } requireEventually.NoError(err) }, time.Minute, time.Second) // perform a create through the admin client wantConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, } wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, wantConfigMap, metav1.CreateOptions{}, ) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, adminClient.CoreV1().ConfigMaps(namespaceName). DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{})) }) // see if the websocket client received an event for the create _, message, err := conn.ReadMessage() if err != nil { t.Fatalf("Unexpected error: %v", err) } var got watchJSON err = json.Unmarshal(message, &got) require.NoError(t, err) if got.Type != watch.Added { t.Errorf("Unexpected type: %v", got.Type) } var actualConfigMap corev1.ConfigMap require.NoError(t, json.Unmarshal(got.Object, &actualConfigMap)) actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. require.Equal(t, *wantConfigMap, actualConfigMap) }) t.Run("http2 client", func(t *testing.T) { parallelIfNotEKS(t) namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value" wantConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}}, } wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx, wantConfigMap, metav1.CreateOptions{}, ) require.NoError(t, err) t.Cleanup(func() { _ = adminClient.CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{}) }) // create rest client restConfig := impersonationProxyRestConfig( refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM), impersonationProxyURL, impersonationProxyCACertPEM, nil, ) tlsConfig, err := rest.TLSConfigFor(restConfig) require.NoError(t, err) httpTransport := http.Transport{ TLSClientConfig: tlsConfig, } if !clusterSupportsLoadBalancers { httpTransport.Proxy = func(req *http.Request) (*url.URL, error) { proxyURL, err := url.Parse(env.Proxy) require.NoError(t, err) t.Logf("passing request for %s through proxy %s", testlib.RedactURLParams(req.URL), proxyURL.String()) return proxyURL, nil } } err = http2.ConfigureTransport(&httpTransport) require.NoError(t, err) httpClient := http.Client{ Transport: &httpTransport, } dest, _ := url.Parse(impersonationProxyURL) dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1" getConfigmapRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) require.NoError(t, err) response, err := httpClient.Do(getConfigmapRequest) require.NoError(t, err) body, _ := io.ReadAll(response.Body) t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body) require.Equal(t, "HTTP/2.0", response.Proto) require.Equal(t, http.StatusOK, response.StatusCode) defer func() { require.NoError(t, response.Body.Close()) }() var actualConfigMap corev1.ConfigMap require.NoError(t, json.Unmarshal(body, &actualConfigMap)) actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. require.Equal(t, *wantConfigMap, actualConfigMap) // watch configmaps dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps" dest.RawQuery = url.Values{ "watch": {"1"}, "labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)}, "resourceVersion": {"0"}, }.Encode() watchConfigmapsRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil) require.NoError(t, err) response, err = httpClient.Do(watchConfigmapsRequest) require.NoError(t, err) require.Equal(t, "HTTP/2.0", response.Proto) require.Equal(t, http.StatusOK, response.StatusCode) defer func() { require.NoError(t, response.Body.Close()) }() // decode decoder := json.NewDecoder(response.Body) var got watchJSON err = decoder.Decode(&got) require.NoError(t, err) if got.Type != watch.Added { t.Errorf("Unexpected type: %v", got.Type) } err = json.Unmarshal(got.Object, &actualConfigMap) require.NoError(t, err) require.Equal(t, "configmap-1", actualConfigMap.Name) actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create. require.Equal(t, *wantConfigMap, actualConfigMap) }) t.Run("honors anonymous authentication of KAS", func(t *testing.T) { parallelIfNotEKS(t) impersonationProxyAnonymousClient := newAnonymousImpersonationProxyClient( t, impersonationProxyURL, impersonationProxyCACertPEM, nil, ) copyConfig := rest.CopyConfig(impersonationProxyAnonymousClient.JSONConfig) copyConfig.GroupVersion = &schema.GroupVersion{} copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer() impersonationProxyAnonymousRestClient, err := rest.RESTClientFor(copyConfig) require.NoError(t, err) adminClientRestConfig := testlib.NewClientConfig(t) clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig) impersonationProxyAdminClientAsAnonymousConfig := newImpersonationProxyClientWithCredentials(t, clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, &rest.ImpersonationConfig{UserName: user.Anonymous}). JSONConfig impersonationProxyAdminClientAsAnonymousConfigCopy := rest.CopyConfig(impersonationProxyAdminClientAsAnonymousConfig) impersonationProxyAdminClientAsAnonymousConfigCopy.GroupVersion = &schema.GroupVersion{} impersonationProxyAdminClientAsAnonymousConfigCopy.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer() impersonationProxyAdminRestClientAsAnonymous, err := rest.RESTClientFor(impersonationProxyAdminClientAsAnonymousConfigCopy) require.NoError(t, err) t.Run("anonymous authentication irrelevant", func(t *testing.T) { parallelIfNotEKS(t) // - hit the token credential request endpoint with an empty body // - through the impersonation proxy // - should succeed as an invalid request whether anonymous authentication is enabled or disabled // - should not reject as unauthorized t.Run("token credential request", func(t *testing.T) { parallelIfNotEKS(t) tkr, err := impersonationProxyAnonymousClient.PinnipedConcierge.LoginV1alpha1().TokenCredentialRequests(). Create(ctx, &loginv1alpha1.TokenCredentialRequest{ Spec: loginv1alpha1.TokenCredentialRequestSpec{ Authenticator: corev1.TypedLocalObjectReference{APIGroup: pointer.String("anything.pinniped.dev")}, }, }, metav1.CreateOptions{}) require.True(t, k8serrors.IsInvalid(err), testlib.Sdump(err)) require.Equal(t, `.login.concierge.pinniped.dev "" is invalid: spec.token.value: Required value: token must be supplied`, err.Error()) require.Equal(t, &loginv1alpha1.TokenCredentialRequest{}, tkr) }) // - hit the healthz endpoint (non-resource endpoint) // - through the impersonation proxy // - as cluster admin, impersonating anonymous user // - should succeed, authentication happens as cluster-admin // - whoami should confirm we are using impersonation // - healthz should succeed, anonymous users can request this endpoint // - healthz/log should fail, forbidden anonymous t.Run("non-resource request while impersonating anonymous - nested impersonation", func(t *testing.T) { parallelIfNotEKS(t) whoami, errWho := impersonationProxyAdminRestClientAsAnonymous.Post().Body([]byte(`{}`)).AbsPath("/apis/identity.concierge." + env.APIGroupSuffix + "/v1alpha1/whoamirequests").DoRaw(ctx) require.NoError(t, errWho, testlib.Sdump(errWho)) require.True(t, strings.HasPrefix(string(whoami), `{"kind":"WhoAmIRequest","apiVersion":"identity.concierge.`+env.APIGroupSuffix+`/v1alpha1","metadata":{"creationTimestamp":null},"spec":{},"status":{"kubernetesUserInfo":{"user":{"username":"system:anonymous","groups":["system:unauthenticated"],"extra":{"original-user-info.impersonation-proxy.concierge.pinniped.dev":["{\"username\":`), string(whoami)) healthz, errHealth := impersonationProxyAdminRestClientAsAnonymous.Get().AbsPath("/healthz").DoRaw(ctx) require.NoError(t, errHealth, testlib.Sdump(errHealth)) require.Equal(t, "ok", string(healthz)) healthzLog, errHealthzLog := impersonationProxyAdminRestClientAsAnonymous.Get().AbsPath("/healthz/log").DoRaw(ctx) require.True(t, k8serrors.IsForbidden(errHealthzLog), "%s\n%s", testlib.Sdump(errHealthzLog), string(healthzLog)) require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"forbidden: User \"system:anonymous\" cannot get path \"/healthz/log\": decision made by impersonation-proxy.concierge.pinniped.dev","reason":"Forbidden","details":{},"code":403}`+"\n", string(healthzLog)) }) }) t.Run("anonymous authentication enabled", func(t *testing.T) { testlib.IntegrationEnv(t).WithCapability(testlib.AnonymousAuthenticationSupported) parallelIfNotEKS(t) // anonymous auth enabled // - hit the healthz endpoint (non-resource endpoint) // - through the impersonation proxy // - should succeed 200 // - should respond "ok" t.Run("non-resource request", func(t *testing.T) { parallelIfNotEKS(t) healthz, errHealth := impersonationProxyAnonymousRestClient.Get().AbsPath("/healthz").DoRaw(ctx) require.NoError(t, errHealth, testlib.Sdump(errHealth)) require.Equal(t, "ok", string(healthz)) }) // - hit the pods endpoint (a resource endpoint) // - through the impersonation proxy // - should fail forbidden // - system:anonymous cannot get pods t.Run("resource", func(t *testing.T) { parallelIfNotEKS(t) pod, err := impersonationProxyAnonymousClient.Kubernetes.CoreV1().Pods(metav1.NamespaceSystem). Get(ctx, "does-not-matter", metav1.GetOptions{}) require.True(t, k8serrors.IsForbidden(err), testlib.Sdump(err)) require.EqualError(t, err, `pods "does-not-matter" is forbidden: User "system:anonymous" cannot get resource "pods" in API group "" in the namespace "kube-system": `+ `decision made by impersonation-proxy.concierge.pinniped.dev`, testlib.Sdump(err)) require.Equal(t, &corev1.Pod{}, pod) }) // - request to whoami (pinniped resource endpoint) // - through the impersonation proxy // - should succeed 200 // - should respond "you are system:anonymous" t.Run("pinniped resource request", func(t *testing.T) { parallelIfNotEKS(t) whoAmI, err := impersonationProxyAnonymousClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.NoError(t, err) require.Equal(t, expectedWhoAmIRequestResponse( "system:anonymous", []string{"system:unauthenticated"}, nil, ), whoAmI, ) }) }) t.Run("anonymous authentication disabled", func(t *testing.T) { testlib.IntegrationEnv(t).WithoutCapability(testlib.AnonymousAuthenticationSupported) parallelIfNotEKS(t) // - hit the healthz endpoint (non-resource endpoint) // - through the impersonation proxy // - should fail unauthorized // - kube api server should reject it t.Run("non-resource request", func(t *testing.T) { parallelIfNotEKS(t) healthz, err := impersonationProxyAnonymousRestClient.Get().AbsPath("/healthz").DoRaw(ctx) require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err)) require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401}`+"\n", string(healthz)) }) // - hit the pods endpoint (a resource endpoint) // - through the impersonation proxy // - should fail unauthorized // - kube api server should reject it t.Run("resource", func(t *testing.T) { parallelIfNotEKS(t) pod, err := impersonationProxyAnonymousClient.Kubernetes.CoreV1().Pods(metav1.NamespaceSystem). Get(ctx, "does-not-matter", metav1.GetOptions{}) require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err)) require.Equal(t, &corev1.Pod{}, pod) }) // - request to whoami (pinniped resource endpoing) // - through the impersonation proxy // - should fail unauthorized // - kube api server should reject it t.Run("pinniped resource request", func(t *testing.T) { parallelIfNotEKS(t) whoAmI, err := impersonationProxyAnonymousClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err)) require.Equal(t, &identityv1alpha1.WhoAmIRequest{}, whoAmI) }) }) }) t.Run("assert impersonator runs with secure TLS config", func(t *testing.T) { parallelIfNotEKS(t) cancelCtx, cancel := context.WithCancel(ctx) t.Cleanup(cancel) startKubectlPortForward(cancelCtx, t, "10445", "443", env.ConciergeAppName+"-proxy", env.ConciergeNamespace) stdout, stderr := testlib.RunNmapSSLEnum(t, "127.0.0.1", 10445) require.Empty(t, stderr) require.Contains(t, stdout, testlib.GetExpectedCiphers(ptls.Default(nil)), "stdout:\n%s", stdout) }) }) t.Run("assert correct impersonator service account is being used", func(t *testing.T) { // pick an API that everyone can access but always make invalid requests to it // we can tell that the request is reaching KAS because only it has the validation logic impersonationProxySSRRClient := impersonationProxyKubeClient(t).AuthorizationV1().SelfSubjectRulesReviews() crbClient := adminClient.RbacV1().ClusterRoleBindings() impersonationProxyName := env.ConciergeAppName + "-impersonation-proxy" saFullName := serviceaccount.MakeUsername(env.ConciergeNamespace, impersonationProxyName) invalidSSRR := &authorizationv1.SelfSubjectRulesReview{} // sanity check default expected error message _, err := impersonationProxySSRRClient.Create(ctx, invalidSSRR, metav1.CreateOptions{}) require.True(t, k8serrors.IsBadRequest(err), testlib.Sdump(err)) require.EqualError(t, err, "no namespace on request") // remove the impersonation proxy SA's permissions crb, err := crbClient.Get(ctx, impersonationProxyName, metav1.GetOptions{}) require.NoError(t, err) // sanity check the subject require.Len(t, crb.Subjects, 1) sub := crb.Subjects[0].DeepCopy() require.Equal(t, &rbacv1.Subject{ Kind: "ServiceAccount", APIGroup: "", Name: impersonationProxyName, Namespace: env.ConciergeNamespace, }, sub) crb.Subjects = nil _, err = crbClient.Update(ctx, crb, metav1.UpdateOptions{}) require.NoError(t, err) // make sure to put the permissions back at the end t.Cleanup(func() { crbEnd, errEnd := crbClient.Get(ctx, impersonationProxyName, metav1.GetOptions{}) require.NoError(t, errEnd) crbEnd.Subjects = []rbacv1.Subject{*sub} _, errUpdate := crbClient.Update(ctx, crbEnd, metav1.UpdateOptions{}) require.NoError(t, errUpdate) testlib.WaitForUserToHaveAccess(t, saFullName, nil, &authorizationv1.ResourceAttributes{ Verb: "impersonate", Resource: "users", }) }) // assert that the impersonation proxy stops working when we remove its permissions testlib.RequireEventuallyWithoutError(t, func() (bool, error) { _, errCreate := impersonationProxySSRRClient.Create(ctx, invalidSSRR, metav1.CreateOptions{}) switch { case errCreate == nil: return false, fmt.Errorf("unexpected nil error for test user create invalid SSRR") case k8serrors.IsBadRequest(errCreate) && errCreate.Error() == "no namespace on request": t.Log("waiting for impersonation proxy service account to lose impersonate permissions") return false, nil // RBAC change has not rolled out yet case k8serrors.IsForbidden(errCreate) && errCreate.Error() == `users "`+env.TestUser.ExpectedUsername+`" is forbidden: User "`+saFullName+ `" cannot impersonate resource "users" in API group "" at the cluster scope`: return true, nil // expected RBAC error default: return false, fmt.Errorf("unexpected error for test user create invalid SSRR: %w", errCreate) } }, time.Minute, time.Second) }) t.Run("adding an annotation reconciles the LoadBalancer service", func(t *testing.T) { if !(impersonatorShouldHaveStartedAutomaticallyByDefault && clusterSupportsLoadBalancers) { t.Skip("only running when the cluster is meant to be using LoadBalancer services") } // Use this string in all annotation keys added by this test, so the assertions can ignore annotation keys // which might exist on the Service which are not related to this test. recognizableAnnotationKeyString := "pinniped.dev" // Grab the state of the CredentialIssuer prior to this test, so we can restore things back afterwards. previous, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) require.NoError(t, err) updateServiceAnnotations := func(annotations map[string]string) { require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { service, err := adminClient.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) if err != nil { return err } updated := service.DeepCopy() if updated.Annotations == nil { updated.Annotations = map[string]string{} } // Add/update each requested annotation, without overwriting others that are already there. for k, v := range annotations { updated.Annotations[k] = v } if equality.Semantic.DeepEqual(service, updated) { return nil } t.Logf("updating Service with annotations: %v", annotations) _, err = adminClient.CoreV1().Services(env.ConciergeNamespace).Update(ctx, updated, metav1.UpdateOptions{}) return err })) } deleteServiceAnnotations := func(annotations map[string]string) { require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { service, err := adminClient.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) if err != nil { return err } updated := service.DeepCopy() if updated.Annotations != nil { for k := range annotations { delete(updated.Annotations, k) } } if equality.Semantic.DeepEqual(service, updated) { return nil } t.Logf("updating Service to remove annotations: %v", annotations) _, err = adminClient.CoreV1().Services(env.ConciergeNamespace).Update(ctx, updated, metav1.UpdateOptions{}) return err })) } applyCredentialIssuerAnnotations := func(annotations map[string]string) { require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error { issuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) if err != nil { return err } updated := issuer.DeepCopy() updated.Spec.ImpersonationProxy.Service.Annotations = annotations if equality.Semantic.DeepEqual(issuer, updated) { return nil } t.Logf("updating CredentialIssuer with spec.impersonationProxy.service.annotations: %v", annotations) _, err = adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Update(ctx, updated, metav1.UpdateOptions{}) return err })) } waitForServiceAnnotations := func(wantAnnotations map[string]string, annotationKeyFilter string) { testlib.RequireEventuallyWithoutError(t, func() (bool, error) { service, err := adminClient.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) if err != nil { return false, err } filteredActualAnnotations := map[string]string{} for k, v := range service.Annotations { // We do want to pay attention to any annotation for which we intend to make an explicit assertion, // e.g. "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" which is from our // default CredentialIssuer spec. _, wantToMakeAssertionOnThisAnnotation := wantAnnotations[k] // We do not want to pay attention to Service annotations added by other controllers, // e.g. the "cloud.google.com/neg" annotation that is sometimes added by GKE on Services. // These can come and go in time intervals outside of our control. annotationContainsFilterString := strings.Contains(k, annotationKeyFilter) if wantToMakeAssertionOnThisAnnotation || annotationContainsFilterString { filteredActualAnnotations[k] = v } } t.Logf("found Service %s of type %s with actual annotations %q; filtered by interesting keys results in %q; expected annotations %q", service.Name, service.Spec.Type, service.Annotations, filteredActualAnnotations, wantAnnotations) return equality.Semantic.DeepEqual(filteredActualAnnotations, wantAnnotations), nil }, 1*time.Minute, 1*time.Second) } expectedAnnotations := func(credentialIssuerSpecAnnotations map[string]string, otherAnnotations map[string]string) map[string]string { credentialIssuerSpecAnnotationKeys := []string{} expectedAnnotations := map[string]string{} // Expect the annotations specified on the CredentialIssuer spec to be present. for k, v := range credentialIssuerSpecAnnotations { credentialIssuerSpecAnnotationKeys = append(credentialIssuerSpecAnnotationKeys, k) expectedAnnotations[k] = v } // Aside from the annotations requested on the CredentialIssuer spec, also expect the other annotation to still be there too. for k, v := range otherAnnotations { expectedAnnotations[k] = v } // Also expect the internal bookkeeping annotation to be present. It tracks the requested keys from the spec. // Our controller sorts these keys to make the order in the annotation's value predictable. sort.Strings(credentialIssuerSpecAnnotationKeys) credentialIssuerSpecAnnotationKeysJSON, err := json.Marshal(credentialIssuerSpecAnnotationKeys) require.NoError(t, err) // The name of this annotation key is decided by our controller. expectedAnnotations["credentialissuer."+recognizableAnnotationKeyString+"/annotation-keys"] = string(credentialIssuerSpecAnnotationKeysJSON) return expectedAnnotations } otherActorAnnotations := map[string]string{ recognizableAnnotationKeyString + "/test-other-actor-" + testlib.RandHex(t, 8): "test-other-actor-" + testlib.RandHex(t, 8), } // Whatever happens, set the annotations back to the original value and expect the Service to be updated. t.Cleanup(func() { t.Log("reverting CredentialIssuer back to previous configuration") deleteServiceAnnotations(otherActorAnnotations) applyCredentialIssuerAnnotations(previous.Spec.ImpersonationProxy.Service.DeepCopy().Annotations) waitForServiceAnnotations( expectedAnnotations(previous.Spec.ImpersonationProxy.Service.DeepCopy().Annotations, map[string]string{}), recognizableAnnotationKeyString, ) }) // Having another actor, like a human or a non-Pinniped controller, add unrelated annotations to the Service // should not cause the Pinniped controllers to overwrite those annotations. updateServiceAnnotations(otherActorAnnotations) // Set a new annotation in the CredentialIssuer spec.impersonationProxy.service.annotations field. newAnnotationKey := recognizableAnnotationKeyString + "/test-" + testlib.RandHex(t, 8) newAnnotationValue := "test-" + testlib.RandHex(t, 8) updatedAnnotations := previous.Spec.ImpersonationProxy.Service.DeepCopy().Annotations updatedAnnotations[newAnnotationKey] = newAnnotationValue applyCredentialIssuerAnnotations(updatedAnnotations) // Expect them to be applied to the Service. waitForServiceAnnotations( expectedAnnotations(updatedAnnotations, otherActorAnnotations), recognizableAnnotationKeyString, ) }) t.Run("running impersonation proxy with ClusterIP service", func(t *testing.T) { if env.Proxy == "" { t.Skip("Skipping ClusterIP test because squid proxy is not present") } clusterIPServiceURL := fmt.Sprintf("%s.%s.svc.cluster.local", impersonationProxyClusterIPName(env), env.ConciergeNamespace) updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{ ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{ Mode: conciergev1alpha.ImpersonationProxyModeEnabled, ExternalEndpoint: clusterIPServiceURL, Service: conciergev1alpha.ImpersonationProxyServiceSpec{ Type: conciergev1alpha.ImpersonationProxyServiceTypeClusterIP, }, }, }) // wait until the credential issuer is updated with the new url testlib.RequireEventuallyWithoutError(t, func() (bool, error) { newImpersonationProxyURL, _ := performImpersonatorDiscoveryURL(ctx, t, env, adminConciergeClient) return newImpersonationProxyURL == "https://"+clusterIPServiceURL, nil }, 30*time.Second, 500*time.Millisecond) newImpersonationProxyURL, newImpersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminClient, adminConciergeClient, refreshCredential) anonymousClient := newAnonymousImpersonationProxyClientWithProxy(t, newImpersonationProxyURL, newImpersonationProxyCACertPEM, nil).PinnipedConcierge refreshedCredentials := refreshCredentialHelper(t, anonymousClient) client := newImpersonationProxyClientWithCredentialsAndProxy(t, refreshedCredentials, newImpersonationProxyURL, newImpersonationProxyCACertPEM, nil).Kubernetes // everything should work properly through the cluster ip service t.Run( "access as user", testlib.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, client), ) }) t.Run("manually disabling the impersonation proxy feature", func(t *testing.T) { // Update configuration to force the proxy to disabled mode updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{ ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{ Mode: conciergev1alpha.ImpersonationProxyModeDisabled, }, }) if clusterSupportsLoadBalancers { // The load balancer should have been deleted when we disabled the impersonation proxy. // Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS). testlib.RequireEventuallyWithoutError(t, func() (bool, error) { hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient) return !hasService, err }, 2*time.Minute, 500*time.Millisecond) } // Check that the impersonation proxy port has shut down. // Ideally we could always check that the impersonation proxy's port has shut down, but on clusters where we // do not run the squid proxy we have no easy way to see beyond the load balancer to see inside the cluster, // so we'll skip this check on clusters which have load balancers but don't run the squid proxy. // The other cluster types that do run the squid proxy will give us sufficient coverage here. if env.Proxy != "" { testlib.RequireEventually(t, func(requireEventually *require.Assertions) { // It's okay if this returns RBAC errors because this user has no role bindings. // What we want to see is that the proxy eventually shuts down entirely. _, err := impersonationProxyViaSquidKubeClientWithoutCredential(t, proxyServiceEndpoint).CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) isErr, _ := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint) requireEventually.Truef(isErr, "wanted service unavailable via squid error, got %v", err) }, 20*time.Second, 500*time.Millisecond) } // Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up // when we disable the impersonator. testlib.RequireEventually(t, func(requireEventually *require.Assertions) { _, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{}) requireEventually.Truef(k8serrors.IsNotFound(err), "expected NotFound error, got %v", err) }, time.Minute, time.Second) // Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this // around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when // possible because they make their way into kubeconfigs on client machines. _, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{}) require.NoError(t, err) // At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to // include an unsuccessful impersonation strategy saying that it was manually configured to be disabled. requireDisabledStrategy(ctx, t, env, adminConciergeClient) if !env.HasCapability(testlib.ClusterSigningKeyIsAvailable) && env.HasCapability(testlib.AnonymousAuthenticationSupported) { // This cluster does not support the cluster signing key strategy, so now that we've manually disabled the // impersonation strategy, we should be left with no working strategies. // Given that there are no working strategies, a TokenCredentialRequest which would otherwise work should now // fail, because there is no point handing out credentials that are not going to work for any strategy. // Note that library.CreateTokenCredentialRequest makes an unauthenticated request, so we can't meaningfully // perform this part of the test on a cluster which does not allow anonymous authentication. tokenCredentialRequestResponse, err := testlib.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials) require.NoError(t, err) require.NotNil(t, tokenCredentialRequestResponse.Status.Message, "expected an error message but got nil") require.Equal(t, "authentication failed", *tokenCredentialRequestResponse.Status.Message) require.Nil(t, tokenCredentialRequestResponse.Status.Credential) } }) } func ensureDNSResolves(t *testing.T, urlString string) { t.Helper() parsedURL, err := url.Parse(urlString) require.NoError(t, err) host := parsedURL.Hostname() if net.ParseIP(host) != nil { return // ignore IPs } var d net.Dialer loggingDialer := func(ctx context.Context, network, address string) (net.Conn, error) { t.Logf("dns lookup, network=%s address=%s", network, address) conn, connErr := d.DialContext(ctx, network, address) if connErr != nil { t.Logf("dns lookup, err=%v", connErr) } else { local := conn.LocalAddr() remote := conn.RemoteAddr() t.Logf("dns lookup, local conn network=%s addr=%s", local.Network(), local.String()) t.Logf("dns lookup, remote conn network=%s addr=%s", remote.Network(), remote.String()) } return conn, connErr } goResolver := &net.Resolver{ PreferGo: true, StrictErrors: true, Dial: loggingDialer, } notGoResolver := &net.Resolver{ PreferGo: false, StrictErrors: true, Dial: loggingDialer, } testlib.RequireEventually(t, func(requireEventually *require.Assertions) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() for _, resolver := range []*net.Resolver{goResolver, notGoResolver} { resolver := resolver ips, ipErr := resolver.LookupIPAddr(ctx, host) requireEventually.NoError(ipErr) requireEventually.NotEmpty(ips) } }, 5*time.Minute, 1*time.Second) } func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, namespaceName string) (name, token string, uid types.UID) { t.Helper() serviceAccount, err := adminClient.CoreV1().ServiceAccounts(namespaceName).Create(ctx, &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{GenerateName: "int-test-service-account-"}}, metav1.CreateOptions{}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, adminClient.CoreV1().ServiceAccounts(namespaceName). Delete(context.Background(), serviceAccount.Name, metav1.DeleteOptions{})) }) secret, err := adminClient.CoreV1().Secrets(namespaceName).Create(ctx, &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "int-test-service-account-token-", Annotations: map[string]string{ corev1.ServiceAccountNameKey: serviceAccount.Name, }, }, Type: corev1.SecretTypeServiceAccountToken, }, metav1.CreateOptions{}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, adminClient.CoreV1().Secrets(namespaceName). Delete(context.Background(), secret.Name, metav1.DeleteOptions{})) }) testlib.RequireEventuallyWithoutError(t, func() (bool, error) { secret, err = adminClient.CoreV1().Secrets(namespaceName).Get(ctx, secret.Name, metav1.GetOptions{}) if err != nil { return false, err } return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil }, time.Minute, time.Second) return serviceAccount.Name, string(secret.Data[corev1.ServiceAccountTokenKey]), serviceAccount.UID } func expectedWhoAmIRequestResponse(username string, groups []string, extra map[string]identityv1alpha1.ExtraValue) *identityv1alpha1.WhoAmIRequest { return &identityv1alpha1.WhoAmIRequest{ Status: identityv1alpha1.WhoAmIRequestStatus{ KubernetesUserInfo: identityv1alpha1.KubernetesUserInfo{ User: identityv1alpha1.UserInfo{ Username: username, UID: "", // no way to impersonate UID: https://github.com/kubernetes/kubernetes/issues/93699 Groups: groups, Extra: extra, }, }, }, } } func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminClient kubernetes.Interface, adminConciergeClient pinnipedconciergeclientset.Interface, refreshCredential func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential) (string, []byte) { t.Helper() impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscoveryURL(ctx, t, env, adminConciergeClient) if len(env.Proxy) == 0 { t.Log("no test proxy is available, skipping readiness checks for concierge impersonation proxy pods") return impersonationProxyURL, impersonationProxyCACertPEM } impersonationProxyParsedURL, err := url.Parse(impersonationProxyURL) require.NoError(t, err) expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...) expectedGroups = append(expectedGroups, "system:authenticated") // probe each pod directly for readiness since the concierge status is a lie - it just means a single pod is ready testlib.RequireEventually(t, func(requireEventually *require.Assertions) { pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx, metav1.ListOptions{LabelSelector: "deployment.pinniped.dev=concierge"}) requireEventually.NoError(err) requireEventually.Len(pods.Items, 2) // has to stay in sync with the defaults in our YAML for _, pod := range pods.Items { t.Logf("checking if concierge impersonation proxy pod %q is ready", pod.Name) requireEventually.NotEmptyf(pod.Status.PodIP, "pod %q does not have an IP", pod.Name) credentials := refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM).DeepCopy() credentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests config := newImpersonationProxyConfigWithCredentials(t, credentials, impersonationProxyURL, impersonationProxyCACertPEM, nil) config = rest.CopyConfig(config) config.Proxy = kubeconfigProxyFunc(t, env.Proxy) // always use the proxy since we are talking directly to a pod IP config.Host = "https://" + pod.Status.PodIP + ":8444" // hardcode the internal port - it should not change config.TLSClientConfig.ServerName = impersonationProxyParsedURL.Hostname() // make SNI hostname TLS verification work even when using IP whoAmI, err := testlib.NewKubeclient(t, config).PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests(). Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{}) requireEventually.NoError(err) requireEventually.Equal( expectedWhoAmIRequestResponse( env.TestUser.ExpectedUsername, expectedGroups, nil, ), whoAmI, ) } }, 10*time.Minute, 10*time.Second) return impersonationProxyURL, impersonationProxyCACertPEM } func performImpersonatorDiscoveryURL(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface) (string, []byte) { t.Helper() var impersonationProxyURL string var impersonationProxyCACertPEM []byte t.Log("Waiting for CredentialIssuer strategy to be successful") testlib.RequireEventuallyWithoutError(t, func() (bool, error) { credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) if err != nil || credentialIssuer.Status.Strategies == nil { t.Log("Did not find any CredentialIssuer with any strategies") return false, nil // didn't find it, but keep trying } for _, strategy := range credentialIssuer.Status.Strategies { // There will be other strategy types in the list, so ignore those. if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif if strategy.Frontend == nil { return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test } if strategy.Frontend.ImpersonationProxyInfo == nil { return false, fmt.Errorf("did not find an ImpersonationProxyInfo") // unexpected, fail the test } impersonationProxyURL = strategy.Frontend.ImpersonationProxyInfo.Endpoint impersonationProxyCACertPEM, err = base64.StdEncoding.DecodeString(strategy.Frontend.ImpersonationProxyInfo.CertificateAuthorityData) if err != nil { return false, err // unexpected, fail the test } return true, nil // found it, continue the test! } else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType { t.Logf("Waiting for successful impersonation proxy strategy on %s: found status %s with reason %s and message: %s", credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason { // The server encountered an unexpected error while starting the impersonator, so fail the test fast. return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) } } } t.Log("Did not find any impersonation proxy strategy on CredentialIssuer") return false, nil // didn't find it, but keep trying }, 10*time.Minute, 10*time.Second) t.Log("Found successful CredentialIssuer strategy") return impersonationProxyURL, impersonationProxyCACertPEM } func requireDisabledStrategy(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface) { t.Helper() testlib.RequireEventuallyWithoutError(t, func() (bool, error) { credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) if err != nil || credentialIssuer.Status.Strategies == nil { t.Log("Did not find any CredentialIssuer with any strategies") return false, nil // didn't find it, but keep trying } for _, strategy := range credentialIssuer.Status.Strategies { // There will be other strategy types in the list, so ignore those. if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && strategy.Status == conciergev1alpha.ErrorStrategyStatus && strategy.Reason == conciergev1alpha.DisabledStrategyReason { return true, nil // found it, continue the test! } else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType { t.Logf("Waiting for disabled impersonation proxy strategy on %s: found status %s with reason %s and message: %s", credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message) if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason { // The server encountered an unexpected error while stopping the impersonator, so fail the test fast. return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message) } } } t.Log("Did not find any impersonation proxy strategy on CredentialIssuer") return false, nil // didn't find it, but keep trying }, 1*time.Minute, 500*time.Millisecond) } func credentialAlmostExpired(t *testing.T, credential *loginv1alpha1.TokenCredentialRequest) bool { t.Helper() pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData)) parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes) require.NoError(t, err) timeRemaining := time.Until(parsedCredential.NotAfter) if timeRemaining < 2*time.Minute { t.Logf("The TokenCredentialRequest cred is almost expired and needs to be refreshed. Expires in %s.", timeRemaining) return true } t.Logf("The TokenCredentialRequest cred is good for some more time (%s) so using it.", timeRemaining) return false } func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *rest.Config { config := rest.Config{ Host: host, TLSClientConfig: rest.TLSClientConfig{ Insecure: caData == nil, CAData: caData, CertData: []byte(credential.ClientCertificateData), KeyData: []byte(credential.ClientKeyData), }, // kubectl would set both the client cert and the token, so we'll do that too. // The Kube API server will ignore the token if the client cert successfully authenticates. // Only if the client cert is not present or fails to authenticate will it use the token. // Historically, it works that way because some web browsers will always send your // corporate-assigned client cert even if it is not valid, and it doesn't want to treat // that as a failure if you also sent a perfectly good token. // We would like the impersonation proxy to imitate that behavior, so we test it here. BearerToken: credential.Token, } if nestedImpersonationConfig != nil { config.Impersonate = *nestedImpersonationConfig } return &config } func kubeconfigProxyFunc(t *testing.T, squidProxyURL string) func(req *http.Request) (*url.URL, error) { return func(req *http.Request) (*url.URL, error) { t.Helper() parsedSquidProxyURL, err := url.Parse(squidProxyURL) require.NoError(t, err) t.Logf("passing request for %s through proxy %s", testlib.RedactURLParams(req.URL), parsedSquidProxyURL.String()) return parsedSquidProxyURL, nil } } func updateCredentialIssuer(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface, spec conciergev1alpha.CredentialIssuerSpec) { t.Helper() err := retry.RetryOnConflict(retry.DefaultRetry, func() error { newCredentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{}) if err != nil { return err } spec.DeepCopyInto(&newCredentialIssuer.Spec) _, err = adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Update(ctx, newCredentialIssuer, metav1.UpdateOptions{}) return err }) require.NoError(t, err) } func hasImpersonationProxyLoadBalancerService(ctx context.Context, env *testlib.TestEnv, client kubernetes.Interface) (bool, error) { service, err := client.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{}) if k8serrors.IsNotFound(err) { return false, nil } if err != nil { return false, err } return service.Spec.Type == corev1.ServiceTypeLoadBalancer, nil } func impersonationProxyTLSSecretName(env *testlib.TestEnv) string { return env.ConciergeAppName + "-impersonation-proxy-tls-serving-certificate" } func impersonationProxyCASecretName(env *testlib.TestEnv) string { return env.ConciergeAppName + "-impersonation-proxy-ca-certificate" } func impersonationProxyLoadBalancerName(env *testlib.TestEnv) string { return env.ConciergeAppName + "-impersonation-proxy-load-balancer" } func impersonationProxyClusterIPName(env *testlib.TestEnv) string { return env.ConciergeAppName + "-impersonation-proxy-cluster-ip" } func credentialIssuerName(env *testlib.TestEnv) string { return env.ConciergeAppName + "-config" } func getImpersonationKubeconfig(t *testing.T, env *testlib.TestEnv, impersonationProxyURL string, impersonationProxyCACertPEM []byte, authenticator corev1.TypedLocalObjectReference) (string, []string, string) { t.Helper() pinnipedExe := testlib.PinnipedCLIPath(t) tempDir := testutil.TempDir(t) var envVarsWithProxy []string if !env.HasCapability(testlib.HasExternalLoadBalancerProvider) { // Only if you don't have a load balancer, use the squid proxy when it's available. envVarsWithProxy = append(os.Environ(), env.ProxyEnv()...) } // Get the kubeconfig. getKubeConfigCmd := []string{"get", "kubeconfig", "--concierge-api-group-suffix", env.APIGroupSuffix, "--oidc-skip-browser", "--static-token", env.TestUser.Token, "--concierge-authenticator-name", authenticator.Name, "--concierge-authenticator-type", "webhook", // Force the use of impersonation proxy strategy, but let it auto-discover the endpoint and CA. "--concierge-mode", "ImpersonationProxy"} t.Log("Running:", pinnipedExe, getKubeConfigCmd) kubeconfigYAML, getKubeConfigStderr := runPinnipedCLI(t, envVarsWithProxy, pinnipedExe, getKubeConfigCmd...) // "pinniped get kubectl" prints some status messages to stderr t.Log(getKubeConfigStderr) // Make sure that the "pinniped get kubeconfig" auto-discovered the impersonation proxy and we're going to // make our kubectl requests through the impersonation proxy. Avoid using require.Contains because the error // message would contain credentials. require.True(t, strings.Contains(kubeconfigYAML, "server: "+impersonationProxyURL+"\n"), "the generated kubeconfig did not include the expected impersonation server address: %s", impersonationProxyURL, ) require.True(t, strings.Contains(kubeconfigYAML, "- --concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(impersonationProxyCACertPEM)+"\n"), "the generated kubeconfig did not include the base64 encoded version of this expected impersonation CA cert: %s", impersonationProxyCACertPEM, ) // Write the kubeconfig to a temp file. kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml") require.NoError(t, os.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600)) return kubeconfigPath, envVarsWithProxy, tempDir } // func to create kubectl commands with a kubeconfig. func kubectlCommand(timeout context.Context, t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) { t.Helper() allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...) kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...) var stdout, stderr syncBuffer kubectlCmd.Stdout = &stdout kubectlCmd.Stderr = &stderr kubectlCmd.Env = envVarsWithProxy t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " ")) return kubectlCmd, &stdout, &stderr } // Func to run kubeconfig commands. func runKubectl(t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (string, error) { timeout, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute) defer cancelFunc() kubectlCmd, stdout, stderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, args...) err := kubectlCmd.Run() t.Logf("kubectl stdout output: %s", stdout.String()) t.Logf("kubectl stderr output: %s", stderr.String()) return stdout.String(), err } // watchJSON defines the expected JSON wire equivalent of watch.Event. type watchJSON struct { Type watch.EventType `json:"type,omitempty"` Object json.RawMessage `json:"object,omitempty"` } // requireServiceUnavailableViaSquidError returns whether the provided err is the error that is // returned by squid when the impersonation proxy port inside the cluster is not listening. func isServiceUnavailableViaSquidError(err error, proxyServiceEndpoint string) (bool, string) { if err == nil { return false, "error is nil" } for _, wantContains := range []string{ fmt.Sprintf(`Get "https://%s/api/v1/namespaces"`, proxyServiceEndpoint), ": Service Unavailable", } { if !strings.Contains(err.Error(), wantContains) { return false, fmt.Sprintf("error does not contain %q", wantContains) } } return true, "" } func requireClose(t *testing.T, c chan struct{}, timeout time.Duration) { t.Helper() timer := time.NewTimer(timeout) select { case <-c: if !timer.Stop() { <-timer.C } case <-timer.C: require.FailNow(t, "failed to receive from channel within "+timeout.String()) } } func createTokenCredentialRequest( spec loginv1alpha1.TokenCredentialRequestSpec, client pinnipedconciergeclientset.Interface, ) (*loginv1alpha1.TokenCredentialRequest, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx, &loginv1alpha1.TokenCredentialRequest{Spec: spec}, metav1.CreateOptions{}, ) } func newImpersonationProxyClientWithCredentials(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client { t.Helper() kubeconfig := newImpersonationProxyConfigWithCredentials(t, credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) return testlib.NewKubeclient(t, kubeconfig) } func newImpersonationProxyConfigWithCredentials(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *rest.Config { t.Helper() env := testlib.IntegrationEnv(t) clusterSupportsLoadBalancers := env.HasCapability(testlib.HasExternalLoadBalancerProvider) kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) if !clusterSupportsLoadBalancers { // Only if there is no possibility to send traffic through a load balancer, then send the traffic through the Squid proxy. // Prefer to go through a load balancer because that's how the impersonator is intended to be used in the real world. kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) } return kubeconfig } func newAnonymousImpersonationProxyClient(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client { t.Helper() emptyCredentials := &loginv1alpha1.ClusterCredential{} return newImpersonationProxyClientWithCredentials(t, emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) } func newImpersonationProxyClientWithCredentialsAndProxy(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client { t.Helper() env := testlib.IntegrationEnv(t) kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) return testlib.NewKubeclient(t, kubeconfig) } // this uses a proxy in all cases, the other will only use it if you don't have load balancer capabilities. func newAnonymousImpersonationProxyClientWithProxy(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client { t.Helper() env := testlib.IntegrationEnv(t) emptyCredentials := &loginv1alpha1.ClusterCredential{} kubeconfig := impersonationProxyRestConfig(emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) return testlib.NewKubeclient(t, kubeconfig) } func impersonationProxyViaSquidKubeClientWithoutCredential(t *testing.T, proxyServiceEndpoint string) kubernetes.Interface { t.Helper() env := testlib.IntegrationEnv(t) proxyURL := "https://" + proxyServiceEndpoint kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, nil) kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy) return testlib.NewKubeclient(t, kubeconfig).Kubernetes } func newImpersonationProxyClient( t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig, refreshCredentialFunc func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential, ) *kubeclient.Client { t.Helper() refreshedCredentials := refreshCredentialFunc(t, impersonationProxyURL, impersonationProxyCACertPEM).DeepCopy() refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests return newImpersonationProxyClientWithCredentials(t, refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig) } // getCredForConfig is mostly just a hacky workaround for impersonationProxyRestConfig needing creds directly. func getCredForConfig(t *testing.T, config *rest.Config) *loginv1alpha1.ClusterCredential { t.Helper() out := &loginv1alpha1.ClusterCredential{} config = rest.CopyConfig(config) config.Wrap(func(rt http.RoundTripper) http.RoundTripper { return roundtripper.WrapFunc(rt, func(req *http.Request) (*http.Response, error) { resp, err := rt.RoundTrip(req) r := req if resp != nil && resp.Request != nil { r = resp.Request } _, _, _ = bearertoken.New(authenticator.TokenFunc(func(_ context.Context, token string) (*authenticator.Response, bool, error) { out.Token = token return nil, false, nil })).AuthenticateRequest(r) return resp, err }) }) transportConfig, err := config.TransportConfig() require.NoError(t, err) rt, err := transport.New(transportConfig) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() req, err := http.NewRequestWithContext(ctx, "GET", "https://localhost", nil) require.NoError(t, err) resp, _ := rt.RoundTrip(req) if resp != nil && resp.Body != nil { _ = resp.Body.Close() } tlsConfig, err := transport.TLSConfigFor(transportConfig) require.NoError(t, err) if tlsConfig != nil && tlsConfig.GetClientCertificate != nil { cert, err := tlsConfig.GetClientCertificate(nil) require.NoError(t, err) if len(cert.Certificate) > 0 { require.Len(t, cert.Certificate, 1) publicKey := pem.EncodeToMemory(&pem.Block{ Type: "CERTIFICATE", Bytes: cert.Certificate[0], }) out.ClientCertificateData = string(publicKey) privateKey, err := keyutil.MarshalPrivateKeyToPEM(cert.PrivateKey) require.NoError(t, err) out.ClientKeyData = string(privateKey) } } if *out == (loginv1alpha1.ClusterCredential{}) { t.Fatal("failed to get creds for config") } return out } func getUIDAndExtraViaCSR(ctx context.Context, t *testing.T, uid string, client kubernetes.Interface) (string, map[string][]string) { t.Helper() privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) require.NoError(t, err) csrPEM, err := cert.MakeCSR(privateKey, &pkix.Name{ CommonName: "panda-man", Organization: []string{"living-the-dream", "need-more-sleep"}, }, nil, nil) require.NoError(t, err) csrName, _, err := csr.RequestCertificate( client, csrPEM, "", certificatesv1.KubeAPIServerClientSignerName, nil, []certificatesv1.KeyUsage{certificatesv1.UsageClientAuth}, privateKey, ) require.NoError(t, err) outUID := uid // in the future this may not be empty on some clusters extrasAsStrings := map[string][]string{} if testutil.KubeServerSupportsCertificatesV1API(t, client.Discovery()) { csReq, err := client.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) require.NoError(t, err) err = client.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}) require.NoError(t, err) if len(outUID) == 0 { outUID = csReq.Spec.UID } // Convert each `ExtraValue` to `[]string` to return, so we don't have to deal with v1beta1 types versus v1 types for k, v := range csReq.Spec.Extra { extrasAsStrings[k] = v } } else { // On old Kubernetes clusters use CertificatesV1beta1 csReq, err := client.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) require.NoError(t, err) err = client.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}) require.NoError(t, err) if len(outUID) == 0 { outUID = csReq.Spec.UID } // Convert each `ExtraValue` to `[]string` to return, so we don't have to deal with v1beta1 types versus v1 types for k, v := range csReq.Spec.Extra { extrasAsStrings[k] = v } } return outUID, extrasAsStrings } func parallelIfNotEKS(t *testing.T) { if testlib.IntegrationEnv(t).KubernetesDistribution == testlib.EKSDistro { return } t.Parallel() }