From 609b55a6d70549186da6dc42a83d45bdaaf8de76 Mon Sep 17 00:00:00 2001 From: Margo Crawford Date: Tue, 1 Feb 2022 16:50:30 -0800 Subject: [PATCH] Pinniped Supervisor should issue a warning when groups change during refresh --- internal/oidc/token/token_handler.go | 32 ++ internal/oidc/token/token_handler_test.go | 49 +++ test/integration/supervisor_warnings_test.go | 439 +++++++++++++++++++ 3 files changed, 520 insertions(+) create mode 100644 test/integration/supervisor_warnings_test.go diff --git a/internal/oidc/token/token_handler.go b/internal/oidc/token/token_handler.go index 55e73352..5cd82ada 100644 --- a/internal/oidc/token/token_handler.go +++ b/internal/oidc/token/token_handler.go @@ -7,11 +7,13 @@ package token import ( "context" "errors" + "fmt" "net/http" "github.com/ory/fosite" "github.com/ory/x/errorsx" "golang.org/x/oauth2" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/warning" "go.pinniped.dev/internal/httputil/httperr" @@ -187,6 +189,15 @@ func upstreamOIDCRefresh(ctx context.Context, session *psession.PinnipedSession, WithDebugf("provider name: %q, provider type: %q", s.ProviderName, s.ProviderType)) } if refreshedGroups != nil { + oldGroups, err := getDownstreamGroupsFromPinnipedSession(session) + if err != nil { + return err + } + username, err := getDownstreamUsernameFromPinnipedSession(session) + if err != nil { + return err + } + warnIfGroupsChanged(ctx, oldGroups, refreshedGroups, username) session.Fosite.Claims.Extra[oidc.DownstreamGroupsClaim] = refreshedGroups } @@ -202,6 +213,15 @@ func upstreamOIDCRefresh(ctx context.Context, session *psession.PinnipedSession, return nil } +// print out the diff between two lists of sorted groups. +func diffSortedGroups(oldGroups, newGroups []string) ([]string, []string) { + oldGroupsAsSet := sets.NewString(oldGroups...) + newGroupsAsSet := sets.NewString(newGroups...) + added := newGroupsAsSet.Difference(oldGroupsAsSet) // groups in newGroups that are not in oldGroups i.e. added + removed := oldGroupsAsSet.Difference(newGroupsAsSet) // groups in oldGroups that are not in newGroups i.e. removed + return added.List(), removed.List() +} + func validateIdentityUnchangedSinceInitialLogin(mergedClaims map[string]interface{}, session *psession.PinnipedSession, usernameClaimName string) error { s := session.Custom @@ -320,6 +340,8 @@ func upstreamLDAPRefresh(ctx context.Context, providerCache oidc.UpstreamIdentit // Replace the old value with the new value. session.Fosite.Claims.Extra[oidc.DownstreamGroupsClaim] = groups + warnIfGroupsChanged(ctx, oldGroups, groups, username) + return nil } @@ -393,3 +415,13 @@ func getDownstreamGroupsFromPinnipedSession(session *psession.PinnipedSession) ( } return downstreamGroups, nil } + +func warnIfGroupsChanged(ctx context.Context, oldGroups []string, refreshedGroups []string, username string) { + added, removed := diffSortedGroups(oldGroups, refreshedGroups) + if len(added) > 0 { + warning.AddWarning(ctx, "", fmt.Sprintf("User %q has been added to the following groups: %q", username, added)) + } + if len(removed) > 0 { + warning.AddWarning(ctx, "", fmt.Sprintf("User %q has been removed from the following groups: %q: ", username, removed)) + } +} diff --git a/internal/oidc/token/token_handler_test.go b/internal/oidc/token/token_handler_test.go index a81c3972..4e1bb96d 100644 --- a/internal/oidc/token/token_handler_test.go +++ b/internal/oidc/token/token_handler_test.go @@ -3624,3 +3624,52 @@ func toSliceOfInterface(s []string) []interface{} { } return r } + +func TestDiffSortedGroups(t *testing.T) { + tests := []struct { + name string + oldGroups []string + newGroups []string + wantAdded []string + wantRemoved []string + }{ + { + name: "groups were added", + oldGroups: []string{"b", "c"}, + newGroups: []string{"a", "b", "bb", "c", "d"}, + wantAdded: []string{"a", "bb", "d"}, + wantRemoved: []string{}, + }, + { + name: "groups were removed", + oldGroups: []string{"a", "b", "bb", "c", "d"}, + newGroups: []string{"b", "c"}, + wantAdded: []string{}, + wantRemoved: []string{"a", "bb", "d"}, + }, + { + name: "groups were added and removed", + oldGroups: []string{"a", "c"}, + newGroups: []string{"b", "c", "d"}, + wantAdded: []string{"b", "d"}, + wantRemoved: []string{"a"}, + }, + { + name: "groups are exactly the same", + oldGroups: []string{"a", "b", "c"}, + newGroups: []string{"a", "b", "c"}, + wantAdded: []string{}, + wantRemoved: []string{}, + }, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + added, removed := diffSortedGroups(test.oldGroups, test.newGroups) + require.Equal(t, test.wantAdded, added) + require.Equal(t, test.wantRemoved, removed) + }) + } +} diff --git a/test/integration/supervisor_warnings_test.go b/test/integration/supervisor_warnings_test.go new file mode 100644 index 00000000..735926d1 --- /dev/null +++ b/test/integration/supervisor_warnings_test.go @@ -0,0 +1,439 @@ +// Copyright 2022 the Pinniped contributors. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +package integration + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" + "testing" + "time" + + coreosoidc "github.com/coreos/go-oidc/v3/oidc" + "github.com/creack/pty" + "github.com/stretchr/testify/require" + authorizationv1 "k8s.io/api/authorization/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + + authv1alpha "go.pinniped.dev/generated/latest/apis/concierge/authentication/v1alpha1" + configv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/config/v1alpha1" + idpv1alpha1 "go.pinniped.dev/generated/latest/apis/supervisor/idp/v1alpha1" + "go.pinniped.dev/internal/certauthority" + "go.pinniped.dev/internal/oidc" + "go.pinniped.dev/internal/psession" + "go.pinniped.dev/internal/testutil" + "go.pinniped.dev/pkg/oidcclient" + "go.pinniped.dev/pkg/oidcclient/filesession" + "go.pinniped.dev/test/testlib" + "go.pinniped.dev/test/testlib/browsertest" +) + +func TestSupervisorWarnings_Browser(t *testing.T) { + env := testlib.IntegrationEnv(t) + + ctx, cancelFunc := context.WithTimeout(context.Background(), 10*time.Minute) + defer cancelFunc() + + // Build pinniped CLI. + pinnipedExe := testlib.PinnipedCLIPath(t) + tempDir := testutil.TempDir(t) + + // Infer the downstream issuer URL from the callback associated with the upstream test client registration. + issuerURL, err := url.Parse(env.SupervisorUpstreamOIDC.CallbackURL) + require.NoError(t, err) + require.True(t, strings.HasSuffix(issuerURL.Path, "/callback")) + issuerURL.Path = strings.TrimSuffix(issuerURL.Path, "/callback") + t.Logf("testing with downstream issuer URL %s", issuerURL.String()) + + // Generate a CA bundle with which to serve this provider. + t.Logf("generating test CA") + ca, err := certauthority.New("Downstream Test CA", 1*time.Hour) + require.NoError(t, err) + + // Save that bundle plus the one that signs the upstream issuer, for test purposes. + testCABundlePath := filepath.Join(tempDir, "test-ca.pem") + testCABundlePEM := []byte(string(ca.Bundle()) + "\n" + env.SupervisorUpstreamOIDC.CABundle) + testCABundleBase64 := base64.StdEncoding.EncodeToString(testCABundlePEM) + require.NoError(t, ioutil.WriteFile(testCABundlePath, testCABundlePEM, 0600)) + + // Use the CA to issue a TLS server cert. + t.Logf("issuing test certificate") + tlsCert, err := ca.IssueServerCert([]string{issuerURL.Hostname()}, nil, 1*time.Hour) + require.NoError(t, err) + certPEM, keyPEM, err := certauthority.ToPEM(tlsCert) + require.NoError(t, err) + + // Write the serving cert to a secret. + certSecret := testlib.CreateTestSecret(t, + env.SupervisorNamespace, + "oidc-provider-tls", + corev1.SecretTypeTLS, + map[string]string{"tls.crt": string(certPEM), "tls.key": string(keyPEM)}, + ) + + // Create the downstream FederationDomain and expect it to go into the success status condition. + downstream := testlib.CreateTestFederationDomain(ctx, t, + issuerURL.String(), + certSecret.Name, + configv1alpha1.SuccessFederationDomainStatusCondition, + ) + + // Create a JWTAuthenticator that will validate the tokens from the downstream issuer. + clusterAudience := "test-cluster-" + testlib.RandHex(t, 8) + authenticator := testlib.CreateTestJWTAuthenticator(ctx, t, authv1alpha.JWTAuthenticatorSpec{ + Issuer: downstream.Spec.Issuer, + Audience: clusterAudience, + TLS: &authv1alpha.TLSSpec{CertificateAuthorityData: testCABundleBase64}, + }) + + t.Run("LDAP group refresh flow", func(t *testing.T) { + if len(env.ToolsNamespace) == 0 && !env.HasCapability(testlib.CanReachInternetLDAPPorts) { + t.Skip("LDAP integration test requires connectivity to an LDAP server") + } + + expectedUsername := env.SupervisorUpstreamLDAP.TestUserMailAttributeValue + + setupClusterForEndToEndLDAPTest(t, expectedUsername, env) + + // Use a specific session cache for this test. + sessionCachePath := tempDir + "/ldap-test-refresh-sessions.yaml" + credentialCachePath := tempDir + "/ldap-test-refresh-credentials.yaml" + kubeconfigPath := runPinnipedGetKubeconfig(t, env, pinnipedExe, tempDir, []string{ + "get", "kubeconfig", + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--concierge-authenticator-type", "jwt", + "--concierge-authenticator-name", authenticator.Name, + "--oidc-session-cache", sessionCachePath, + "--credential-cache", credentialCachePath, + }) + + // Run "kubectl get namespaces" which should trigger a cli-based login. + start := time.Now() + kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) + kubectlCmd.Env = append(os.Environ(), env.ProxyEnv()...) + var kubectlStdoutPipe io.ReadCloser + if runtime.GOOS != "darwin" { + // For some unknown reason this breaks the pty library on some MacOS machines. + // The problem doesn't reproduce for everyone, so this is just a workaround. + kubectlStdoutPipe, err = kubectlCmd.StdoutPipe() + require.NoError(t, err) + } + ptyFile, err := pty.Start(kubectlCmd) + require.NoError(t, err) + + // Wait for the subprocess to print the username prompt, then type the user's username. + readFromFileUntilStringIsSeen(t, ptyFile, "Username: ") + _, err = ptyFile.WriteString(expectedUsername + "\n") + require.NoError(t, err) + + // Wait for the subprocess to print the password prompt, then type the user's password. + readFromFileUntilStringIsSeen(t, ptyFile, "Password: ") + _, err = ptyFile.WriteString(env.SupervisorUpstreamLDAP.TestUserPassword + "\n") + require.NoError(t, err) + + // Read all of the remaining output from the subprocess until EOF. + t.Logf("waiting for kubectl to output namespace list") + // Read all output from the subprocess until EOF. + // Ignore any errors returned because there is always an error on linux. + kubectlPtyOutputBytes, _ := ioutil.ReadAll(ptyFile) + if kubectlStdoutPipe != nil { + // On non-MacOS check that stdout of the CLI contains the expected output. + kubectlStdOutOutputBytes, _ := ioutil.ReadAll(kubectlStdoutPipe) + requireKubectlGetNamespaceOutput(t, env, string(kubectlStdOutOutputBytes)) + } else { + // On MacOS check that the pty (stdout+stderr+stdin) of the CLI contains the expected output. + requireKubectlGetNamespaceOutput(t, env, string(kubectlPtyOutputBytes)) + } + + t.Logf("first kubectl command took %s", time.Since(start).String()) + + // To simulate the groups having changed without actually changing the groups the user belongs to in the LDAP + // server, we update the refresh token secret to have a different value for the groups. + // Then the refresh flow will update them back to their real values. + // To do this, we get the refresh token signature out of the cache, use it to get the Secret, update it, and + // put it back. + cache := filesession.New(sessionCachePath, filesession.WithErrorReporter(func(err error) { + require.NoError(t, err) + })) + + // construct the cache key + downstreamScopes := []string{coreosoidc.ScopeOfflineAccess, coreosoidc.ScopeOpenID, "pinniped:request-audience"} + sort.Strings(downstreamScopes) + sessionCacheKey := oidcclient.SessionCacheKey{ + Issuer: downstream.Spec.Issuer, + ClientID: "pinniped-cli", + Scopes: downstreamScopes, + RedirectURI: "http://localhost:0/callback", + } + // use it to get the cache entry + token := cache.GetToken(sessionCacheKey) + require.NotNil(t, token) + + // using the refresh token signature contained in the cache, get the refresh token session + // out of kube secret storage. + kubeClient := testlib.NewKubernetesClientset(t).CoreV1() + refreshTokenSignature := strings.Split(token.RefreshToken.Token, ".")[1] + oauthStore := oidc.NewKubeStorage(kubeClient.Secrets(env.SupervisorNamespace), oidc.DefaultOIDCTimeoutsConfiguration()) + storedRefreshSession, err := oauthStore.GetRefreshTokenSession(ctx, refreshTokenSignature, nil) + require.NoError(t, err) + + // change the groups to simulate them changing in the IDP. + pinnipedSession, ok := storedRefreshSession.GetSession().(*psession.PinnipedSession) + require.True(t, ok, "should have been able to cast session data to PinnipedSession") + pinnipedSession.Fosite.Claims.Extra["groups"] = []string{"some-wrong-group", "some-other-group"} + + require.NoError(t, oauthStore.DeleteRefreshTokenSession(ctx, refreshTokenSignature)) + require.NoError(t, oauthStore.CreateRefreshTokenSession(ctx, refreshTokenSignature, storedRefreshSession)) + + // remove the credential cache, which includes the cached cert, so it won't be reused and the refresh flow will be triggered. + err = os.Remove(credentialCachePath) + require.NoError(t, err) + + // wait for the existing tokens to expire, triggering the refresh flow. + ctx2, cancel2 := context.WithTimeout(ctx, 1*time.Minute) + defer cancel2() + + // Run kubectl, which should work without any prompting for authentication. + kubectlCmd2 := exec.CommandContext(ctx2, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) + kubectlCmd2.Env = append(os.Environ(), env.ProxyEnv()...) + startTime2 := time.Now() + var kubectlStdoutPipe2 io.ReadCloser + if runtime.GOOS != "darwin" { + // For some unknown reason this breaks the pty library on some MacOS machines. + // The problem doesn't reproduce for everyone, so this is just a workaround. + kubectlStdoutPipe2, err = kubectlCmd2.StdoutPipe() + require.NoError(t, err) + } + ptyFile2, err := pty.Start(kubectlCmd2) + require.NoError(t, err) + + // Read all of the remaining output from the subprocess until EOF. + t.Logf("waiting for kubectl to output namespace list") + // Read all output from the subprocess until EOF. + // Ignore any errors returned because there is always an error on linux. + kubectlPtyOutputBytes2, _ := ioutil.ReadAll(ptyFile2) + if kubectlStdoutPipe2 != nil { + // On non-MacOS check that stdout of the CLI contains the expected output. + kubectlStdOutOutputBytes2, _ := ioutil.ReadAll(kubectlStdoutPipe2) + requireKubectlGetNamespaceOutput(t, env, string(kubectlStdOutOutputBytes2)) + } else { + // On MacOS check that the pty (stdout+stderr+stdin) of the CLI contains the expected output. + requireKubectlGetNamespaceOutput(t, env, string(kubectlPtyOutputBytes2)) + } + // the output should include a warning that the groups have changed. + require.Contains(t, string(kubectlPtyOutputBytes2), fmt.Sprintf("User %q has been added to the following groups: %q", env.SupervisorUpstreamLDAP.TestUserMailAttributeValue, env.SupervisorUpstreamLDAP.TestUserDirectGroupsDNs)) + require.Contains(t, string(kubectlPtyOutputBytes2), fmt.Sprintf("User %q has been removed from the following groups: [\"some-other-group\" \"some-wrong-group\"]", env.SupervisorUpstreamLDAP.TestUserMailAttributeValue)) + + t.Logf("second kubectl command took %s", time.Since(startTime2).String()) + }) + + t.Run("OIDC group refresh flow", func(t *testing.T) { + if len(env.SupervisorUpstreamOIDC.ExpectedGroups) == 0 { + t.Skip("Skipping OIDC group refresh test since there are no groups") + } + + // Start a fresh browser driver because we don't want to share cookies between the various tests in this file. + page := browsertest.Open(t) + + expectedUsername := env.SupervisorUpstreamOIDC.Username + + // Create a ClusterRoleBinding to give our test user from the upstream read-only access to the cluster. + testlib.CreateTestClusterRoleBinding(t, + rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: expectedUsername}, + rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "view"}, + ) + testlib.WaitForUserToHaveAccess(t, expectedUsername, []string{}, &authorizationv1.ResourceAttributes{ + Verb: "get", + Group: "", + Version: "v1", + Resource: "namespaces", + }) + + // Create upstream OIDC provider and wait for it to become ready. + testlib.CreateTestOIDCIdentityProvider(t, idpv1alpha1.OIDCIdentityProviderSpec{ + Issuer: env.SupervisorUpstreamOIDC.Issuer, + TLS: &idpv1alpha1.TLSSpec{ + CertificateAuthorityData: base64.StdEncoding.EncodeToString([]byte(env.SupervisorUpstreamOIDC.CABundle)), + }, + AuthorizationConfig: idpv1alpha1.OIDCAuthorizationConfig{ + AdditionalScopes: env.SupervisorUpstreamOIDC.AdditionalScopes, + }, + Claims: idpv1alpha1.OIDCClaims{ + Username: env.SupervisorUpstreamOIDC.UsernameClaim, + Groups: env.SupervisorUpstreamOIDC.GroupsClaim, + }, + Client: idpv1alpha1.OIDCClient{ + SecretName: testlib.CreateClientCredsSecret(t, env.SupervisorUpstreamOIDC.ClientID, env.SupervisorUpstreamOIDC.ClientSecret).Name, + }, + }, idpv1alpha1.PhaseReady) + + // Use a specific session cache for this test. + sessionCachePath := tempDir + "/ldap-test-refresh-sessions.yaml" + credentialCachePath := tempDir + "/ldap-test-refresh-credentials.yaml" + kubeconfigPath := runPinnipedGetKubeconfig(t, env, pinnipedExe, tempDir, []string{ + "get", "kubeconfig", + "--concierge-api-group-suffix", env.APIGroupSuffix, + "--concierge-authenticator-type", "jwt", + "--concierge-authenticator-name", authenticator.Name, + "--oidc-skip-browser", + "--oidc-skip-listen", + "--oidc-ca-bundle", testCABundlePath, + "--oidc-session-cache", sessionCachePath, + "--credential-cache", credentialCachePath, + }) + + // Run "kubectl get namespaces" which should trigger a cli-based login. + start := time.Now() + kubectlCmd := exec.CommandContext(ctx, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) + kubectlCmd.Env = append(os.Environ(), env.ProxyEnv()...) + var kubectlStdoutPipe io.ReadCloser + if runtime.GOOS != "darwin" { + // For some unknown reason this breaks the pty library on some MacOS machines. + // The problem doesn't reproduce for everyone, so this is just a workaround. + kubectlStdoutPipe, err = kubectlCmd.StdoutPipe() + require.NoError(t, err) + } + ptyFile, err := pty.Start(kubectlCmd) + require.NoError(t, err) + + // Wait for the subprocess to print the login prompt. + t.Logf("waiting for CLI to output login URL and manual prompt") + output := readFromFileUntilStringIsSeen(t, ptyFile, "Optionally, paste your authorization code: ") + require.Contains(t, output, "Log in by visiting this link:") + require.Contains(t, output, "Optionally, paste your authorization code: ") + + // Find the line with the login URL. + var loginURL string + for _, line := range strings.Split(output, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "https://") { + loginURL = trimmed + } + } + require.NotEmptyf(t, loginURL, "didn't find login URL in output: %s", output) + + t.Logf("navigating to login page") + require.NoError(t, page.Navigate(loginURL)) + + // Expect to be redirected to the upstream provider and log in. + browsertest.LoginToUpstream(t, page, env.SupervisorUpstreamOIDC) + + // Expect to be redirected to the downstream callback which is serving the form_post HTML. + t.Logf("waiting for response page %s", downstream.Spec.Issuer) + browsertest.WaitForURL(t, page, regexp.MustCompile(regexp.QuoteMeta(downstream.Spec.Issuer))) + + // The response page should have failed to automatically post, and should now be showing the manual instructions. + authCode := formpostExpectManualState(t, page) + + // Enter the auth code in the waiting prompt, followed by a newline. + t.Logf("'manually' pasting authorization code %q to waiting prompt", authCode) + _, err = ptyFile.WriteString(authCode + "\n") + require.NoError(t, err) + + // Read all of the remaining output from the subprocess until EOF. + t.Logf("waiting for kubectl to output namespace list") + // Read all output from the subprocess until EOF. + // Ignore any errors returned because there is always an error on linux. + kubectlPtyOutputBytes, _ := ioutil.ReadAll(ptyFile) + if kubectlStdoutPipe != nil { + // On non-MacOS check that stdout of the CLI contains the expected output. + kubectlStdOutOutputBytes, _ := ioutil.ReadAll(kubectlStdoutPipe) + requireKubectlGetNamespaceOutput(t, env, string(kubectlStdOutOutputBytes)) + } else { + // On MacOS check that the pty (stdout+stderr+stdin) of the CLI contains the expected output. + requireKubectlGetNamespaceOutput(t, env, string(kubectlPtyOutputBytes)) + } + + t.Logf("first kubectl command took %s", time.Since(start).String()) + + // To simulate the groups having changed without actually changing the groups the user belongs to in the LDAP + // server, we update the refresh token secret to have a different value for the groups. + // Then the refresh flow will update them back to their real values. + // To do this, we get the refresh token signature out of the cache, use it to get the Secret, update it, and + // put it back. + cache := filesession.New(sessionCachePath, filesession.WithErrorReporter(func(err error) { + require.NoError(t, err) + })) + + // construct the cache key + downstreamScopes := []string{coreosoidc.ScopeOfflineAccess, coreosoidc.ScopeOpenID, "pinniped:request-audience"} + sort.Strings(downstreamScopes) + sessionCacheKey := oidcclient.SessionCacheKey{ + Issuer: downstream.Spec.Issuer, + ClientID: "pinniped-cli", + Scopes: downstreamScopes, + RedirectURI: "http://localhost:0/callback", + } + // use it to get the cache entry + token := cache.GetToken(sessionCacheKey) + require.NotNil(t, token) + + // using the refresh token signature contained in the cache, get the refresh token session + // out of kube secret storage. + kubeClient := testlib.NewKubernetesClientset(t).CoreV1() + refreshTokenSignature := strings.Split(token.RefreshToken.Token, ".")[1] + oauthStore := oidc.NewKubeStorage(kubeClient.Secrets(env.SupervisorNamespace), oidc.DefaultOIDCTimeoutsConfiguration()) + storedRefreshSession, err := oauthStore.GetRefreshTokenSession(ctx, refreshTokenSignature, nil) + require.NoError(t, err) + + // change the groups to simulate them changing in the IDP. + pinnipedSession, ok := storedRefreshSession.GetSession().(*psession.PinnipedSession) + require.True(t, ok, "should have been able to cast session data to PinnipedSession") + pinnipedSession.Fosite.Claims.Extra["groups"] = []string{"some-wrong-group", "some-other-group"} + + require.NoError(t, oauthStore.DeleteRefreshTokenSession(ctx, refreshTokenSignature)) + require.NoError(t, oauthStore.CreateRefreshTokenSession(ctx, refreshTokenSignature, storedRefreshSession)) + + // remove the credential cache, which includes the cached cert, so it won't be reused and the refresh flow will be triggered. + err = os.Remove(credentialCachePath) + require.NoError(t, err) + + // wait for the existing tokens to expire, triggering the refresh flow. + ctx2, cancel2 := context.WithTimeout(ctx, 1*time.Minute) + defer cancel2() + + // Run kubectl, which should work without any prompting for authentication. + kubectlCmd2 := exec.CommandContext(ctx2, "kubectl", "get", "namespace", "--kubeconfig", kubeconfigPath) + kubectlCmd2.Env = append(os.Environ(), env.ProxyEnv()...) + startTime2 := time.Now() + var kubectlStdoutPipe2 io.ReadCloser + if runtime.GOOS != "darwin" { + // For some unknown reason this breaks the pty library on some MacOS machines. + // The problem doesn't reproduce for everyone, so this is just a workaround. + kubectlStdoutPipe2, err = kubectlCmd2.StdoutPipe() + require.NoError(t, err) + } + ptyFile2, err := pty.Start(kubectlCmd2) + require.NoError(t, err) + + // Read all of the remaining output from the subprocess until EOF. + t.Logf("waiting for kubectl to output namespace list") + // Read all output from the subprocess until EOF. + // Ignore any errors returned because there is always an error on linux. + kubectlPtyOutputBytes2, _ := ioutil.ReadAll(ptyFile2) + if kubectlStdoutPipe2 != nil { + // On non-MacOS check that stdout of the CLI contains the expected output. + kubectlStdOutOutputBytes2, _ := ioutil.ReadAll(kubectlStdoutPipe2) + requireKubectlGetNamespaceOutput(t, env, string(kubectlStdOutOutputBytes2)) + } else { + // On MacOS check that the pty (stdout+stderr+stdin) of the CLI contains the expected output. + requireKubectlGetNamespaceOutput(t, env, string(kubectlPtyOutputBytes2)) + } + // the output should include a warning that the groups have changed. + require.Contains(t, string(kubectlPtyOutputBytes2), fmt.Sprintf("User %q has been added to the following groups: %q", env.SupervisorUpstreamOIDC.Username, env.SupervisorUpstreamOIDC.ExpectedGroups)) + require.Contains(t, string(kubectlPtyOutputBytes2), fmt.Sprintf("User %q has been removed from the following groups: [\"some-other-group\" \"some-wrong-group\"]", env.SupervisorUpstreamOIDC.Username)) + + t.Logf("second kubectl command took %s", time.Since(startTime2).String()) + }) +}