2512 lines
114 KiB
Go
2512 lines
114 KiB
Go
// Copyright 2020-2022 the Pinniped contributors. All Rights Reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package integration
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/ecdsa"
|
|
"crypto/elliptic"
|
|
"crypto/rand"
|
|
"crypto/x509"
|
|
"crypto/x509/pkix"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"encoding/pem"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/gorilla/websocket"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/net/http2"
|
|
authenticationv1 "k8s.io/api/authentication/v1"
|
|
authorizationv1 "k8s.io/api/authorization/v1"
|
|
certificatesv1 "k8s.io/api/certificates/v1"
|
|
certificatesv1beta1 "k8s.io/api/certificates/v1beta1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
rbacv1 "k8s.io/api/rbac/v1"
|
|
"k8s.io/apimachinery/pkg/api/equality"
|
|
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
|
|
"k8s.io/apimachinery/pkg/labels"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/watch"
|
|
"k8s.io/apiserver/pkg/authentication/authenticator"
|
|
"k8s.io/apiserver/pkg/authentication/request/bearertoken"
|
|
"k8s.io/apiserver/pkg/authentication/serviceaccount"
|
|
"k8s.io/apiserver/pkg/authentication/user"
|
|
k8sinformers "k8s.io/client-go/informers"
|
|
"k8s.io/client-go/kubernetes"
|
|
"k8s.io/client-go/rest"
|
|
"k8s.io/client-go/transport"
|
|
"k8s.io/client-go/util/cert"
|
|
"k8s.io/client-go/util/certificate/csr"
|
|
"k8s.io/client-go/util/keyutil"
|
|
"k8s.io/client-go/util/retry"
|
|
"k8s.io/utils/pointer"
|
|
|
|
conciergev1alpha "go.pinniped.dev/generated/latest/apis/concierge/config/v1alpha1"
|
|
identityv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/identity/v1alpha1"
|
|
loginv1alpha1 "go.pinniped.dev/generated/latest/apis/concierge/login/v1alpha1"
|
|
pinnipedconciergeclientset "go.pinniped.dev/generated/latest/client/concierge/clientset/versioned"
|
|
"go.pinniped.dev/internal/crypto/ptls"
|
|
"go.pinniped.dev/internal/httputil/roundtripper"
|
|
"go.pinniped.dev/internal/kubeclient"
|
|
"go.pinniped.dev/internal/testutil"
|
|
"go.pinniped.dev/test/testlib"
|
|
)
|
|
|
|
// syncBuffer wraps bytes.Buffer with a mutex so we don't have races in our test code.
|
|
type syncBuffer struct {
|
|
buf bytes.Buffer
|
|
mu sync.Mutex
|
|
}
|
|
|
|
func (sb *syncBuffer) String() string {
|
|
sb.mu.Lock()
|
|
defer sb.mu.Unlock()
|
|
return sb.buf.String()
|
|
}
|
|
|
|
func (sb *syncBuffer) Read(b []byte) (int, error) {
|
|
sb.mu.Lock()
|
|
defer sb.mu.Unlock()
|
|
return sb.buf.Read(b)
|
|
}
|
|
|
|
func (sb *syncBuffer) Write(b []byte) (int, error) {
|
|
sb.mu.Lock()
|
|
defer sb.mu.Unlock()
|
|
return sb.buf.Write(b)
|
|
}
|
|
|
|
// Note that this test supports being run on all of our integration test cluster types:
|
|
// - TKGS acceptance (long-running) cluster: auto mode will choose disabled, supports LBs, does not have squid.
|
|
// - GKE acceptance (long-running) cluster: auto will choose enabled, support LBs, does not have squid.
|
|
// - kind: auto mode will choose disabled, does not support LBs, has squid.
|
|
// - GKE ephemeral clusters: auto mode will choose enabled, supports LBs, has squid.
|
|
// - AKS ephemeral clusters: auto mode will choose enabled, supports LBs, has squid.
|
|
// - EKS ephemeral clusters: auto mode will choose enabled, supports LBs, has squid.
|
|
func TestImpersonationProxy(t *testing.T) { //nolint:gocyclo // yeah, it's complex.
|
|
env := testlib.IntegrationEnv(t)
|
|
|
|
impersonatorShouldHaveStartedAutomaticallyByDefault := !env.HasCapability(testlib.ClusterSigningKeyIsAvailable)
|
|
clusterSupportsLoadBalancers := env.HasCapability(testlib.HasExternalLoadBalancerProvider)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
|
|
defer cancel()
|
|
|
|
// Create a client using the admin kubeconfig.
|
|
adminClient := testlib.NewKubernetesClientset(t)
|
|
adminConciergeClient := testlib.NewConciergeClientset(t)
|
|
|
|
// Create a WebhookAuthenticator and prepare a TokenCredentialRequestSpec using the authenticator for use later.
|
|
credentialRequestSpecWithWorkingCredentials := loginv1alpha1.TokenCredentialRequestSpec{
|
|
Token: env.TestUser.Token,
|
|
Authenticator: testlib.CreateTestWebhookAuthenticator(ctx, t),
|
|
}
|
|
|
|
// The address of the ClusterIP service that points at the impersonation proxy's port (used when there is no load balancer).
|
|
proxyServiceEndpoint := fmt.Sprintf("%s-proxy.%s.svc.cluster.local", env.ConciergeAppName, env.ConciergeNamespace)
|
|
|
|
var (
|
|
mostRecentTokenCredentialRequestResponse *loginv1alpha1.TokenCredentialRequest
|
|
mostRecentTokenCredentialRequestResponseLock sync.Mutex
|
|
)
|
|
|
|
refreshCredentialHelper := func(t *testing.T, client pinnipedconciergeclientset.Interface) *loginv1alpha1.ClusterCredential {
|
|
t.Helper()
|
|
|
|
mostRecentTokenCredentialRequestResponseLock.Lock()
|
|
defer mostRecentTokenCredentialRequestResponseLock.Unlock()
|
|
if mostRecentTokenCredentialRequestResponse == nil || credentialAlmostExpired(t, mostRecentTokenCredentialRequestResponse) {
|
|
// Make a TokenCredentialRequest. This can either return a cert signed by the Kube API server's CA (e.g. on kind)
|
|
// or a cert signed by the impersonator's signing CA (e.g. on GKE). Either should be accepted by the impersonation
|
|
// proxy server as a valid authentication.
|
|
//
|
|
// However, we issue short-lived certs, so this cert will only be valid for a few minutes.
|
|
// Cache it until it is almost expired and then refresh it whenever it is close to expired.
|
|
//
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
resp, err := createTokenCredentialRequest(credentialRequestSpecWithWorkingCredentials, client)
|
|
requireEventually.NoError(err)
|
|
requireEventually.NotNil(resp)
|
|
requireEventually.NotNil(resp.Status)
|
|
requireEventually.NotNil(resp.Status.Credential)
|
|
requireEventually.Nilf(resp.Status.Message, "expected no error message but got: %s", testlib.Sdump(resp.Status.Message))
|
|
requireEventually.NotEmpty(resp.Status.Credential.ClientCertificateData)
|
|
requireEventually.NotEmpty(resp.Status.Credential.ClientKeyData)
|
|
|
|
// At the moment the credential request should not have returned a token. In the future, if we make it return
|
|
// tokens, we should revisit this test's rest config below.
|
|
requireEventually.Empty(resp.Status.Credential.Token)
|
|
|
|
mostRecentTokenCredentialRequestResponse = resp
|
|
}, 5*time.Minute, 5*time.Second)
|
|
}
|
|
|
|
return mostRecentTokenCredentialRequestResponse.Status.Credential
|
|
}
|
|
|
|
refreshCredential := func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential {
|
|
// Use an anonymous client which goes through the impersonation proxy to make the request because that's
|
|
// what would normally happen when a user is using a kubeconfig where the server is the impersonation proxy,
|
|
// so it more closely simulates the normal use case, and also because we want this to work on AKS clusters
|
|
// which do not allow anonymous requests.
|
|
client := newAnonymousImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge
|
|
return refreshCredentialHelper(t, client)
|
|
}
|
|
|
|
oldCredentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
// At the end of the test, clean up the CredentialIssuer
|
|
t.Cleanup(func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancel()
|
|
|
|
// Delete any version that was created by this test.
|
|
t.Logf("cleaning up credentialissuer at end of test %s", credentialIssuerName(env))
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
newCredentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
oldCredentialIssuer.Spec.DeepCopyInto(&newCredentialIssuer.Spec)
|
|
_, err = adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Update(ctx, newCredentialIssuer, metav1.UpdateOptions{})
|
|
return err
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// If we are running on an environment that has a load balancer, expect that the
|
|
// CredentialIssuer will be updated eventually with a successful impersonation proxy frontend.
|
|
// We do this to ensure that future tests that use the impersonation proxy (e.g.,
|
|
// TestE2EFullIntegration) will start with a known-good state.
|
|
if clusterSupportsLoadBalancers {
|
|
performImpersonatorDiscovery(ctx, t, env, adminClient, adminConciergeClient, refreshCredential)
|
|
}
|
|
})
|
|
|
|
// Done with set-up and ready to get started with the test. There are several states that we could be in at
|
|
// this point depending on the capabilities of the cluster under test. We handle each possible case here.
|
|
switch {
|
|
case impersonatorShouldHaveStartedAutomaticallyByDefault && clusterSupportsLoadBalancers:
|
|
// configure the credential issuer spec to have the impersonation proxy in auto mode
|
|
updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{
|
|
ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{
|
|
Mode: conciergev1alpha.ImpersonationProxyModeAuto,
|
|
Service: conciergev1alpha.ImpersonationProxyServiceSpec{
|
|
Type: conciergev1alpha.ImpersonationProxyServiceTypeLoadBalancer,
|
|
Annotations: map[string]string{
|
|
"service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout": "4000",
|
|
},
|
|
},
|
|
},
|
|
})
|
|
// Auto mode should have decided that the impersonator will run and should have started a load balancer,
|
|
// and we will be able to use the load balancer to access the impersonator. (e.g. GKE, AKS, EKS)
|
|
// Check that load balancer has been automatically created by the impersonator's "auto" mode.
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient)
|
|
}, 30*time.Second, 500*time.Millisecond)
|
|
|
|
case impersonatorShouldHaveStartedAutomaticallyByDefault && !clusterSupportsLoadBalancers:
|
|
t.Fatal("None of the clusters types that we currently test against should automatically" +
|
|
"enable the impersonation proxy without also supporting load balancers. If we add such a" +
|
|
"cluster type in the future then we should enhance this test.")
|
|
|
|
case !impersonatorShouldHaveStartedAutomaticallyByDefault && clusterSupportsLoadBalancers:
|
|
// Auto mode should have decided that the impersonator will be disabled. We need to manually enable it.
|
|
// The cluster supports load balancers so we should enable it and let the impersonator create a load balancer
|
|
// automatically. (e.g. TKGS)
|
|
// The CredentialIssuer's strategies array should have been updated to include an unsuccessful impersonation
|
|
// strategy saying that it was automatically disabled.
|
|
requireDisabledStrategy(ctx, t, env, adminConciergeClient)
|
|
|
|
// Create configuration to make the impersonation proxy turn on with no endpoint (i.e. automatically create a load balancer).
|
|
updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{
|
|
ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{
|
|
Mode: conciergev1alpha.ImpersonationProxyModeEnabled,
|
|
},
|
|
})
|
|
|
|
default:
|
|
// Auto mode should have decided that the impersonator will be disabled. We need to manually enable it.
|
|
// However, the cluster does not support load balancers so we should enable it without a load balancer
|
|
// and use squid to make requests. (e.g. kind)
|
|
if env.Proxy == "" {
|
|
t.Skip("test cluster does not support load balancers but also doesn't have a squid proxy... " +
|
|
"this is not a supported configuration for test clusters")
|
|
}
|
|
|
|
// Check that no load balancer has been created by the impersonator's "auto" mode.
|
|
testlib.RequireNeverWithoutError(t, func() (bool, error) {
|
|
return hasImpersonationProxyLoadBalancerService(ctx, env, adminClient)
|
|
}, 10*time.Second, 500*time.Millisecond)
|
|
|
|
// Check that we can't use the impersonation proxy to execute kubectl commands yet.
|
|
_, err = impersonationProxyViaSquidKubeClientWithoutCredential(t, proxyServiceEndpoint).CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
|
|
isErr, message := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint)
|
|
require.Truef(t, isErr, "wanted error %q to be service unavailable via squid error, but: %s", err, message)
|
|
|
|
// Create configuration to make the impersonation proxy turn on with a hard coded endpoint (without a load balancer).
|
|
updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{
|
|
ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{
|
|
Mode: conciergev1alpha.ImpersonationProxyModeEnabled,
|
|
ExternalEndpoint: proxyServiceEndpoint,
|
|
},
|
|
})
|
|
}
|
|
|
|
// At this point the impersonator should be starting/running. When it is ready, the CredentialIssuer's
|
|
// strategies array should be updated to include a successful impersonation strategy which can be used
|
|
// to discover the impersonator's URL and CA certificate. Until it has finished starting, it may not be included
|
|
// in the strategies array or it may be included in an error state. It can be in an error state for
|
|
// awhile when it is waiting for the load balancer to be assigned an ip/hostname.
|
|
impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminClient, adminConciergeClient, refreshCredential)
|
|
if !clusterSupportsLoadBalancers {
|
|
// In this case, we specified the endpoint in the configmap, so check that it was reported correctly in the CredentialIssuer.
|
|
require.Equal(t, "https://"+proxyServiceEndpoint, impersonationProxyURL)
|
|
} else {
|
|
// If the impersonationProxyURL is a hostname, make sure DNS will resolve before we move on.
|
|
ensureDNSResolves(t, impersonationProxyURL)
|
|
}
|
|
|
|
// Because our credentials expire so quickly, we'll always use a new client, to give us a chance to refresh our
|
|
// credentials before they expire. Create a closure to capture the arguments to newImpersonationProxyClient
|
|
// so we don't have to keep repeating them.
|
|
// This client performs TLS checks, so it also provides test coverage that the impersonation proxy server is generating TLS certs correctly.
|
|
impersonationProxyKubeClient := func(t *testing.T) kubernetes.Interface {
|
|
return newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM, nil, refreshCredential).Kubernetes
|
|
}
|
|
|
|
t.Run("positive tests", func(t *testing.T) {
|
|
// Create an RBAC rule to allow this user to read/write everything.
|
|
testlib.CreateTestClusterRoleBinding(t,
|
|
rbacv1.Subject{Kind: rbacv1.UserKind, APIGroup: rbacv1.GroupName, Name: env.TestUser.ExpectedUsername},
|
|
rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"},
|
|
)
|
|
// Wait for the above RBAC rule to take effect.
|
|
testlib.WaitForUserToHaveAccess(t, env.TestUser.ExpectedUsername, []string{}, &authorizationv1.ResourceAttributes{
|
|
Verb: "get", Group: "", Version: "v1", Resource: "namespaces",
|
|
})
|
|
|
|
// Get pods in supervisor namespace and pick one.
|
|
// this is for tests that require performing actions against a running pod.
|
|
// We use the supervisor pod because we already have it handy and need to port-forward a running port.
|
|
// We avoid using the concierge for this because it requires TLS 1.3 which is not support by older versions of curl.
|
|
supervisorPods, err := adminClient.CoreV1().Pods(env.SupervisorNamespace).List(ctx,
|
|
metav1.ListOptions{LabelSelector: "deployment.pinniped.dev=supervisor"})
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, supervisorPods.Items, "could not find supervisor pods")
|
|
supervisorPod := supervisorPods.Items[0]
|
|
|
|
// Test that the user can perform basic actions through the client with their username and group membership
|
|
// influencing RBAC checks correctly.
|
|
t.Run(
|
|
"access as user",
|
|
testlib.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, impersonationProxyKubeClient(t)),
|
|
)
|
|
for _, group := range env.TestUser.ExpectedGroups {
|
|
group := group
|
|
t.Run(
|
|
"access as group "+group,
|
|
testlib.AccessAsGroupTest(ctx, group, impersonationProxyKubeClient(t)),
|
|
)
|
|
}
|
|
|
|
if env.KubernetesDistribution == testlib.EKSDistro {
|
|
t.Log("eks: sleeping for 10 minutes to allow DNS propagation")
|
|
time.Sleep(10 * time.Minute)
|
|
}
|
|
|
|
t.Run("kubectl port-forward and keeping the connection open for over a minute (non-idle)", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator)
|
|
|
|
// Run the kubectl port-forward command.
|
|
timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancelFunc()
|
|
portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", supervisorPod.Namespace, supervisorPod.Name, "10443:8443")
|
|
portForwardCmd.Env = envVarsWithProxy
|
|
|
|
// Start, but don't wait for the command to finish.
|
|
err := portForwardCmd.Start()
|
|
require.NoError(t, err, `"kubectl port-forward" failed`)
|
|
go func() {
|
|
assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String())
|
|
}()
|
|
|
|
// The server should recognize this this
|
|
// is going to be a long-running command and keep the connection open as long as the client stays connected.
|
|
|
|
// curl the endpoint as many times as we can within 70 seconds.
|
|
// this will ensure that we don't run into idle timeouts.
|
|
var curlStdOut, curlStdErr bytes.Buffer
|
|
timeout, cancelFunc = context.WithTimeout(ctx, 75*time.Second)
|
|
defer cancelFunc()
|
|
startTime := time.Now()
|
|
for time.Now().Before(startTime.Add(70 * time.Second)) {
|
|
curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10443/healthz") // -sS turns off the progressbar but still prints errors
|
|
curlCmd.Stdout = &curlStdOut
|
|
curlCmd.Stderr = &curlStdErr
|
|
curlErr := curlCmd.Run()
|
|
if curlErr != nil {
|
|
t.Log("curl error: " + curlErr.Error())
|
|
t.Log("curlStdErr: " + curlStdErr.String())
|
|
t.Log("stdout: " + curlStdOut.String())
|
|
}
|
|
t.Log("Running curl through the kubectl port-forward port for 70 seconds. Elapsed time:", time.Since(startTime))
|
|
time.Sleep(1 * time.Second)
|
|
}
|
|
|
|
// curl the endpoint once more, once 70 seconds has elapsed, to make sure the connection is still open.
|
|
timeout, cancelFunc = context.WithTimeout(ctx, 30*time.Second)
|
|
defer cancelFunc()
|
|
curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10443/healthz") // -sS turns off the progressbar but still prints errors
|
|
curlCmd.Stdout = &curlStdOut
|
|
curlCmd.Stderr = &curlStdErr
|
|
curlErr := curlCmd.Run()
|
|
|
|
if curlErr != nil {
|
|
t.Log("curl error: " + curlErr.Error())
|
|
t.Log("curlStdErr: " + curlStdErr.String())
|
|
t.Log("stdout: " + curlStdOut.String())
|
|
}
|
|
require.NoError(t, curlErr)
|
|
require.Contains(t, curlStdOut.String(), "okokokokok") // a few successful healthz responses
|
|
})
|
|
|
|
t.Run("kubectl port-forward and keeping the connection open for over a minute (idle)", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
kubeconfigPath, envVarsWithProxy, _ := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator)
|
|
|
|
// Run the kubectl port-forward command.
|
|
timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancelFunc()
|
|
portForwardCmd, _, portForwardStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "port-forward", "--namespace", supervisorPod.Namespace, supervisorPod.Name, "10444:8443")
|
|
portForwardCmd.Env = envVarsWithProxy
|
|
|
|
// Start, but don't wait for the command to finish.
|
|
err := portForwardCmd.Start()
|
|
require.NoError(t, err, `"kubectl port-forward" failed`)
|
|
go func() {
|
|
assert.EqualErrorf(t, portForwardCmd.Wait(), "signal: killed", `wanted "kubectl port-forward" to get signaled because context was cancelled (stderr: %q)`, portForwardStderr.String())
|
|
}()
|
|
|
|
// Wait to see if we time out. The default timeout is 60 seconds, but the server should recognize that this
|
|
// is going to be a long-running command and keep the connection open as long as the client stays connected.
|
|
time.Sleep(70 * time.Second)
|
|
|
|
timeout, cancelFunc = context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancelFunc()
|
|
curlCmd := exec.CommandContext(timeout, "curl", "-k", "-sS", "https://127.0.0.1:10444/healthz") // -sS turns off the progressbar but still prints errors
|
|
var curlStdOut, curlStdErr bytes.Buffer
|
|
curlCmd.Stdout = &curlStdOut
|
|
curlCmd.Stderr = &curlStdErr
|
|
err = curlCmd.Run()
|
|
if err != nil {
|
|
t.Log("curl error: " + err.Error())
|
|
t.Log("curlStdErr: " + curlStdErr.String())
|
|
t.Log("stdout: " + curlStdOut.String())
|
|
}
|
|
require.NoError(t, err)
|
|
require.Equal(t, curlStdOut.String(), "ok")
|
|
})
|
|
|
|
t.Run("using and watching all the basic verbs", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
// Create a namespace, because it will be easier to exercise "deletecollection" if we have a namespace.
|
|
namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name
|
|
|
|
// Create and start informer to exercise the "watch" verb for us.
|
|
informerFactory := k8sinformers.NewSharedInformerFactoryWithOptions(
|
|
impersonationProxyKubeClient(t),
|
|
0,
|
|
k8sinformers.WithNamespace(namespaceName))
|
|
informer := informerFactory.Core().V1().ConfigMaps()
|
|
informer.Informer() // makes sure that the informer will cache
|
|
stopChannel := make(chan struct{})
|
|
informerFactory.Start(stopChannel)
|
|
t.Cleanup(func() {
|
|
// Shut down the informer.
|
|
close(stopChannel)
|
|
})
|
|
informerFactory.WaitForCacheSync(ctx.Done())
|
|
|
|
// Use labels on our created ConfigMaps to avoid accidentally listing other ConfigMaps that might
|
|
// exist in the namespace. In Kube 1.20+ there is a default ConfigMap in every namespace.
|
|
configMapLabels := labels.Set{
|
|
"pinniped.dev/testConfigMap": testlib.RandHex(t, 8),
|
|
}
|
|
|
|
// Test "create" verb through the impersonation proxy.
|
|
_, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Create(ctx,
|
|
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: configMapLabels}},
|
|
metav1.CreateOptions{},
|
|
)
|
|
require.NoError(t, err)
|
|
_, err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Create(ctx,
|
|
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-2", Labels: configMapLabels}},
|
|
metav1.CreateOptions{},
|
|
)
|
|
require.NoError(t, err)
|
|
_, err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Create(ctx,
|
|
&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "configmap-3", Labels: configMapLabels}},
|
|
metav1.CreateOptions{},
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
// Make sure that all of the created ConfigMaps show up in the informer's cache to
|
|
// demonstrate that the informer's "watch" verb is working through the impersonation proxy.
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
_, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-1")
|
|
requireEventually.NoError(err)
|
|
_, err = informer.Lister().ConfigMaps(namespaceName).Get("configmap-2")
|
|
requireEventually.NoError(err)
|
|
_, err = informer.Lister().ConfigMaps(namespaceName).Get("configmap-3")
|
|
requireEventually.NoError(err)
|
|
}, 10*time.Second, 50*time.Millisecond)
|
|
|
|
// Test "get" verb through the impersonation proxy.
|
|
configMap3, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Get(ctx, "configmap-3", metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// Test "list" verb through the impersonation proxy.
|
|
listResult, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{
|
|
LabelSelector: configMapLabels.String(),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, listResult.Items, 3)
|
|
|
|
// Test "update" verb through the impersonation proxy.
|
|
configMap3.Data = map[string]string{"foo": "bar"}
|
|
updateResult, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Update(ctx, configMap3, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
require.Equal(t, "bar", updateResult.Data["foo"])
|
|
|
|
// Make sure that the updated ConfigMap shows up in the informer's cache.
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3")
|
|
requireEventually.NoError(err)
|
|
requireEventually.Equal("bar", configMap.Data["foo"])
|
|
}, 10*time.Second, 50*time.Millisecond)
|
|
|
|
// Test "patch" verb through the impersonation proxy.
|
|
patchResult, err := impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Patch(ctx,
|
|
"configmap-3",
|
|
types.MergePatchType,
|
|
[]byte(`{"data":{"baz":"42"}}`),
|
|
metav1.PatchOptions{},
|
|
)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "bar", patchResult.Data["foo"])
|
|
require.Equal(t, "42", patchResult.Data["baz"])
|
|
|
|
// Make sure that the patched ConfigMap shows up in the informer's cache.
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
configMap, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3")
|
|
requireEventually.NoError(err)
|
|
requireEventually.Equal("bar", configMap.Data["foo"])
|
|
requireEventually.Equal("42", configMap.Data["baz"])
|
|
}, 10*time.Second, 50*time.Millisecond)
|
|
|
|
// Test "delete" verb through the impersonation proxy.
|
|
err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).Delete(ctx, "configmap-3", metav1.DeleteOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// Make sure that the deleted ConfigMap shows up in the informer's cache.
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
_, err := informer.Lister().ConfigMaps(namespaceName).Get("configmap-3")
|
|
requireEventually.Truef(k8serrors.IsNotFound(err), "expected a NotFound error from get, got %v", err)
|
|
|
|
list, err := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector())
|
|
requireEventually.NoError(err)
|
|
requireEventually.Len(list, 2)
|
|
}, 10*time.Second, 50*time.Millisecond)
|
|
|
|
// Test "deletecollection" verb through the impersonation proxy.
|
|
err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// Make sure that the deleted ConfigMaps shows up in the informer's cache.
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
list, err := informer.Lister().ConfigMaps(namespaceName).List(configMapLabels.AsSelector())
|
|
requireEventually.NoError(err)
|
|
requireEventually.Empty(list)
|
|
}, 10*time.Second, 50*time.Millisecond)
|
|
|
|
// There should be no ConfigMaps left.
|
|
listResult, err = impersonationProxyKubeClient(t).CoreV1().ConfigMaps(namespaceName).List(ctx, metav1.ListOptions{
|
|
LabelSelector: configMapLabels.String(),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, listResult.Items, 0)
|
|
})
|
|
|
|
t.Run("nested impersonation as a regular user is allowed if they have enough RBAC permissions", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
// Make a client which will send requests through the impersonation proxy and will also add
|
|
// impersonate headers to the request.
|
|
nestedImpersonationClient := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{UserName: "other-user-to-impersonate"}, refreshCredential)
|
|
|
|
// Check that we can get some resource through the impersonation proxy without any impersonation headers on the request.
|
|
// We could use any resource for this, but we happen to know that this one should exist.
|
|
_, err := impersonationProxyKubeClient(t).CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// Now we'll see what happens when we add an impersonation header to the request. This should generate a
|
|
// request similar to the one above, except that it will also have an impersonation header.
|
|
_, err = nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
// this user is not allowed to impersonate other users
|
|
require.True(t, k8serrors.IsForbidden(err), err)
|
|
require.EqualError(t, err, fmt.Sprintf(
|
|
`users "other-user-to-impersonate" is forbidden: `+
|
|
`User "%s" cannot impersonate resource "users" in API group "" at the cluster scope: `+
|
|
`decision made by impersonation-proxy.concierge.pinniped.dev`,
|
|
env.TestUser.ExpectedUsername))
|
|
|
|
// impersonate the GC service account instead which can read anything (the binding to edit allows this)
|
|
nestedImpersonationClientAsSA := newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{UserName: "system:serviceaccount:kube-system:generic-garbage-collector"}, refreshCredential)
|
|
|
|
_, err = nestedImpersonationClientAsSA.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups
|
|
expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...)
|
|
expectedGroups = append(expectedGroups, "system:authenticated")
|
|
expectedOriginalUserInfo := authenticationv1.UserInfo{
|
|
Username: env.TestUser.ExpectedUsername,
|
|
Groups: expectedGroups,
|
|
}
|
|
expectedOriginalUserInfoJSON, err := json.Marshal(expectedOriginalUserInfo)
|
|
require.NoError(t, err)
|
|
|
|
// check that we impersonated the correct user and that the original user is retained in the extra
|
|
whoAmI, err := nestedImpersonationClientAsSA.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
"system:serviceaccount:kube-system:generic-garbage-collector",
|
|
[]string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"},
|
|
map[string]identityv1alpha1.ExtraValue{
|
|
"original-user-info.impersonation-proxy.concierge.pinniped.dev": {string(expectedOriginalUserInfoJSON)},
|
|
},
|
|
),
|
|
whoAmI,
|
|
)
|
|
|
|
_, err = newImpersonationProxyClient(t, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{
|
|
UserName: "system:serviceaccount:kube-system:generic-garbage-collector",
|
|
Extra: map[string][]string{
|
|
"some-fancy-key": {"with a dangerous value"},
|
|
},
|
|
},
|
|
refreshCredential).PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
// this user should not be able to impersonate extra
|
|
require.True(t, k8serrors.IsForbidden(err), err)
|
|
require.EqualError(t, err, fmt.Sprintf(
|
|
`userextras.authentication.k8s.io "with a dangerous value" is forbidden: `+
|
|
`User "%s" cannot impersonate resource "userextras/some-fancy-key" in API group "authentication.k8s.io" at the cluster scope: `+
|
|
`decision made by impersonation-proxy.concierge.pinniped.dev`,
|
|
env.TestUser.ExpectedUsername))
|
|
})
|
|
|
|
t.Run("nested impersonation as a cluster admin user is allowed", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
// Copy the admin credentials from the admin kubeconfig.
|
|
adminClientRestConfig := testlib.NewClientConfig(t)
|
|
clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig)
|
|
|
|
// figure out who the admin user is
|
|
whoAmIAdmin, err := newImpersonationProxyClientWithCredentials(t,
|
|
clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil).
|
|
PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// The WhoAmI API is lossy:
|
|
// - It drops UID
|
|
// - It lowercases all extra keys
|
|
// the admin user on EKS has both a UID set and an extra key with uppercase characters
|
|
// Thus we fallback to the CSR API to grab the UID and Extra to handle this scenario
|
|
uid, extra := getUIDAndExtraViaCSR(ctx, t, whoAmIAdmin.Status.KubernetesUserInfo.User.UID,
|
|
newImpersonationProxyClientWithCredentials(t,
|
|
clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil).
|
|
Kubernetes,
|
|
)
|
|
|
|
expectedExtra := make(map[string]authenticationv1.ExtraValue, len(extra))
|
|
for k, v := range extra {
|
|
expectedExtra[k] = authenticationv1.ExtraValue(v)
|
|
}
|
|
expectedOriginalUserInfo := authenticationv1.UserInfo{
|
|
Username: whoAmIAdmin.Status.KubernetesUserInfo.User.Username,
|
|
UID: uid,
|
|
Groups: whoAmIAdmin.Status.KubernetesUserInfo.User.Groups,
|
|
Extra: expectedExtra,
|
|
}
|
|
expectedOriginalUserInfoJSON, err := json.Marshal(expectedOriginalUserInfo)
|
|
require.NoError(t, err)
|
|
|
|
// Make a client using the admin credentials which will send requests through the impersonation proxy
|
|
// and will also add impersonate headers to the request.
|
|
nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t,
|
|
clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{
|
|
UserName: "other-user-to-impersonate",
|
|
Groups: []string{"other-group-1", "other-group-2"},
|
|
Extra: map[string][]string{
|
|
"this-key": {"to this value"},
|
|
},
|
|
},
|
|
)
|
|
|
|
_, err = nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
// the impersonated user lacks the RBAC to perform this call
|
|
require.True(t, k8serrors.IsForbidden(err), err)
|
|
require.EqualError(t, err, fmt.Sprintf(
|
|
`secrets "%s" is forbidden: User "other-user-to-impersonate" cannot get resource "secrets" in API group "" in the namespace "%s": `+
|
|
`decision made by impersonation-proxy.concierge.pinniped.dev`,
|
|
impersonationProxyTLSSecretName(env), env.ConciergeNamespace,
|
|
))
|
|
|
|
// check that we impersonated the correct user and that the original user is retained in the extra
|
|
whoAmI, err := nestedImpersonationClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
"other-user-to-impersonate",
|
|
[]string{"other-group-1", "other-group-2", "system:authenticated"},
|
|
map[string]identityv1alpha1.ExtraValue{
|
|
"this-key": {"to this value"},
|
|
"original-user-info.impersonation-proxy.concierge.pinniped.dev": {string(expectedOriginalUserInfoJSON)},
|
|
},
|
|
),
|
|
whoAmI,
|
|
)
|
|
})
|
|
|
|
t.Run("nested impersonation as a cluster admin fails on reserved key", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
adminClientRestConfig := testlib.NewClientConfig(t)
|
|
clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig)
|
|
|
|
nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t,
|
|
clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{
|
|
UserName: "other-user-to-impersonate",
|
|
Groups: []string{"other-group-1", "other-group-2", "system:masters"}, // impersonate system:masters so we get past authorization checks
|
|
Extra: map[string][]string{
|
|
"this-good-key": {"to this good value"},
|
|
"something.impersonation-proxy.concierge.pinniped.dev": {"super sneaky value"},
|
|
},
|
|
},
|
|
)
|
|
|
|
_, err := nestedImpersonationClient.Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user")
|
|
require.True(t, k8serrors.IsInternalError(err), err)
|
|
require.Equal(t, &k8serrors.StatusError{
|
|
ErrStatus: metav1.Status{
|
|
Status: metav1.StatusFailure,
|
|
Code: http.StatusInternalServerError,
|
|
Reason: metav1.StatusReasonInternalError,
|
|
Details: &metav1.StatusDetails{
|
|
Causes: []metav1.StatusCause{
|
|
{
|
|
Message: "unimplemented functionality - unable to act as current user",
|
|
},
|
|
},
|
|
},
|
|
Message: "Internal error occurred: unimplemented functionality - unable to act as current user",
|
|
},
|
|
}, err)
|
|
})
|
|
|
|
t.Run("nested impersonation as a cluster admin fails if UID impersonation is attempted", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
adminClientRestConfig := testlib.NewClientConfig(t)
|
|
clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig)
|
|
|
|
nestedImpersonationUIDOnly := newImpersonationProxyConfigWithCredentials(t,
|
|
clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nil,
|
|
)
|
|
nestedImpersonationUIDOnly.Wrap(func(rt http.RoundTripper) http.RoundTripper {
|
|
return roundtripper.WrapFunc(rt, func(r *http.Request) (*http.Response, error) {
|
|
r.Header.Set("iMperSONATE-uid", "some-awesome-uid")
|
|
return rt.RoundTrip(r)
|
|
})
|
|
})
|
|
|
|
_, errUID := testlib.NewKubeclient(t, nestedImpersonationUIDOnly).Kubernetes.CoreV1().Secrets("foo").Get(ctx, "bar", metav1.GetOptions{})
|
|
msg := `Internal Server Error: "/api/v1/namespaces/foo/secrets/bar": requested [{UID some-awesome-uid authentication.k8s.io/v1 }] without impersonating a user`
|
|
full := fmt.Sprintf(`an error on the server (%q) has prevented the request from succeeding (get secrets bar)`, msg)
|
|
require.EqualError(t, errUID, full)
|
|
require.True(t, k8serrors.IsInternalError(errUID), errUID)
|
|
require.Equal(t, &k8serrors.StatusError{
|
|
ErrStatus: metav1.Status{
|
|
Status: metav1.StatusFailure,
|
|
Code: http.StatusInternalServerError,
|
|
Reason: metav1.StatusReasonInternalError,
|
|
Details: &metav1.StatusDetails{
|
|
Name: "bar",
|
|
Kind: "secrets",
|
|
Causes: []metav1.StatusCause{
|
|
{
|
|
Type: metav1.CauseTypeUnexpectedServerResponse,
|
|
Message: msg,
|
|
},
|
|
},
|
|
},
|
|
Message: full,
|
|
},
|
|
}, errUID)
|
|
|
|
nestedImpersonationUID := newImpersonationProxyConfigWithCredentials(t,
|
|
clusterAdminCredentials, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{
|
|
UserName: "other-user-to-impersonate",
|
|
Groups: []string{"system:masters"}, // impersonate system:masters so we get past authorization checks
|
|
},
|
|
)
|
|
nestedImpersonationUID.Wrap(func(rt http.RoundTripper) http.RoundTripper {
|
|
return roundtripper.WrapFunc(rt, func(r *http.Request) (*http.Response, error) {
|
|
r.Header.Set("imperSONate-uiD", "some-fancy-uid")
|
|
return rt.RoundTrip(r)
|
|
})
|
|
})
|
|
|
|
_, err := testlib.NewKubeclient(t, nestedImpersonationUID).Kubernetes.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
require.EqualError(t, err, "Internal error occurred: unimplemented functionality - unable to act as current user")
|
|
require.True(t, k8serrors.IsInternalError(err), err)
|
|
require.Equal(t, &k8serrors.StatusError{
|
|
ErrStatus: metav1.Status{
|
|
Status: metav1.StatusFailure,
|
|
Code: http.StatusInternalServerError,
|
|
Reason: metav1.StatusReasonInternalError,
|
|
Details: &metav1.StatusDetails{
|
|
Causes: []metav1.StatusCause{
|
|
{
|
|
Message: "unimplemented functionality - unable to act as current user",
|
|
},
|
|
},
|
|
},
|
|
Message: "Internal error occurred: unimplemented functionality - unable to act as current user",
|
|
},
|
|
}, err)
|
|
})
|
|
|
|
// this works because impersonation cannot set UID and thus the final user info the proxy sees has no UID
|
|
t.Run("nested impersonation as a service account is allowed if it has enough RBAC permissions", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name
|
|
saName, saToken, saUID := createServiceAccountToken(ctx, t, adminClient, namespaceName)
|
|
nestedImpersonationClient := newImpersonationProxyClientWithCredentials(t,
|
|
&loginv1alpha1.ClusterCredential{Token: saToken}, impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{UserName: "system:serviceaccount:kube-system:root-ca-cert-publisher"}).PinnipedConcierge
|
|
_, err := nestedImpersonationClient.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
// this SA is not yet allowed to impersonate SAs
|
|
require.True(t, k8serrors.IsForbidden(err), err)
|
|
require.EqualError(t, err, fmt.Sprintf(
|
|
`serviceaccounts "root-ca-cert-publisher" is forbidden: `+
|
|
`User "%s" cannot impersonate resource "serviceaccounts" in API group "" in the namespace "kube-system": `+
|
|
`decision made by impersonation-proxy.concierge.pinniped.dev`,
|
|
serviceaccount.MakeUsername(namespaceName, saName)))
|
|
|
|
// webhook authorizer deny cache TTL is 10 seconds so we need to wait long enough for it to drain
|
|
time.Sleep(15 * time.Second)
|
|
|
|
// allow the test SA to impersonate any SA
|
|
testlib.CreateTestClusterRoleBinding(t,
|
|
rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: saName, Namespace: namespaceName},
|
|
rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "edit"},
|
|
)
|
|
testlib.WaitForUserToHaveAccess(t, serviceaccount.MakeUsername(namespaceName, saName), []string{}, &authorizationv1.ResourceAttributes{
|
|
Verb: "impersonate", Group: "", Version: "v1", Resource: "serviceaccounts",
|
|
})
|
|
|
|
whoAmI, err := nestedImpersonationClient.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
"system:serviceaccount:kube-system:root-ca-cert-publisher",
|
|
[]string{"system:serviceaccounts", "system:serviceaccounts:kube-system", "system:authenticated"},
|
|
map[string]identityv1alpha1.ExtraValue{
|
|
"original-user-info.impersonation-proxy.concierge.pinniped.dev": {
|
|
fmt.Sprintf(`{"username":"%s","uid":"%s","groups":["system:serviceaccounts","system:serviceaccounts:%s","system:authenticated"]}`,
|
|
serviceaccount.MakeUsername(namespaceName, saName), saUID, namespaceName),
|
|
},
|
|
},
|
|
),
|
|
whoAmI,
|
|
)
|
|
})
|
|
|
|
t.Run("WhoAmIRequests and different kinds of authentication through the impersonation proxy", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
// Test using the TokenCredentialRequest for authentication.
|
|
impersonationProxyPinnipedConciergeClient := newImpersonationProxyClient(t,
|
|
impersonationProxyURL, impersonationProxyCACertPEM, nil, refreshCredential,
|
|
).PinnipedConcierge
|
|
whoAmI, err := impersonationProxyPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups
|
|
expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...)
|
|
expectedGroups = append(expectedGroups, "system:authenticated")
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
env.TestUser.ExpectedUsername,
|
|
expectedGroups,
|
|
nil,
|
|
),
|
|
whoAmI,
|
|
)
|
|
|
|
// Test an unauthenticated request which does not include any credentials.
|
|
impersonationProxyAnonymousPinnipedConciergeClient := newAnonymousImpersonationProxyClient(
|
|
t, impersonationProxyURL, impersonationProxyCACertPEM, nil,
|
|
).PinnipedConcierge
|
|
whoAmI, err = impersonationProxyAnonymousPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
|
|
// we expect the impersonation proxy to match the behavior of KAS in regards to anonymous requests
|
|
if env.HasCapability(testlib.AnonymousAuthenticationSupported) {
|
|
require.NoError(t, err)
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
"system:anonymous",
|
|
[]string{"system:unauthenticated"},
|
|
nil,
|
|
),
|
|
whoAmI,
|
|
)
|
|
} else {
|
|
require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err))
|
|
}
|
|
|
|
// Test using a service account token.
|
|
namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name
|
|
saName, saToken, _ := createServiceAccountToken(ctx, t, adminClient, namespaceName)
|
|
impersonationProxyServiceAccountPinnipedConciergeClient := newImpersonationProxyClientWithCredentials(t,
|
|
&loginv1alpha1.ClusterCredential{Token: saToken},
|
|
impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge
|
|
whoAmI, err = impersonationProxyServiceAccountPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
serviceaccount.MakeUsername(namespaceName, saName),
|
|
[]string{"system:serviceaccounts", "system:serviceaccounts:" + namespaceName, "system:authenticated"},
|
|
nil,
|
|
),
|
|
whoAmI,
|
|
)
|
|
})
|
|
|
|
t.Run("WhoAmIRequests and SA token request", func(t *testing.T) {
|
|
namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name
|
|
kubeClient := adminClient.CoreV1()
|
|
saName, _, saUID := createServiceAccountToken(ctx, t, adminClient, namespaceName)
|
|
expectedUsername := serviceaccount.MakeUsername(namespaceName, saName)
|
|
expectedUID := string(saUID)
|
|
expectedGroups := []string{"system:serviceaccounts", "system:serviceaccounts:" + namespaceName, "system:authenticated"}
|
|
|
|
_, tokenRequestProbeErr := kubeClient.ServiceAccounts(namespaceName).CreateToken(ctx, saName, &authenticationv1.TokenRequest{}, metav1.CreateOptions{})
|
|
if k8serrors.IsNotFound(tokenRequestProbeErr) && tokenRequestProbeErr.Error() == "the server could not find the requested resource" {
|
|
return // stop test early since the token request API is not enabled on this cluster - other errors are caught below
|
|
}
|
|
|
|
pod := testlib.CreatePod(ctx, t, "impersonation-proxy", namespaceName,
|
|
corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "sleeper",
|
|
Image: env.ShellContainerImage,
|
|
ImagePullPolicy: corev1.PullIfNotPresent,
|
|
Command: []string{"sh", "-c", "sleep 3600"},
|
|
},
|
|
},
|
|
ServiceAccountName: saName,
|
|
})
|
|
|
|
tokenRequestBadAudience, err := kubeClient.ServiceAccounts(namespaceName).CreateToken(ctx, saName, &authenticationv1.TokenRequest{
|
|
Spec: authenticationv1.TokenRequestSpec{
|
|
Audiences: []string{"should-fail-because-wrong-audience"}, // anything that is not an API server audience
|
|
BoundObjectRef: &authenticationv1.BoundObjectReference{
|
|
Kind: "Pod",
|
|
APIVersion: "",
|
|
Name: pod.Name,
|
|
UID: pod.UID,
|
|
},
|
|
},
|
|
}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
impersonationProxySABadAudPinnipedConciergeClient := newImpersonationProxyClientWithCredentials(t,
|
|
&loginv1alpha1.ClusterCredential{Token: tokenRequestBadAudience.Status.Token},
|
|
impersonationProxyURL, impersonationProxyCACertPEM, nil).PinnipedConcierge
|
|
|
|
_, badAudErr := impersonationProxySABadAudPinnipedConciergeClient.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.True(t, k8serrors.IsUnauthorized(badAudErr), testlib.Sdump(badAudErr))
|
|
|
|
tokenRequest, err := kubeClient.ServiceAccounts(namespaceName).CreateToken(ctx, saName, &authenticationv1.TokenRequest{
|
|
Spec: authenticationv1.TokenRequestSpec{
|
|
Audiences: []string{},
|
|
BoundObjectRef: &authenticationv1.BoundObjectReference{
|
|
Kind: "Pod",
|
|
APIVersion: "",
|
|
Name: pod.Name,
|
|
UID: pod.UID,
|
|
},
|
|
},
|
|
}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
impersonationProxySAClient := newImpersonationProxyClientWithCredentials(t,
|
|
&loginv1alpha1.ClusterCredential{Token: tokenRequest.Status.Token},
|
|
impersonationProxyURL, impersonationProxyCACertPEM, nil)
|
|
|
|
whoAmITokenReq, err := impersonationProxySAClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// new service account tokens include the pod info in the extra fields
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
expectedUsername,
|
|
expectedGroups,
|
|
map[string]identityv1alpha1.ExtraValue{
|
|
"authentication.kubernetes.io/pod-name": {pod.Name},
|
|
"authentication.kubernetes.io/pod-uid": {string(pod.UID)},
|
|
},
|
|
),
|
|
whoAmITokenReq,
|
|
)
|
|
|
|
// allow the test SA to create CSRs
|
|
testlib.CreateTestClusterRoleBinding(t,
|
|
rbacv1.Subject{Kind: rbacv1.ServiceAccountKind, Name: saName, Namespace: namespaceName},
|
|
rbacv1.RoleRef{Kind: "ClusterRole", APIGroup: rbacv1.GroupName, Name: "system:node-bootstrapper"},
|
|
)
|
|
testlib.WaitForUserToHaveAccess(t, expectedUsername, []string{}, &authorizationv1.ResourceAttributes{
|
|
Verb: "create", Group: certificatesv1.GroupName, Version: "*", Resource: "certificatesigningrequests",
|
|
})
|
|
|
|
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
require.NoError(t, err)
|
|
|
|
csrPEM, err := cert.MakeCSR(privateKey, &pkix.Name{
|
|
CommonName: "panda-man",
|
|
Organization: []string{"living-the-dream", "need-more-sleep"},
|
|
}, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
csrName, _, err := csr.RequestCertificate(
|
|
impersonationProxySAClient.Kubernetes,
|
|
csrPEM,
|
|
"",
|
|
certificatesv1.KubeAPIServerClientSignerName,
|
|
nil,
|
|
[]certificatesv1.KeyUsage{certificatesv1.UsageClientAuth},
|
|
privateKey,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
if testutil.KubeServerSupportsCertificatesV1API(t, adminClient.Discovery()) {
|
|
saCSR, err := impersonationProxySAClient.Kubernetes.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
err = adminClient.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{})
|
|
require.NoError(t, err)
|
|
// make sure the user info that the CSR captured matches the SA, including the UID
|
|
require.Equal(t, expectedUsername, saCSR.Spec.Username)
|
|
require.Equal(t, expectedUID, saCSR.Spec.UID)
|
|
require.Equal(t, expectedGroups, saCSR.Spec.Groups)
|
|
require.Equal(t, map[string]certificatesv1.ExtraValue{
|
|
"authentication.kubernetes.io/pod-name": {pod.Name},
|
|
"authentication.kubernetes.io/pod-uid": {string(pod.UID)},
|
|
}, saCSR.Spec.Extra)
|
|
} else {
|
|
// On old Kubernetes clusters use CertificatesV1beta1
|
|
saCSR, err := impersonationProxySAClient.Kubernetes.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
err = adminClient.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{})
|
|
require.NoError(t, err)
|
|
// make sure the user info that the CSR captured matches the SA, including the UID
|
|
require.Equal(t, expectedUsername, saCSR.Spec.Username)
|
|
require.Equal(t, expectedUID, saCSR.Spec.UID)
|
|
require.Equal(t, expectedGroups, saCSR.Spec.Groups)
|
|
require.Equal(t, map[string]certificatesv1beta1.ExtraValue{
|
|
"authentication.kubernetes.io/pod-name": {pod.Name},
|
|
"authentication.kubernetes.io/pod-uid": {string(pod.UID)},
|
|
}, saCSR.Spec.Extra)
|
|
}
|
|
})
|
|
|
|
t.Run("kubectl as a client", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
kubeconfigPath, envVarsWithProxy, tempDir := getImpersonationKubeconfig(t, env, impersonationProxyURL, impersonationProxyCACertPEM, credentialRequestSpecWithWorkingCredentials.Authenticator)
|
|
|
|
// Run a new test pod so we can interact with it using kubectl. We use a fresh pod here rather than the
|
|
// existing Concierge pod because we need more tools than we can get from a scratch/distroless base image.
|
|
runningTestPod := testlib.CreatePod(ctx, t, "impersonation-proxy", env.ConciergeNamespace, corev1.PodSpec{Containers: []corev1.Container{{
|
|
Name: "impersonation-proxy-test",
|
|
Image: env.ShellContainerImage,
|
|
ImagePullPolicy: corev1.PullIfNotPresent,
|
|
Command: []string{"bash", "-c", `while true; do read VAR; echo "VAR: $VAR"; done`},
|
|
Stdin: true,
|
|
Resources: corev1.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("16Mi"),
|
|
corev1.ResourceCPU: resource.MustParse("10m"),
|
|
},
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("16Mi"),
|
|
corev1.ResourceCPU: resource.MustParse("10m"),
|
|
},
|
|
},
|
|
}}})
|
|
|
|
// Try "kubectl exec" through the impersonation proxy.
|
|
echoString := "hello world"
|
|
remoteEchoFile := fmt.Sprintf("/tmp/test-impersonation-proxy-echo-file-%d.txt", time.Now().Unix())
|
|
stdout, err := runKubectl(t, kubeconfigPath, envVarsWithProxy, "exec", "--namespace", runningTestPod.Namespace, runningTestPod.Name, "--", "bash", "-c", fmt.Sprintf(`echo "%s" | tee %s`, echoString, remoteEchoFile))
|
|
require.NoError(t, err, `"kubectl exec" failed`)
|
|
require.Equal(t, echoString+"\n", stdout)
|
|
|
|
// run the kubectl cp command
|
|
localEchoFile := filepath.Join(tempDir, filepath.Base(remoteEchoFile))
|
|
_, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "cp", fmt.Sprintf("%s/%s:%s", runningTestPod.Namespace, runningTestPod.Name, remoteEchoFile), localEchoFile)
|
|
require.NoError(t, err, `"kubectl cp" failed`)
|
|
localEchoFileData, err := os.ReadFile(localEchoFile)
|
|
require.NoError(t, err)
|
|
require.Equal(t, echoString+"\n", string(localEchoFileData))
|
|
|
|
// run the kubectl logs command
|
|
logLinesCount := 10
|
|
stdout, err = runKubectl(t, kubeconfigPath, envVarsWithProxy, "logs", "--namespace", supervisorPod.Namespace, supervisorPod.Name, fmt.Sprintf("--tail=%d", logLinesCount))
|
|
require.NoError(t, err, `"kubectl logs" failed`)
|
|
// Expect _approximately_ logLinesCount lines in the output
|
|
// (we can't match 100% exactly due to https://github.com/kubernetes/kubernetes/issues/72628).
|
|
require.InDeltaf(t, logLinesCount, strings.Count(stdout, "\n"), 1, "wanted %d newlines in kubectl logs output:\n%s", logLinesCount, stdout)
|
|
|
|
// run the kubectl attach command
|
|
timeout, cancelFunc := context.WithTimeout(ctx, 2*time.Minute)
|
|
defer cancelFunc()
|
|
attachCmd, attachStdout, attachStderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, "attach", "--stdin=true", "--namespace", runningTestPod.Namespace, runningTestPod.Name, "-v=10")
|
|
attachCmd.Env = envVarsWithProxy
|
|
attachStdin, err := attachCmd.StdinPipe()
|
|
require.NoError(t, err)
|
|
|
|
// start but don't wait for the attach command
|
|
err = attachCmd.Start()
|
|
require.NoError(t, err)
|
|
attachExitCh := make(chan struct{})
|
|
go func() {
|
|
assert.NoError(t, attachCmd.Wait())
|
|
close(attachExitCh)
|
|
}()
|
|
|
|
// write to stdin on the attach process
|
|
_, err = attachStdin.Write([]byte(echoString + "\n"))
|
|
require.NoError(t, err)
|
|
|
|
// see that we can read stdout and it spits out stdin output back to us
|
|
wantAttachStdout := fmt.Sprintf("VAR: %s\n", echoString)
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
requireEventually.Equal(
|
|
wantAttachStdout,
|
|
attachStdout.String(),
|
|
`got "kubectl attach" stdout: %q, wanted: %q (stderr: %q)`,
|
|
attachStdout.String(),
|
|
wantAttachStdout,
|
|
attachStderr.String(),
|
|
)
|
|
}, time.Second*60, time.Millisecond*250)
|
|
|
|
// close stdin and attach process should exit
|
|
err = attachStdin.Close()
|
|
require.NoError(t, err)
|
|
requireClose(t, attachExitCh, time.Second*20)
|
|
})
|
|
|
|
t.Run("websocket client", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name
|
|
|
|
impersonationRestConfig := impersonationProxyRestConfig(
|
|
refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM),
|
|
impersonationProxyURL, impersonationProxyCACertPEM, nil,
|
|
)
|
|
tlsConfig, err := rest.TLSConfigFor(impersonationRestConfig)
|
|
require.NoError(t, err)
|
|
|
|
wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value"
|
|
dest, _ := url.Parse(impersonationProxyURL)
|
|
dest.Scheme = "wss"
|
|
dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps"
|
|
dest.RawQuery = url.Values{
|
|
"watch": {"1"},
|
|
"labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)},
|
|
"resourceVersion": {"0"},
|
|
}.Encode()
|
|
dialer := websocket.Dialer{
|
|
TLSClientConfig: tlsConfig,
|
|
}
|
|
if !clusterSupportsLoadBalancers {
|
|
dialer.Proxy = func(req *http.Request) (*url.URL, error) {
|
|
proxyURL, err := url.Parse(env.Proxy)
|
|
require.NoError(t, err)
|
|
t.Logf("passing request for %s through proxy %s", testlib.RedactURLParams(req.URL), proxyURL.String())
|
|
return proxyURL, nil
|
|
}
|
|
}
|
|
var (
|
|
resp *http.Response
|
|
conn *websocket.Conn
|
|
)
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
var err error
|
|
conn, resp, err = dialer.Dial(dest.String(), http.Header{"Origin": {dest.String()}})
|
|
if resp != nil {
|
|
defer func() { requireEventually.NoError(resp.Body.Close()) }()
|
|
}
|
|
if err != nil && resp != nil {
|
|
body, _ := io.ReadAll(resp.Body)
|
|
t.Logf("websocket dial failed: %d:%s", resp.StatusCode, body)
|
|
}
|
|
requireEventually.NoError(err)
|
|
}, time.Minute, time.Second)
|
|
|
|
// perform a create through the admin client
|
|
wantConfigMap := &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}},
|
|
}
|
|
wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx,
|
|
wantConfigMap,
|
|
metav1.CreateOptions{},
|
|
)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
require.NoError(t, adminClient.CoreV1().ConfigMaps(namespaceName).
|
|
DeleteCollection(context.Background(), metav1.DeleteOptions{}, metav1.ListOptions{}))
|
|
})
|
|
|
|
// see if the websocket client received an event for the create
|
|
_, message, err := conn.ReadMessage()
|
|
if err != nil {
|
|
t.Fatalf("Unexpected error: %v", err)
|
|
}
|
|
var got watchJSON
|
|
err = json.Unmarshal(message, &got)
|
|
require.NoError(t, err)
|
|
if got.Type != watch.Added {
|
|
t.Errorf("Unexpected type: %v", got.Type)
|
|
}
|
|
var actualConfigMap corev1.ConfigMap
|
|
require.NoError(t, json.Unmarshal(got.Object, &actualConfigMap))
|
|
actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create.
|
|
require.Equal(t, *wantConfigMap, actualConfigMap)
|
|
})
|
|
|
|
t.Run("http2 client", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
namespaceName := testlib.CreateNamespace(ctx, t, "impersonation").Name
|
|
|
|
wantConfigMapLabelKey, wantConfigMapLabelValue := "some-label-key", "some-label-value"
|
|
wantConfigMap := &corev1.ConfigMap{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "configmap-1", Labels: map[string]string{wantConfigMapLabelKey: wantConfigMapLabelValue}},
|
|
}
|
|
wantConfigMap, err = adminClient.CoreV1().ConfigMaps(namespaceName).Create(ctx,
|
|
wantConfigMap,
|
|
metav1.CreateOptions{},
|
|
)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
_ = adminClient.CoreV1().ConfigMaps(namespaceName).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{})
|
|
})
|
|
|
|
// create rest client
|
|
restConfig := impersonationProxyRestConfig(
|
|
refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM),
|
|
impersonationProxyURL, impersonationProxyCACertPEM, nil,
|
|
)
|
|
|
|
tlsConfig, err := rest.TLSConfigFor(restConfig)
|
|
require.NoError(t, err)
|
|
httpTransport := http.Transport{
|
|
TLSClientConfig: tlsConfig,
|
|
}
|
|
if !clusterSupportsLoadBalancers {
|
|
httpTransport.Proxy = func(req *http.Request) (*url.URL, error) {
|
|
proxyURL, err := url.Parse(env.Proxy)
|
|
require.NoError(t, err)
|
|
t.Logf("passing request for %s through proxy %s", testlib.RedactURLParams(req.URL), proxyURL.String())
|
|
return proxyURL, nil
|
|
}
|
|
}
|
|
err = http2.ConfigureTransport(&httpTransport)
|
|
require.NoError(t, err)
|
|
|
|
httpClient := http.Client{
|
|
Transport: &httpTransport,
|
|
}
|
|
|
|
dest, _ := url.Parse(impersonationProxyURL)
|
|
dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps/configmap-1"
|
|
getConfigmapRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil)
|
|
require.NoError(t, err)
|
|
response, err := httpClient.Do(getConfigmapRequest)
|
|
require.NoError(t, err)
|
|
body, _ := io.ReadAll(response.Body)
|
|
t.Logf("http2 status code: %d, proto: %s, message: %s", response.StatusCode, response.Proto, body)
|
|
require.Equal(t, "HTTP/2.0", response.Proto)
|
|
require.Equal(t, http.StatusOK, response.StatusCode)
|
|
defer func() {
|
|
require.NoError(t, response.Body.Close())
|
|
}()
|
|
var actualConfigMap corev1.ConfigMap
|
|
require.NoError(t, json.Unmarshal(body, &actualConfigMap))
|
|
actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create.
|
|
require.Equal(t, *wantConfigMap, actualConfigMap)
|
|
|
|
// watch configmaps
|
|
dest.Path = "/api/v1/namespaces/" + namespaceName + "/configmaps"
|
|
dest.RawQuery = url.Values{
|
|
"watch": {"1"},
|
|
"labelSelector": {fmt.Sprintf("%s=%s", wantConfigMapLabelKey, wantConfigMapLabelValue)},
|
|
"resourceVersion": {"0"},
|
|
}.Encode()
|
|
watchConfigmapsRequest, err := http.NewRequestWithContext(ctx, http.MethodGet, dest.String(), nil)
|
|
require.NoError(t, err)
|
|
response, err = httpClient.Do(watchConfigmapsRequest)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "HTTP/2.0", response.Proto)
|
|
require.Equal(t, http.StatusOK, response.StatusCode)
|
|
defer func() {
|
|
require.NoError(t, response.Body.Close())
|
|
}()
|
|
|
|
// decode
|
|
decoder := json.NewDecoder(response.Body)
|
|
var got watchJSON
|
|
err = decoder.Decode(&got)
|
|
require.NoError(t, err)
|
|
if got.Type != watch.Added {
|
|
t.Errorf("Unexpected type: %v", got.Type)
|
|
}
|
|
err = json.Unmarshal(got.Object, &actualConfigMap)
|
|
require.NoError(t, err)
|
|
require.Equal(t, "configmap-1", actualConfigMap.Name)
|
|
actualConfigMap.TypeMeta = metav1.TypeMeta{} // This isn't filled out in the wantConfigMap we got back from create.
|
|
require.Equal(t, *wantConfigMap, actualConfigMap)
|
|
})
|
|
|
|
t.Run("honors anonymous authentication of KAS", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
impersonationProxyAnonymousClient := newAnonymousImpersonationProxyClient(
|
|
t, impersonationProxyURL, impersonationProxyCACertPEM, nil,
|
|
)
|
|
|
|
copyConfig := rest.CopyConfig(impersonationProxyAnonymousClient.JSONConfig)
|
|
copyConfig.GroupVersion = &schema.GroupVersion{}
|
|
copyConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
|
|
impersonationProxyAnonymousRestClient, err := rest.RESTClientFor(copyConfig)
|
|
require.NoError(t, err)
|
|
|
|
adminClientRestConfig := testlib.NewClientConfig(t)
|
|
clusterAdminCredentials := getCredForConfig(t, adminClientRestConfig)
|
|
impersonationProxyAdminClientAsAnonymousConfig := newImpersonationProxyClientWithCredentials(t,
|
|
clusterAdminCredentials,
|
|
impersonationProxyURL, impersonationProxyCACertPEM,
|
|
&rest.ImpersonationConfig{UserName: user.Anonymous}).
|
|
JSONConfig
|
|
impersonationProxyAdminClientAsAnonymousConfigCopy := rest.CopyConfig(impersonationProxyAdminClientAsAnonymousConfig)
|
|
impersonationProxyAdminClientAsAnonymousConfigCopy.GroupVersion = &schema.GroupVersion{}
|
|
impersonationProxyAdminClientAsAnonymousConfigCopy.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
|
|
impersonationProxyAdminRestClientAsAnonymous, err := rest.RESTClientFor(impersonationProxyAdminClientAsAnonymousConfigCopy)
|
|
require.NoError(t, err)
|
|
|
|
t.Run("anonymous authentication irrelevant", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
// - hit the token credential request endpoint with an empty body
|
|
// - through the impersonation proxy
|
|
// - should succeed as an invalid request whether anonymous authentication is enabled or disabled
|
|
// - should not reject as unauthorized
|
|
t.Run("token credential request", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
tkr, err := impersonationProxyAnonymousClient.PinnipedConcierge.LoginV1alpha1().TokenCredentialRequests().
|
|
Create(ctx, &loginv1alpha1.TokenCredentialRequest{
|
|
Spec: loginv1alpha1.TokenCredentialRequestSpec{
|
|
Authenticator: corev1.TypedLocalObjectReference{APIGroup: pointer.String("anything.pinniped.dev")},
|
|
},
|
|
}, metav1.CreateOptions{})
|
|
require.True(t, k8serrors.IsInvalid(err), testlib.Sdump(err))
|
|
require.Equal(t, `.login.concierge.pinniped.dev "" is invalid: spec.token.value: Required value: token must be supplied`, err.Error())
|
|
require.Equal(t, &loginv1alpha1.TokenCredentialRequest{}, tkr)
|
|
})
|
|
|
|
// - hit the healthz endpoint (non-resource endpoint)
|
|
// - through the impersonation proxy
|
|
// - as cluster admin, impersonating anonymous user
|
|
// - should succeed, authentication happens as cluster-admin
|
|
// - whoami should confirm we are using impersonation
|
|
// - healthz should succeed, anonymous users can request this endpoint
|
|
// - healthz/log should fail, forbidden anonymous
|
|
t.Run("non-resource request while impersonating anonymous - nested impersonation", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
whoami, errWho := impersonationProxyAdminRestClientAsAnonymous.Post().Body([]byte(`{}`)).AbsPath("/apis/identity.concierge." + env.APIGroupSuffix + "/v1alpha1/whoamirequests").DoRaw(ctx)
|
|
require.NoError(t, errWho, testlib.Sdump(errWho))
|
|
require.True(t, strings.HasPrefix(string(whoami), `{"kind":"WhoAmIRequest","apiVersion":"identity.concierge.`+env.APIGroupSuffix+`/v1alpha1","metadata":{"creationTimestamp":null},"spec":{},"status":{"kubernetesUserInfo":{"user":{"username":"system:anonymous","groups":["system:unauthenticated"],"extra":{"original-user-info.impersonation-proxy.concierge.pinniped.dev":["{\"username\":`), string(whoami))
|
|
|
|
healthz, errHealth := impersonationProxyAdminRestClientAsAnonymous.Get().AbsPath("/healthz").DoRaw(ctx)
|
|
require.NoError(t, errHealth, testlib.Sdump(errHealth))
|
|
require.Equal(t, "ok", string(healthz))
|
|
|
|
healthzLog, errHealthzLog := impersonationProxyAdminRestClientAsAnonymous.Get().AbsPath("/healthz/log").DoRaw(ctx)
|
|
require.True(t, k8serrors.IsForbidden(errHealthzLog), "%s\n%s", testlib.Sdump(errHealthzLog), string(healthzLog))
|
|
require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"forbidden: User \"system:anonymous\" cannot get path \"/healthz/log\": decision made by impersonation-proxy.concierge.pinniped.dev","reason":"Forbidden","details":{},"code":403}`+"\n", string(healthzLog))
|
|
})
|
|
})
|
|
|
|
t.Run("anonymous authentication enabled", func(t *testing.T) {
|
|
testlib.IntegrationEnv(t).WithCapability(testlib.AnonymousAuthenticationSupported)
|
|
parallelIfNotEKS(t)
|
|
|
|
// anonymous auth enabled
|
|
// - hit the healthz endpoint (non-resource endpoint)
|
|
// - through the impersonation proxy
|
|
// - should succeed 200
|
|
// - should respond "ok"
|
|
t.Run("non-resource request", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
healthz, errHealth := impersonationProxyAnonymousRestClient.Get().AbsPath("/healthz").DoRaw(ctx)
|
|
require.NoError(t, errHealth, testlib.Sdump(errHealth))
|
|
require.Equal(t, "ok", string(healthz))
|
|
})
|
|
|
|
// - hit the pods endpoint (a resource endpoint)
|
|
// - through the impersonation proxy
|
|
// - should fail forbidden
|
|
// - system:anonymous cannot get pods
|
|
t.Run("resource", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
pod, err := impersonationProxyAnonymousClient.Kubernetes.CoreV1().Pods(metav1.NamespaceSystem).
|
|
Get(ctx, "does-not-matter", metav1.GetOptions{})
|
|
require.True(t, k8serrors.IsForbidden(err), testlib.Sdump(err))
|
|
require.EqualError(t, err, `pods "does-not-matter" is forbidden: User "system:anonymous" cannot get resource "pods" in API group "" in the namespace "kube-system": `+
|
|
`decision made by impersonation-proxy.concierge.pinniped.dev`, testlib.Sdump(err))
|
|
require.Equal(t, &corev1.Pod{}, pod)
|
|
})
|
|
|
|
// - request to whoami (pinniped resource endpoint)
|
|
// - through the impersonation proxy
|
|
// - should succeed 200
|
|
// - should respond "you are system:anonymous"
|
|
t.Run("pinniped resource request", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
whoAmI, err := impersonationProxyAnonymousClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
require.Equal(t,
|
|
expectedWhoAmIRequestResponse(
|
|
"system:anonymous",
|
|
[]string{"system:unauthenticated"},
|
|
nil,
|
|
),
|
|
whoAmI,
|
|
)
|
|
})
|
|
})
|
|
|
|
t.Run("anonymous authentication disabled", func(t *testing.T) {
|
|
testlib.IntegrationEnv(t).WithoutCapability(testlib.AnonymousAuthenticationSupported)
|
|
parallelIfNotEKS(t)
|
|
|
|
// - hit the healthz endpoint (non-resource endpoint)
|
|
// - through the impersonation proxy
|
|
// - should fail unauthorized
|
|
// - kube api server should reject it
|
|
t.Run("non-resource request", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
healthz, err := impersonationProxyAnonymousRestClient.Get().AbsPath("/healthz").DoRaw(ctx)
|
|
require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err))
|
|
require.Equal(t, `{"kind":"Status","apiVersion":"v1","metadata":{},"status":"Failure","message":"Unauthorized","reason":"Unauthorized","code":401}`+"\n", string(healthz))
|
|
})
|
|
|
|
// - hit the pods endpoint (a resource endpoint)
|
|
// - through the impersonation proxy
|
|
// - should fail unauthorized
|
|
// - kube api server should reject it
|
|
t.Run("resource", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
pod, err := impersonationProxyAnonymousClient.Kubernetes.CoreV1().Pods(metav1.NamespaceSystem).
|
|
Get(ctx, "does-not-matter", metav1.GetOptions{})
|
|
require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err))
|
|
require.Equal(t, &corev1.Pod{}, pod)
|
|
})
|
|
|
|
// - request to whoami (pinniped resource endpoing)
|
|
// - through the impersonation proxy
|
|
// - should fail unauthorized
|
|
// - kube api server should reject it
|
|
t.Run("pinniped resource request", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
whoAmI, err := impersonationProxyAnonymousClient.PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
require.True(t, k8serrors.IsUnauthorized(err), testlib.Sdump(err))
|
|
require.Equal(t, &identityv1alpha1.WhoAmIRequest{}, whoAmI)
|
|
})
|
|
})
|
|
})
|
|
|
|
t.Run("assert impersonator runs with secure TLS config", func(t *testing.T) {
|
|
parallelIfNotEKS(t)
|
|
|
|
cancelCtx, cancel := context.WithCancel(ctx)
|
|
t.Cleanup(cancel)
|
|
|
|
startKubectlPortForward(cancelCtx, t, "10445", "443", env.ConciergeAppName+"-proxy", env.ConciergeNamespace)
|
|
|
|
stdout, stderr := testlib.RunNmapSSLEnum(t, "127.0.0.1", 10445)
|
|
|
|
require.Empty(t, stderr)
|
|
require.Contains(t, stdout, testlib.GetExpectedCiphers(ptls.Default(nil)), "stdout:\n%s", stdout)
|
|
})
|
|
})
|
|
|
|
t.Run("assert correct impersonator service account is being used", func(t *testing.T) {
|
|
// pick an API that everyone can access but always make invalid requests to it
|
|
// we can tell that the request is reaching KAS because only it has the validation logic
|
|
impersonationProxySSRRClient := impersonationProxyKubeClient(t).AuthorizationV1().SelfSubjectRulesReviews()
|
|
crbClient := adminClient.RbacV1().ClusterRoleBindings()
|
|
impersonationProxyName := env.ConciergeAppName + "-impersonation-proxy"
|
|
saFullName := serviceaccount.MakeUsername(env.ConciergeNamespace, impersonationProxyName)
|
|
invalidSSRR := &authorizationv1.SelfSubjectRulesReview{}
|
|
|
|
// sanity check default expected error message
|
|
_, err := impersonationProxySSRRClient.Create(ctx, invalidSSRR, metav1.CreateOptions{})
|
|
require.True(t, k8serrors.IsBadRequest(err), testlib.Sdump(err))
|
|
require.EqualError(t, err, "no namespace on request")
|
|
|
|
// remove the impersonation proxy SA's permissions
|
|
crb, err := crbClient.Get(ctx, impersonationProxyName, metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// sanity check the subject
|
|
require.Len(t, crb.Subjects, 1)
|
|
sub := crb.Subjects[0].DeepCopy()
|
|
require.Equal(t, &rbacv1.Subject{
|
|
Kind: "ServiceAccount",
|
|
APIGroup: "",
|
|
Name: impersonationProxyName,
|
|
Namespace: env.ConciergeNamespace,
|
|
}, sub)
|
|
|
|
crb.Subjects = nil
|
|
_, err = crbClient.Update(ctx, crb, metav1.UpdateOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// make sure to put the permissions back at the end
|
|
t.Cleanup(func() {
|
|
crbEnd, errEnd := crbClient.Get(ctx, impersonationProxyName, metav1.GetOptions{})
|
|
require.NoError(t, errEnd)
|
|
|
|
crbEnd.Subjects = []rbacv1.Subject{*sub}
|
|
_, errUpdate := crbClient.Update(ctx, crbEnd, metav1.UpdateOptions{})
|
|
require.NoError(t, errUpdate)
|
|
|
|
testlib.WaitForUserToHaveAccess(t, saFullName, nil, &authorizationv1.ResourceAttributes{
|
|
Verb: "impersonate",
|
|
Resource: "users",
|
|
})
|
|
})
|
|
|
|
// assert that the impersonation proxy stops working when we remove its permissions
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
_, errCreate := impersonationProxySSRRClient.Create(ctx, invalidSSRR, metav1.CreateOptions{})
|
|
|
|
switch {
|
|
case errCreate == nil:
|
|
return false, fmt.Errorf("unexpected nil error for test user create invalid SSRR")
|
|
|
|
case k8serrors.IsBadRequest(errCreate) && errCreate.Error() == "no namespace on request":
|
|
t.Log("waiting for impersonation proxy service account to lose impersonate permissions")
|
|
return false, nil // RBAC change has not rolled out yet
|
|
|
|
case k8serrors.IsForbidden(errCreate) && errCreate.Error() ==
|
|
`users "`+env.TestUser.ExpectedUsername+`" is forbidden: User "`+saFullName+
|
|
`" cannot impersonate resource "users" in API group "" at the cluster scope`:
|
|
return true, nil // expected RBAC error
|
|
|
|
default:
|
|
return false, fmt.Errorf("unexpected error for test user create invalid SSRR: %w", errCreate)
|
|
}
|
|
}, time.Minute, time.Second)
|
|
})
|
|
|
|
t.Run("adding an annotation reconciles the LoadBalancer service", func(t *testing.T) {
|
|
if !(impersonatorShouldHaveStartedAutomaticallyByDefault && clusterSupportsLoadBalancers) {
|
|
t.Skip("only running when the cluster is meant to be using LoadBalancer services")
|
|
}
|
|
|
|
// Use this string in all annotation keys added by this test, so the assertions can ignore annotation keys
|
|
// which might exist on the Service which are not related to this test.
|
|
recognizableAnnotationKeyString := "pinniped.dev"
|
|
|
|
// Grab the state of the CredentialIssuer prior to this test, so we can restore things back afterwards.
|
|
previous, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
updateServiceAnnotations := func(annotations map[string]string) {
|
|
require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
service, err := adminClient.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
updated := service.DeepCopy()
|
|
if updated.Annotations == nil {
|
|
updated.Annotations = map[string]string{}
|
|
}
|
|
// Add/update each requested annotation, without overwriting others that are already there.
|
|
for k, v := range annotations {
|
|
updated.Annotations[k] = v
|
|
}
|
|
if equality.Semantic.DeepEqual(service, updated) {
|
|
return nil
|
|
}
|
|
|
|
t.Logf("updating Service with annotations: %v", annotations)
|
|
_, err = adminClient.CoreV1().Services(env.ConciergeNamespace).Update(ctx, updated, metav1.UpdateOptions{})
|
|
return err
|
|
}))
|
|
}
|
|
|
|
deleteServiceAnnotations := func(annotations map[string]string) {
|
|
require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
service, err := adminClient.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
updated := service.DeepCopy()
|
|
if updated.Annotations != nil {
|
|
for k := range annotations {
|
|
delete(updated.Annotations, k)
|
|
}
|
|
}
|
|
if equality.Semantic.DeepEqual(service, updated) {
|
|
return nil
|
|
}
|
|
|
|
t.Logf("updating Service to remove annotations: %v", annotations)
|
|
_, err = adminClient.CoreV1().Services(env.ConciergeNamespace).Update(ctx, updated, metav1.UpdateOptions{})
|
|
return err
|
|
}))
|
|
}
|
|
|
|
applyCredentialIssuerAnnotations := func(annotations map[string]string) {
|
|
require.NoError(t, retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
issuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
updated := issuer.DeepCopy()
|
|
updated.Spec.ImpersonationProxy.Service.Annotations = annotations
|
|
if equality.Semantic.DeepEqual(issuer, updated) {
|
|
return nil
|
|
}
|
|
|
|
t.Logf("updating CredentialIssuer with spec.impersonationProxy.service.annotations: %v", annotations)
|
|
_, err = adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Update(ctx, updated, metav1.UpdateOptions{})
|
|
return err
|
|
}))
|
|
}
|
|
|
|
waitForServiceAnnotations := func(wantAnnotations map[string]string, annotationKeyFilter string) {
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
service, err := adminClient.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
filteredActualAnnotations := map[string]string{}
|
|
for k, v := range service.Annotations {
|
|
// We do want to pay attention to any annotation for which we intend to make an explicit assertion,
|
|
// e.g. "service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout" which is from our
|
|
// default CredentialIssuer spec.
|
|
_, wantToMakeAssertionOnThisAnnotation := wantAnnotations[k]
|
|
// We do not want to pay attention to Service annotations added by other controllers,
|
|
// e.g. the "cloud.google.com/neg" annotation that is sometimes added by GKE on Services.
|
|
// These can come and go in time intervals outside of our control.
|
|
annotationContainsFilterString := strings.Contains(k, annotationKeyFilter)
|
|
if wantToMakeAssertionOnThisAnnotation || annotationContainsFilterString {
|
|
filteredActualAnnotations[k] = v
|
|
}
|
|
}
|
|
t.Logf("found Service %s of type %s with actual annotations %q; filtered by interesting keys results in %q; expected annotations %q",
|
|
service.Name, service.Spec.Type, service.Annotations, filteredActualAnnotations, wantAnnotations)
|
|
return equality.Semantic.DeepEqual(filteredActualAnnotations, wantAnnotations), nil
|
|
}, 1*time.Minute, 1*time.Second)
|
|
}
|
|
|
|
expectedAnnotations := func(credentialIssuerSpecAnnotations map[string]string, otherAnnotations map[string]string) map[string]string {
|
|
credentialIssuerSpecAnnotationKeys := []string{}
|
|
expectedAnnotations := map[string]string{}
|
|
// Expect the annotations specified on the CredentialIssuer spec to be present.
|
|
for k, v := range credentialIssuerSpecAnnotations {
|
|
credentialIssuerSpecAnnotationKeys = append(credentialIssuerSpecAnnotationKeys, k)
|
|
expectedAnnotations[k] = v
|
|
}
|
|
// Aside from the annotations requested on the CredentialIssuer spec, also expect the other annotation to still be there too.
|
|
for k, v := range otherAnnotations {
|
|
expectedAnnotations[k] = v
|
|
}
|
|
// Also expect the internal bookkeeping annotation to be present. It tracks the requested keys from the spec.
|
|
// Our controller sorts these keys to make the order in the annotation's value predictable.
|
|
sort.Strings(credentialIssuerSpecAnnotationKeys)
|
|
credentialIssuerSpecAnnotationKeysJSON, err := json.Marshal(credentialIssuerSpecAnnotationKeys)
|
|
require.NoError(t, err)
|
|
// The name of this annotation key is decided by our controller.
|
|
expectedAnnotations["credentialissuer."+recognizableAnnotationKeyString+"/annotation-keys"] = string(credentialIssuerSpecAnnotationKeysJSON)
|
|
return expectedAnnotations
|
|
}
|
|
|
|
otherActorAnnotations := map[string]string{
|
|
recognizableAnnotationKeyString + "/test-other-actor-" + testlib.RandHex(t, 8): "test-other-actor-" + testlib.RandHex(t, 8),
|
|
}
|
|
|
|
// Whatever happens, set the annotations back to the original value and expect the Service to be updated.
|
|
t.Cleanup(func() {
|
|
t.Log("reverting CredentialIssuer back to previous configuration")
|
|
deleteServiceAnnotations(otherActorAnnotations)
|
|
applyCredentialIssuerAnnotations(previous.Spec.ImpersonationProxy.Service.DeepCopy().Annotations)
|
|
waitForServiceAnnotations(
|
|
expectedAnnotations(previous.Spec.ImpersonationProxy.Service.DeepCopy().Annotations, map[string]string{}),
|
|
recognizableAnnotationKeyString,
|
|
)
|
|
})
|
|
|
|
// Having another actor, like a human or a non-Pinniped controller, add unrelated annotations to the Service
|
|
// should not cause the Pinniped controllers to overwrite those annotations.
|
|
updateServiceAnnotations(otherActorAnnotations)
|
|
|
|
// Set a new annotation in the CredentialIssuer spec.impersonationProxy.service.annotations field.
|
|
newAnnotationKey := recognizableAnnotationKeyString + "/test-" + testlib.RandHex(t, 8)
|
|
newAnnotationValue := "test-" + testlib.RandHex(t, 8)
|
|
updatedAnnotations := previous.Spec.ImpersonationProxy.Service.DeepCopy().Annotations
|
|
updatedAnnotations[newAnnotationKey] = newAnnotationValue
|
|
applyCredentialIssuerAnnotations(updatedAnnotations)
|
|
|
|
// Expect them to be applied to the Service.
|
|
waitForServiceAnnotations(
|
|
expectedAnnotations(updatedAnnotations, otherActorAnnotations),
|
|
recognizableAnnotationKeyString,
|
|
)
|
|
})
|
|
|
|
t.Run("running impersonation proxy with ClusterIP service", func(t *testing.T) {
|
|
if env.Proxy == "" {
|
|
t.Skip("Skipping ClusterIP test because squid proxy is not present")
|
|
}
|
|
clusterIPServiceURL := fmt.Sprintf("%s.%s.svc.cluster.local", impersonationProxyClusterIPName(env), env.ConciergeNamespace)
|
|
updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{
|
|
ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{
|
|
Mode: conciergev1alpha.ImpersonationProxyModeEnabled,
|
|
ExternalEndpoint: clusterIPServiceURL,
|
|
Service: conciergev1alpha.ImpersonationProxyServiceSpec{
|
|
Type: conciergev1alpha.ImpersonationProxyServiceTypeClusterIP,
|
|
},
|
|
},
|
|
})
|
|
|
|
// wait until the credential issuer is updated with the new url
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
newImpersonationProxyURL, _ := performImpersonatorDiscoveryURL(ctx, t, env, adminConciergeClient)
|
|
return newImpersonationProxyURL == "https://"+clusterIPServiceURL, nil
|
|
}, 30*time.Second, 500*time.Millisecond)
|
|
newImpersonationProxyURL, newImpersonationProxyCACertPEM := performImpersonatorDiscovery(ctx, t, env, adminClient, adminConciergeClient, refreshCredential)
|
|
|
|
anonymousClient := newAnonymousImpersonationProxyClientWithProxy(t, newImpersonationProxyURL, newImpersonationProxyCACertPEM, nil).PinnipedConcierge
|
|
refreshedCredentials := refreshCredentialHelper(t, anonymousClient)
|
|
|
|
client := newImpersonationProxyClientWithCredentialsAndProxy(t, refreshedCredentials, newImpersonationProxyURL, newImpersonationProxyCACertPEM, nil).Kubernetes
|
|
|
|
// everything should work properly through the cluster ip service
|
|
t.Run(
|
|
"access as user",
|
|
testlib.AccessAsUserTest(ctx, env.TestUser.ExpectedUsername, client),
|
|
)
|
|
})
|
|
|
|
t.Run("manually disabling the impersonation proxy feature", func(t *testing.T) {
|
|
// Update configuration to force the proxy to disabled mode
|
|
updateCredentialIssuer(ctx, t, env, adminConciergeClient, conciergev1alpha.CredentialIssuerSpec{
|
|
ImpersonationProxy: &conciergev1alpha.ImpersonationProxySpec{
|
|
Mode: conciergev1alpha.ImpersonationProxyModeDisabled,
|
|
},
|
|
})
|
|
|
|
if clusterSupportsLoadBalancers {
|
|
// The load balancer should have been deleted when we disabled the impersonation proxy.
|
|
// Note that this can take kind of a long time on real cloud providers (e.g. ~22 seconds on EKS).
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
hasService, err := hasImpersonationProxyLoadBalancerService(ctx, env, adminClient)
|
|
return !hasService, err
|
|
}, 2*time.Minute, 500*time.Millisecond)
|
|
}
|
|
|
|
// Check that the impersonation proxy port has shut down.
|
|
// Ideally we could always check that the impersonation proxy's port has shut down, but on clusters where we
|
|
// do not run the squid proxy we have no easy way to see beyond the load balancer to see inside the cluster,
|
|
// so we'll skip this check on clusters which have load balancers but don't run the squid proxy.
|
|
// The other cluster types that do run the squid proxy will give us sufficient coverage here.
|
|
if env.Proxy != "" {
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
// It's okay if this returns RBAC errors because this user has no role bindings.
|
|
// What we want to see is that the proxy eventually shuts down entirely.
|
|
_, err := impersonationProxyViaSquidKubeClientWithoutCredential(t, proxyServiceEndpoint).CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
|
|
isErr, _ := isServiceUnavailableViaSquidError(err, proxyServiceEndpoint)
|
|
requireEventually.Truef(isErr, "wanted service unavailable via squid error, got %v", err)
|
|
}, 20*time.Second, 500*time.Millisecond)
|
|
}
|
|
|
|
// Check that the generated TLS cert Secret was deleted by the controller because it's supposed to clean this up
|
|
// when we disable the impersonator.
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
_, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyTLSSecretName(env), metav1.GetOptions{})
|
|
requireEventually.Truef(k8serrors.IsNotFound(err), "expected NotFound error, got %v", err)
|
|
}, time.Minute, time.Second)
|
|
|
|
// Check that the generated CA cert Secret was not deleted by the controller because it's supposed to keep this
|
|
// around in case we decide to later re-enable the impersonator. We want to avoid generating new CA certs when
|
|
// possible because they make their way into kubeconfigs on client machines.
|
|
_, err := adminClient.CoreV1().Secrets(env.ConciergeNamespace).Get(ctx, impersonationProxyCASecretName(env), metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
// At this point the impersonator should be stopped. The CredentialIssuer's strategies array should be updated to
|
|
// include an unsuccessful impersonation strategy saying that it was manually configured to be disabled.
|
|
requireDisabledStrategy(ctx, t, env, adminConciergeClient)
|
|
|
|
if !env.HasCapability(testlib.ClusterSigningKeyIsAvailable) && env.HasCapability(testlib.AnonymousAuthenticationSupported) {
|
|
// This cluster does not support the cluster signing key strategy, so now that we've manually disabled the
|
|
// impersonation strategy, we should be left with no working strategies.
|
|
// Given that there are no working strategies, a TokenCredentialRequest which would otherwise work should now
|
|
// fail, because there is no point handing out credentials that are not going to work for any strategy.
|
|
// Note that library.CreateTokenCredentialRequest makes an unauthenticated request, so we can't meaningfully
|
|
// perform this part of the test on a cluster which does not allow anonymous authentication.
|
|
tokenCredentialRequestResponse, err := testlib.CreateTokenCredentialRequest(ctx, t, credentialRequestSpecWithWorkingCredentials)
|
|
require.NoError(t, err)
|
|
|
|
require.NotNil(t, tokenCredentialRequestResponse.Status.Message, "expected an error message but got nil")
|
|
require.Equal(t, "authentication failed", *tokenCredentialRequestResponse.Status.Message)
|
|
require.Nil(t, tokenCredentialRequestResponse.Status.Credential)
|
|
}
|
|
})
|
|
}
|
|
|
|
func ensureDNSResolves(t *testing.T, urlString string) {
|
|
t.Helper()
|
|
|
|
parsedURL, err := url.Parse(urlString)
|
|
require.NoError(t, err)
|
|
|
|
host := parsedURL.Hostname()
|
|
|
|
if net.ParseIP(host) != nil {
|
|
return // ignore IPs
|
|
}
|
|
|
|
var d net.Dialer
|
|
loggingDialer := func(ctx context.Context, network, address string) (net.Conn, error) {
|
|
t.Logf("dns lookup, network=%s address=%s", network, address)
|
|
conn, connErr := d.DialContext(ctx, network, address)
|
|
if connErr != nil {
|
|
t.Logf("dns lookup, err=%v", connErr)
|
|
} else {
|
|
local := conn.LocalAddr()
|
|
remote := conn.RemoteAddr()
|
|
t.Logf("dns lookup, local conn network=%s addr=%s", local.Network(), local.String())
|
|
t.Logf("dns lookup, remote conn network=%s addr=%s", remote.Network(), remote.String())
|
|
}
|
|
return conn, connErr
|
|
}
|
|
|
|
goResolver := &net.Resolver{
|
|
PreferGo: true,
|
|
StrictErrors: true,
|
|
Dial: loggingDialer,
|
|
}
|
|
notGoResolver := &net.Resolver{
|
|
PreferGo: false,
|
|
StrictErrors: true,
|
|
Dial: loggingDialer,
|
|
}
|
|
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
|
defer cancel()
|
|
|
|
for _, resolver := range []*net.Resolver{goResolver, notGoResolver} {
|
|
resolver := resolver
|
|
|
|
ips, ipErr := resolver.LookupIPAddr(ctx, host)
|
|
requireEventually.NoError(ipErr)
|
|
requireEventually.NotEmpty(ips)
|
|
}
|
|
}, 5*time.Minute, 1*time.Second)
|
|
}
|
|
|
|
func createServiceAccountToken(ctx context.Context, t *testing.T, adminClient kubernetes.Interface, namespaceName string) (name, token string, uid types.UID) {
|
|
t.Helper()
|
|
|
|
serviceAccount, err := adminClient.CoreV1().ServiceAccounts(namespaceName).Create(ctx,
|
|
&corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{GenerateName: "int-test-service-account-"}}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
require.NoError(t, adminClient.CoreV1().ServiceAccounts(namespaceName).
|
|
Delete(context.Background(), serviceAccount.Name, metav1.DeleteOptions{}))
|
|
})
|
|
|
|
secret, err := adminClient.CoreV1().Secrets(namespaceName).Create(ctx, &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "int-test-service-account-token-",
|
|
Annotations: map[string]string{
|
|
corev1.ServiceAccountNameKey: serviceAccount.Name,
|
|
},
|
|
},
|
|
Type: corev1.SecretTypeServiceAccountToken,
|
|
}, metav1.CreateOptions{})
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
require.NoError(t, adminClient.CoreV1().Secrets(namespaceName).
|
|
Delete(context.Background(), secret.Name, metav1.DeleteOptions{}))
|
|
})
|
|
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
secret, err = adminClient.CoreV1().Secrets(namespaceName).Get(ctx, secret.Name, metav1.GetOptions{})
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return len(secret.Data[corev1.ServiceAccountTokenKey]) > 0, nil
|
|
}, time.Minute, time.Second)
|
|
|
|
return serviceAccount.Name, string(secret.Data[corev1.ServiceAccountTokenKey]), serviceAccount.UID
|
|
}
|
|
|
|
func expectedWhoAmIRequestResponse(username string, groups []string, extra map[string]identityv1alpha1.ExtraValue) *identityv1alpha1.WhoAmIRequest {
|
|
return &identityv1alpha1.WhoAmIRequest{
|
|
Status: identityv1alpha1.WhoAmIRequestStatus{
|
|
KubernetesUserInfo: identityv1alpha1.KubernetesUserInfo{
|
|
User: identityv1alpha1.UserInfo{
|
|
Username: username,
|
|
UID: "", // no way to impersonate UID: https://github.com/kubernetes/kubernetes/issues/93699
|
|
Groups: groups,
|
|
Extra: extra,
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func performImpersonatorDiscovery(ctx context.Context, t *testing.T, env *testlib.TestEnv,
|
|
adminClient kubernetes.Interface, adminConciergeClient pinnipedconciergeclientset.Interface,
|
|
refreshCredential func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential) (string, []byte) {
|
|
t.Helper()
|
|
|
|
impersonationProxyURL, impersonationProxyCACertPEM := performImpersonatorDiscoveryURL(ctx, t, env, adminConciergeClient)
|
|
|
|
if len(env.Proxy) == 0 {
|
|
t.Log("no test proxy is available, skipping readiness checks for concierge impersonation proxy pods")
|
|
return impersonationProxyURL, impersonationProxyCACertPEM
|
|
}
|
|
|
|
impersonationProxyParsedURL, err := url.Parse(impersonationProxyURL)
|
|
require.NoError(t, err)
|
|
|
|
expectedGroups := make([]string, 0, len(env.TestUser.ExpectedGroups)+1) // make sure we do not mutate env.TestUser.ExpectedGroups
|
|
expectedGroups = append(expectedGroups, env.TestUser.ExpectedGroups...)
|
|
expectedGroups = append(expectedGroups, "system:authenticated")
|
|
|
|
// probe each pod directly for readiness since the concierge status is a lie - it just means a single pod is ready
|
|
testlib.RequireEventually(t, func(requireEventually *require.Assertions) {
|
|
pods, err := adminClient.CoreV1().Pods(env.ConciergeNamespace).List(ctx,
|
|
metav1.ListOptions{LabelSelector: "deployment.pinniped.dev=concierge"})
|
|
requireEventually.NoError(err)
|
|
requireEventually.Len(pods.Items, 2) // has to stay in sync with the defaults in our YAML
|
|
|
|
for _, pod := range pods.Items {
|
|
t.Logf("checking if concierge impersonation proxy pod %q is ready", pod.Name)
|
|
|
|
requireEventually.NotEmptyf(pod.Status.PodIP, "pod %q does not have an IP", pod.Name)
|
|
|
|
credentials := refreshCredential(t, impersonationProxyURL, impersonationProxyCACertPEM).DeepCopy()
|
|
credentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests
|
|
|
|
config := newImpersonationProxyConfigWithCredentials(t, credentials, impersonationProxyURL, impersonationProxyCACertPEM, nil)
|
|
config = rest.CopyConfig(config)
|
|
config.Proxy = kubeconfigProxyFunc(t, env.Proxy) // always use the proxy since we are talking directly to a pod IP
|
|
config.Host = "https://" + pod.Status.PodIP + ":8444" // hardcode the internal port - it should not change
|
|
config.TLSClientConfig.ServerName = impersonationProxyParsedURL.Hostname() // make SNI hostname TLS verification work even when using IP
|
|
|
|
whoAmI, err := testlib.NewKubeclient(t, config).PinnipedConcierge.IdentityV1alpha1().WhoAmIRequests().
|
|
Create(ctx, &identityv1alpha1.WhoAmIRequest{}, metav1.CreateOptions{})
|
|
requireEventually.NoError(err)
|
|
requireEventually.Equal(
|
|
expectedWhoAmIRequestResponse(
|
|
env.TestUser.ExpectedUsername,
|
|
expectedGroups,
|
|
nil,
|
|
),
|
|
whoAmI,
|
|
)
|
|
}
|
|
}, 10*time.Minute, 10*time.Second)
|
|
|
|
return impersonationProxyURL, impersonationProxyCACertPEM
|
|
}
|
|
|
|
func performImpersonatorDiscoveryURL(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface) (string, []byte) {
|
|
t.Helper()
|
|
|
|
var impersonationProxyURL string
|
|
var impersonationProxyCACertPEM []byte
|
|
|
|
t.Log("Waiting for CredentialIssuer strategy to be successful")
|
|
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
if err != nil || credentialIssuer.Status.Strategies == nil {
|
|
t.Log("Did not find any CredentialIssuer with any strategies")
|
|
return false, nil // didn't find it, but keep trying
|
|
}
|
|
for _, strategy := range credentialIssuer.Status.Strategies {
|
|
// There will be other strategy types in the list, so ignore those.
|
|
if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType && strategy.Status == conciergev1alpha.SuccessStrategyStatus { //nolint:nestif
|
|
if strategy.Frontend == nil {
|
|
return false, fmt.Errorf("did not find a Frontend") // unexpected, fail the test
|
|
}
|
|
if strategy.Frontend.ImpersonationProxyInfo == nil {
|
|
return false, fmt.Errorf("did not find an ImpersonationProxyInfo") // unexpected, fail the test
|
|
}
|
|
impersonationProxyURL = strategy.Frontend.ImpersonationProxyInfo.Endpoint
|
|
|
|
impersonationProxyCACertPEM, err = base64.StdEncoding.DecodeString(strategy.Frontend.ImpersonationProxyInfo.CertificateAuthorityData)
|
|
if err != nil {
|
|
return false, err // unexpected, fail the test
|
|
}
|
|
return true, nil // found it, continue the test!
|
|
} else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType {
|
|
t.Logf("Waiting for successful impersonation proxy strategy on %s: found status %s with reason %s and message: %s",
|
|
credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message)
|
|
if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason {
|
|
// The server encountered an unexpected error while starting the impersonator, so fail the test fast.
|
|
return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message)
|
|
}
|
|
}
|
|
}
|
|
t.Log("Did not find any impersonation proxy strategy on CredentialIssuer")
|
|
return false, nil // didn't find it, but keep trying
|
|
}, 10*time.Minute, 10*time.Second)
|
|
|
|
t.Log("Found successful CredentialIssuer strategy")
|
|
return impersonationProxyURL, impersonationProxyCACertPEM
|
|
}
|
|
|
|
func requireDisabledStrategy(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface) {
|
|
t.Helper()
|
|
|
|
testlib.RequireEventuallyWithoutError(t, func() (bool, error) {
|
|
credentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
if err != nil || credentialIssuer.Status.Strategies == nil {
|
|
t.Log("Did not find any CredentialIssuer with any strategies")
|
|
return false, nil // didn't find it, but keep trying
|
|
}
|
|
for _, strategy := range credentialIssuer.Status.Strategies {
|
|
// There will be other strategy types in the list, so ignore those.
|
|
if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType &&
|
|
strategy.Status == conciergev1alpha.ErrorStrategyStatus &&
|
|
strategy.Reason == conciergev1alpha.DisabledStrategyReason {
|
|
return true, nil // found it, continue the test!
|
|
} else if strategy.Type == conciergev1alpha.ImpersonationProxyStrategyType {
|
|
t.Logf("Waiting for disabled impersonation proxy strategy on %s: found status %s with reason %s and message: %s",
|
|
credentialIssuerName(env), strategy.Status, strategy.Reason, strategy.Message)
|
|
if strategy.Reason == conciergev1alpha.ErrorDuringSetupStrategyReason {
|
|
// The server encountered an unexpected error while stopping the impersonator, so fail the test fast.
|
|
return false, fmt.Errorf("found impersonation strategy in %s state with message: %s", strategy.Reason, strategy.Message)
|
|
}
|
|
}
|
|
}
|
|
t.Log("Did not find any impersonation proxy strategy on CredentialIssuer")
|
|
return false, nil // didn't find it, but keep trying
|
|
}, 1*time.Minute, 500*time.Millisecond)
|
|
}
|
|
|
|
func credentialAlmostExpired(t *testing.T, credential *loginv1alpha1.TokenCredentialRequest) bool {
|
|
t.Helper()
|
|
|
|
pemBlock, _ := pem.Decode([]byte(credential.Status.Credential.ClientCertificateData))
|
|
parsedCredential, err := x509.ParseCertificate(pemBlock.Bytes)
|
|
require.NoError(t, err)
|
|
timeRemaining := time.Until(parsedCredential.NotAfter)
|
|
if timeRemaining < 2*time.Minute {
|
|
t.Logf("The TokenCredentialRequest cred is almost expired and needs to be refreshed. Expires in %s.", timeRemaining)
|
|
return true
|
|
}
|
|
t.Logf("The TokenCredentialRequest cred is good for some more time (%s) so using it.", timeRemaining)
|
|
return false
|
|
}
|
|
|
|
func impersonationProxyRestConfig(credential *loginv1alpha1.ClusterCredential, host string, caData []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *rest.Config {
|
|
config := rest.Config{
|
|
Host: host,
|
|
TLSClientConfig: rest.TLSClientConfig{
|
|
Insecure: caData == nil,
|
|
CAData: caData,
|
|
CertData: []byte(credential.ClientCertificateData),
|
|
KeyData: []byte(credential.ClientKeyData),
|
|
},
|
|
// kubectl would set both the client cert and the token, so we'll do that too.
|
|
// The Kube API server will ignore the token if the client cert successfully authenticates.
|
|
// Only if the client cert is not present or fails to authenticate will it use the token.
|
|
// Historically, it works that way because some web browsers will always send your
|
|
// corporate-assigned client cert even if it is not valid, and it doesn't want to treat
|
|
// that as a failure if you also sent a perfectly good token.
|
|
// We would like the impersonation proxy to imitate that behavior, so we test it here.
|
|
BearerToken: credential.Token,
|
|
}
|
|
if nestedImpersonationConfig != nil {
|
|
config.Impersonate = *nestedImpersonationConfig
|
|
}
|
|
return &config
|
|
}
|
|
|
|
func kubeconfigProxyFunc(t *testing.T, squidProxyURL string) func(req *http.Request) (*url.URL, error) {
|
|
return func(req *http.Request) (*url.URL, error) {
|
|
t.Helper()
|
|
|
|
parsedSquidProxyURL, err := url.Parse(squidProxyURL)
|
|
require.NoError(t, err)
|
|
t.Logf("passing request for %s through proxy %s", testlib.RedactURLParams(req.URL), parsedSquidProxyURL.String())
|
|
return parsedSquidProxyURL, nil
|
|
}
|
|
}
|
|
|
|
func updateCredentialIssuer(ctx context.Context, t *testing.T, env *testlib.TestEnv, adminConciergeClient pinnipedconciergeclientset.Interface, spec conciergev1alpha.CredentialIssuerSpec) {
|
|
t.Helper()
|
|
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
newCredentialIssuer, err := adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Get(ctx, credentialIssuerName(env), metav1.GetOptions{})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
spec.DeepCopyInto(&newCredentialIssuer.Spec)
|
|
_, err = adminConciergeClient.ConfigV1alpha1().CredentialIssuers().Update(ctx, newCredentialIssuer, metav1.UpdateOptions{})
|
|
return err
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func hasImpersonationProxyLoadBalancerService(ctx context.Context, env *testlib.TestEnv, client kubernetes.Interface) (bool, error) {
|
|
service, err := client.CoreV1().Services(env.ConciergeNamespace).Get(ctx, impersonationProxyLoadBalancerName(env), metav1.GetOptions{})
|
|
if k8serrors.IsNotFound(err) {
|
|
return false, nil
|
|
}
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
return service.Spec.Type == corev1.ServiceTypeLoadBalancer, nil
|
|
}
|
|
|
|
func impersonationProxyTLSSecretName(env *testlib.TestEnv) string {
|
|
return env.ConciergeAppName + "-impersonation-proxy-tls-serving-certificate"
|
|
}
|
|
|
|
func impersonationProxyCASecretName(env *testlib.TestEnv) string {
|
|
return env.ConciergeAppName + "-impersonation-proxy-ca-certificate"
|
|
}
|
|
|
|
func impersonationProxyLoadBalancerName(env *testlib.TestEnv) string {
|
|
return env.ConciergeAppName + "-impersonation-proxy-load-balancer"
|
|
}
|
|
|
|
func impersonationProxyClusterIPName(env *testlib.TestEnv) string {
|
|
return env.ConciergeAppName + "-impersonation-proxy-cluster-ip"
|
|
}
|
|
|
|
func credentialIssuerName(env *testlib.TestEnv) string {
|
|
return env.ConciergeAppName + "-config"
|
|
}
|
|
|
|
func getImpersonationKubeconfig(t *testing.T, env *testlib.TestEnv, impersonationProxyURL string, impersonationProxyCACertPEM []byte, authenticator corev1.TypedLocalObjectReference) (string, []string, string) {
|
|
t.Helper()
|
|
|
|
pinnipedExe := testlib.PinnipedCLIPath(t)
|
|
tempDir := testutil.TempDir(t)
|
|
|
|
var envVarsWithProxy []string
|
|
if !env.HasCapability(testlib.HasExternalLoadBalancerProvider) {
|
|
// Only if you don't have a load balancer, use the squid proxy when it's available.
|
|
envVarsWithProxy = append(os.Environ(), env.ProxyEnv()...)
|
|
}
|
|
|
|
// Get the kubeconfig.
|
|
getKubeConfigCmd := []string{"get", "kubeconfig",
|
|
"--concierge-api-group-suffix", env.APIGroupSuffix,
|
|
"--oidc-skip-browser",
|
|
"--static-token", env.TestUser.Token,
|
|
"--concierge-authenticator-name", authenticator.Name,
|
|
"--concierge-authenticator-type", "webhook",
|
|
// Force the use of impersonation proxy strategy, but let it auto-discover the endpoint and CA.
|
|
"--concierge-mode", "ImpersonationProxy"}
|
|
t.Log("Running:", pinnipedExe, getKubeConfigCmd)
|
|
kubeconfigYAML, getKubeConfigStderr := runPinnipedCLI(t, envVarsWithProxy, pinnipedExe, getKubeConfigCmd...)
|
|
// "pinniped get kubectl" prints some status messages to stderr
|
|
t.Log(getKubeConfigStderr)
|
|
// Make sure that the "pinniped get kubeconfig" auto-discovered the impersonation proxy and we're going to
|
|
// make our kubectl requests through the impersonation proxy. Avoid using require.Contains because the error
|
|
// message would contain credentials.
|
|
require.True(t,
|
|
strings.Contains(kubeconfigYAML, "server: "+impersonationProxyURL+"\n"),
|
|
"the generated kubeconfig did not include the expected impersonation server address: %s",
|
|
impersonationProxyURL,
|
|
)
|
|
require.True(t,
|
|
strings.Contains(kubeconfigYAML, "- --concierge-ca-bundle-data="+base64.StdEncoding.EncodeToString(impersonationProxyCACertPEM)+"\n"),
|
|
"the generated kubeconfig did not include the base64 encoded version of this expected impersonation CA cert: %s",
|
|
impersonationProxyCACertPEM,
|
|
)
|
|
|
|
// Write the kubeconfig to a temp file.
|
|
kubeconfigPath := filepath.Join(tempDir, "kubeconfig.yaml")
|
|
require.NoError(t, os.WriteFile(kubeconfigPath, []byte(kubeconfigYAML), 0600))
|
|
|
|
return kubeconfigPath, envVarsWithProxy, tempDir
|
|
}
|
|
|
|
// func to create kubectl commands with a kubeconfig.
|
|
func kubectlCommand(timeout context.Context, t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (*exec.Cmd, *syncBuffer, *syncBuffer) {
|
|
t.Helper()
|
|
|
|
allArgs := append([]string{"--kubeconfig", kubeconfigPath}, args...)
|
|
kubectlCmd := exec.CommandContext(timeout, "kubectl", allArgs...)
|
|
var stdout, stderr syncBuffer
|
|
kubectlCmd.Stdout = &stdout
|
|
kubectlCmd.Stderr = &stderr
|
|
kubectlCmd.Env = envVarsWithProxy
|
|
|
|
t.Log("starting kubectl subprocess: kubectl", strings.Join(allArgs, " "))
|
|
return kubectlCmd, &stdout, &stderr
|
|
}
|
|
|
|
// Func to run kubeconfig commands.
|
|
func runKubectl(t *testing.T, kubeconfigPath string, envVarsWithProxy []string, args ...string) (string, error) {
|
|
timeout, cancelFunc := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancelFunc()
|
|
|
|
kubectlCmd, stdout, stderr := kubectlCommand(timeout, t, kubeconfigPath, envVarsWithProxy, args...)
|
|
|
|
err := kubectlCmd.Run()
|
|
t.Logf("kubectl stdout output: %s", stdout.String())
|
|
t.Logf("kubectl stderr output: %s", stderr.String())
|
|
return stdout.String(), err
|
|
}
|
|
|
|
// watchJSON defines the expected JSON wire equivalent of watch.Event.
|
|
type watchJSON struct {
|
|
Type watch.EventType `json:"type,omitempty"`
|
|
Object json.RawMessage `json:"object,omitempty"`
|
|
}
|
|
|
|
// requireServiceUnavailableViaSquidError returns whether the provided err is the error that is
|
|
// returned by squid when the impersonation proxy port inside the cluster is not listening.
|
|
func isServiceUnavailableViaSquidError(err error, proxyServiceEndpoint string) (bool, string) {
|
|
if err == nil {
|
|
return false, "error is nil"
|
|
}
|
|
|
|
for _, wantContains := range []string{
|
|
fmt.Sprintf(`Get "https://%s/api/v1/namespaces"`, proxyServiceEndpoint),
|
|
": Service Unavailable",
|
|
} {
|
|
if !strings.Contains(err.Error(), wantContains) {
|
|
return false, fmt.Sprintf("error does not contain %q", wantContains)
|
|
}
|
|
}
|
|
|
|
return true, ""
|
|
}
|
|
|
|
func requireClose(t *testing.T, c chan struct{}, timeout time.Duration) {
|
|
t.Helper()
|
|
|
|
timer := time.NewTimer(timeout)
|
|
select {
|
|
case <-c:
|
|
if !timer.Stop() {
|
|
<-timer.C
|
|
}
|
|
case <-timer.C:
|
|
require.FailNow(t, "failed to receive from channel within "+timeout.String())
|
|
}
|
|
}
|
|
|
|
func createTokenCredentialRequest(
|
|
spec loginv1alpha1.TokenCredentialRequestSpec,
|
|
client pinnipedconciergeclientset.Interface,
|
|
) (*loginv1alpha1.TokenCredentialRequest, error) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
return client.LoginV1alpha1().TokenCredentialRequests().Create(ctx,
|
|
&loginv1alpha1.TokenCredentialRequest{Spec: spec}, metav1.CreateOptions{},
|
|
)
|
|
}
|
|
|
|
func newImpersonationProxyClientWithCredentials(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client {
|
|
t.Helper()
|
|
|
|
kubeconfig := newImpersonationProxyConfigWithCredentials(t, credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig)
|
|
return testlib.NewKubeclient(t, kubeconfig)
|
|
}
|
|
|
|
func newImpersonationProxyConfigWithCredentials(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *rest.Config {
|
|
t.Helper()
|
|
|
|
env := testlib.IntegrationEnv(t)
|
|
clusterSupportsLoadBalancers := env.HasCapability(testlib.HasExternalLoadBalancerProvider)
|
|
|
|
kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig)
|
|
if !clusterSupportsLoadBalancers {
|
|
// Only if there is no possibility to send traffic through a load balancer, then send the traffic through the Squid proxy.
|
|
// Prefer to go through a load balancer because that's how the impersonator is intended to be used in the real world.
|
|
kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy)
|
|
}
|
|
return kubeconfig
|
|
}
|
|
|
|
func newAnonymousImpersonationProxyClient(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client {
|
|
t.Helper()
|
|
|
|
emptyCredentials := &loginv1alpha1.ClusterCredential{}
|
|
return newImpersonationProxyClientWithCredentials(t, emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig)
|
|
}
|
|
|
|
func newImpersonationProxyClientWithCredentialsAndProxy(t *testing.T, credentials *loginv1alpha1.ClusterCredential, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client {
|
|
t.Helper()
|
|
|
|
env := testlib.IntegrationEnv(t)
|
|
|
|
kubeconfig := impersonationProxyRestConfig(credentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig)
|
|
kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy)
|
|
return testlib.NewKubeclient(t, kubeconfig)
|
|
}
|
|
|
|
// this uses a proxy in all cases, the other will only use it if you don't have load balancer capabilities.
|
|
func newAnonymousImpersonationProxyClientWithProxy(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte, nestedImpersonationConfig *rest.ImpersonationConfig) *kubeclient.Client {
|
|
t.Helper()
|
|
env := testlib.IntegrationEnv(t)
|
|
|
|
emptyCredentials := &loginv1alpha1.ClusterCredential{}
|
|
kubeconfig := impersonationProxyRestConfig(emptyCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig)
|
|
|
|
kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy)
|
|
|
|
return testlib.NewKubeclient(t, kubeconfig)
|
|
}
|
|
|
|
func impersonationProxyViaSquidKubeClientWithoutCredential(t *testing.T, proxyServiceEndpoint string) kubernetes.Interface {
|
|
t.Helper()
|
|
|
|
env := testlib.IntegrationEnv(t)
|
|
proxyURL := "https://" + proxyServiceEndpoint
|
|
kubeconfig := impersonationProxyRestConfig(&loginv1alpha1.ClusterCredential{}, proxyURL, nil, nil)
|
|
kubeconfig.Proxy = kubeconfigProxyFunc(t, env.Proxy)
|
|
return testlib.NewKubeclient(t, kubeconfig).Kubernetes
|
|
}
|
|
|
|
func newImpersonationProxyClient(
|
|
t *testing.T,
|
|
impersonationProxyURL string,
|
|
impersonationProxyCACertPEM []byte,
|
|
nestedImpersonationConfig *rest.ImpersonationConfig,
|
|
refreshCredentialFunc func(t *testing.T, impersonationProxyURL string, impersonationProxyCACertPEM []byte) *loginv1alpha1.ClusterCredential,
|
|
) *kubeclient.Client {
|
|
t.Helper()
|
|
|
|
refreshedCredentials := refreshCredentialFunc(t, impersonationProxyURL, impersonationProxyCACertPEM).DeepCopy()
|
|
refreshedCredentials.Token = "not a valid token" // demonstrates that client certs take precedence over tokens by setting both on the requests
|
|
return newImpersonationProxyClientWithCredentials(t, refreshedCredentials, impersonationProxyURL, impersonationProxyCACertPEM, nestedImpersonationConfig)
|
|
}
|
|
|
|
// getCredForConfig is mostly just a hacky workaround for impersonationProxyRestConfig needing creds directly.
|
|
func getCredForConfig(t *testing.T, config *rest.Config) *loginv1alpha1.ClusterCredential {
|
|
t.Helper()
|
|
|
|
out := &loginv1alpha1.ClusterCredential{}
|
|
|
|
config = rest.CopyConfig(config)
|
|
|
|
config.Wrap(func(rt http.RoundTripper) http.RoundTripper {
|
|
return roundtripper.WrapFunc(rt, func(req *http.Request) (*http.Response, error) {
|
|
resp, err := rt.RoundTrip(req)
|
|
|
|
r := req
|
|
if resp != nil && resp.Request != nil {
|
|
r = resp.Request
|
|
}
|
|
|
|
_, _, _ = bearertoken.New(authenticator.TokenFunc(func(_ context.Context, token string) (*authenticator.Response, bool, error) {
|
|
out.Token = token
|
|
return nil, false, nil
|
|
})).AuthenticateRequest(r)
|
|
|
|
return resp, err
|
|
})
|
|
})
|
|
|
|
transportConfig, err := config.TransportConfig()
|
|
require.NoError(t, err)
|
|
|
|
rt, err := transport.New(transportConfig)
|
|
require.NoError(t, err)
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, "GET", "https://localhost", nil)
|
|
require.NoError(t, err)
|
|
resp, _ := rt.RoundTrip(req)
|
|
if resp != nil && resp.Body != nil {
|
|
_ = resp.Body.Close()
|
|
}
|
|
|
|
tlsConfig, err := transport.TLSConfigFor(transportConfig)
|
|
require.NoError(t, err)
|
|
|
|
if tlsConfig != nil && tlsConfig.GetClientCertificate != nil {
|
|
cert, err := tlsConfig.GetClientCertificate(nil)
|
|
require.NoError(t, err)
|
|
if len(cert.Certificate) > 0 {
|
|
require.Len(t, cert.Certificate, 1)
|
|
publicKey := pem.EncodeToMemory(&pem.Block{
|
|
Type: "CERTIFICATE",
|
|
Bytes: cert.Certificate[0],
|
|
})
|
|
out.ClientCertificateData = string(publicKey)
|
|
|
|
privateKey, err := keyutil.MarshalPrivateKeyToPEM(cert.PrivateKey)
|
|
require.NoError(t, err)
|
|
out.ClientKeyData = string(privateKey)
|
|
}
|
|
}
|
|
|
|
if *out == (loginv1alpha1.ClusterCredential{}) {
|
|
t.Fatal("failed to get creds for config")
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func getUIDAndExtraViaCSR(ctx context.Context, t *testing.T, uid string, client kubernetes.Interface) (string, map[string][]string) {
|
|
t.Helper()
|
|
|
|
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
|
require.NoError(t, err)
|
|
|
|
csrPEM, err := cert.MakeCSR(privateKey, &pkix.Name{
|
|
CommonName: "panda-man",
|
|
Organization: []string{"living-the-dream", "need-more-sleep"},
|
|
}, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
csrName, _, err := csr.RequestCertificate(
|
|
client,
|
|
csrPEM,
|
|
"",
|
|
certificatesv1.KubeAPIServerClientSignerName,
|
|
nil,
|
|
[]certificatesv1.KeyUsage{certificatesv1.UsageClientAuth},
|
|
privateKey,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
outUID := uid // in the future this may not be empty on some clusters
|
|
extrasAsStrings := map[string][]string{}
|
|
|
|
if testutil.KubeServerSupportsCertificatesV1API(t, client.Discovery()) {
|
|
csReq, err := client.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
err = client.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{})
|
|
require.NoError(t, err)
|
|
|
|
if len(outUID) == 0 {
|
|
outUID = csReq.Spec.UID
|
|
}
|
|
|
|
// Convert each `ExtraValue` to `[]string` to return, so we don't have to deal with v1beta1 types versus v1 types
|
|
for k, v := range csReq.Spec.Extra {
|
|
extrasAsStrings[k] = v
|
|
}
|
|
} else {
|
|
// On old Kubernetes clusters use CertificatesV1beta1
|
|
csReq, err := client.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{})
|
|
require.NoError(t, err)
|
|
|
|
err = client.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{})
|
|
require.NoError(t, err)
|
|
|
|
if len(outUID) == 0 {
|
|
outUID = csReq.Spec.UID
|
|
}
|
|
|
|
// Convert each `ExtraValue` to `[]string` to return, so we don't have to deal with v1beta1 types versus v1 types
|
|
for k, v := range csReq.Spec.Extra {
|
|
extrasAsStrings[k] = v
|
|
}
|
|
}
|
|
|
|
return outUID, extrasAsStrings
|
|
}
|
|
|
|
func parallelIfNotEKS(t *testing.T) {
|
|
if testlib.IntegrationEnv(t).KubernetesDistribution == testlib.EKSDistro {
|
|
return
|
|
}
|
|
|
|
t.Parallel()
|
|
}
|