ContainerImage.Pinniped/internal/concierge/impersonator/impersonator.go

// Copyright 2020-2022 the Pinniped contributors. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0

package impersonator

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"regexp"
	"strings"
	"sync"
	"time"

	authenticationv1 "k8s.io/api/authentication/v1"
	apiequality "k8s.io/apimachinery/pkg/api/equality"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/apimachinery/pkg/util/httpstream"
	utilnet "k8s.io/apimachinery/pkg/util/net"
	"k8s.io/apimachinery/pkg/util/sets"
	auditinternal "k8s.io/apiserver/pkg/apis/audit"
	"k8s.io/apiserver/pkg/audit"
	"k8s.io/apiserver/pkg/audit/policy"
	"k8s.io/apiserver/pkg/authentication/authenticator"
	"k8s.io/apiserver/pkg/authentication/request/bearertoken"
	"k8s.io/apiserver/pkg/authentication/user"
	"k8s.io/apiserver/pkg/authorization/authorizer"
	"k8s.io/apiserver/pkg/endpoints/filterlatency"
	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
	"k8s.io/apiserver/pkg/endpoints/request"
	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
	genericapiserver "k8s.io/apiserver/pkg/server"
	"k8s.io/apiserver/pkg/server/dynamiccertificates"
	"k8s.io/apiserver/pkg/server/filters"
	genericoptions "k8s.io/apiserver/pkg/server/options"
	auditfake "k8s.io/apiserver/plugin/pkg/audit/fake"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/transport"

	"go.pinniped.dev/internal/constable"
	"go.pinniped.dev/internal/crypto/ptls"
	"go.pinniped.dev/internal/dynamiccert"
	"go.pinniped.dev/internal/httputil/securityheader"
	"go.pinniped.dev/internal/kubeclient"
	"go.pinniped.dev/internal/plog"
	"go.pinniped.dev/internal/valuelesscontext"
)

// FactoryFunc is a function which can create an impersonator server.
// It returns a function which will start the impersonator server.
// That start function takes a stopCh which can be used to stop the server.
// Once a server has been stopped, don't start it again using the start function.
// Instead, call the factory function again to get a new start function.
type FactoryFunc func(
	port int,
	dynamicCertProvider dynamiccert.Private,
	impersonationProxySignerCA dynamiccert.Public,
) (func(stopCh <-chan struct{}) error, error)

func New(
	port int,
	dynamicCertProvider dynamiccert.Private,
	impersonationProxySignerCA dynamiccert.Public,
) (func(stopCh <-chan struct{}) error, error) {
	return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, kubeclient.Secure, nil, nil, nil)
}

func newInternal( //nolint:funlen // yeah, it's kind of long.
	port int,
	dynamicCertProvider dynamiccert.Private,
	impersonationProxySignerCA dynamiccert.Public,
	restConfigFunc ptls.RestConfigFunc, // for unit testing, should always be kubeclient.Secure in production
	clientOpts []kubeclient.Option, // for unit testing, should always be nil in production
	recOpts func(*genericoptions.RecommendedOptions), // for unit testing, should always be nil in production
	recConfig func(*genericapiserver.RecommendedConfig), // for unit testing, should always be nil in production
) (func(stopCh <-chan struct{}) error, error) {
	var listener net.Listener

	constructServer := func() (func(stopCh <-chan struct{}) error, error) {
		// Bare minimum server side scheme to allow for status messages to be encoded.
		scheme := runtime.NewScheme()
		metav1.AddToGroupVersion(scheme, metav1.Unversioned)
		codecs := serializer.NewCodecFactory(scheme)

		// This is unused for now but it is a safe value that we could use in the future.
		defaultEtcdPathPrefix := "/pinniped-impersonation-proxy-registry"

		recommendedOptions := genericoptions.NewRecommendedOptions(
			defaultEtcdPathPrefix,
			codecs.LegacyCodec(),
		)
		recommendedOptions.Etcd = nil                                                   // turn off etcd storage because we don't need it yet
		recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider // serving certs (end user facing)
		recommendedOptions.SecureServing.BindPort = port

		// secure TLS for connections coming from external clients and going to the Kube API server
		// this is best effort because not all options provide the right hooks to override TLS config
		// since any client could connect to the impersonation proxy, this uses the default TLS config
		if err := ptls.DefaultRecommendedOptions(recommendedOptions, restConfigFunc); err != nil {
			return nil, fmt.Errorf("failed to secure recommended options: %w", err)
		}

		// Wire up the impersonation proxy signer CA as another valid authenticator for client cert auth,
		// along with the Kube API server's CA.
		// Note: any changes to the Authentication stack need to be kept in sync with any assumptions made
		// by getTransportForUser, especially if we ever update the TCR API to start returning bearer tokens.
		kubeClientUnsafeForProxying, err := kubeclient.New(clientOpts...)
		if err != nil {
			return nil, err
		}
		kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
			"client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClientUnsafeForProxying.Kubernetes,
		)
		if err != nil {
			return nil, err
		}
		recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider(
			impersonationProxySignerCA, kubeClientCA,
		)

		if recOpts != nil {
			recOpts(recommendedOptions)
		}

		serverConfig := genericapiserver.NewRecommendedConfig(codecs)

		// Note that ApplyTo is going to create a network listener and bind to the requested port.
		// It puts this listener into serverConfig.SecureServing.Listener.
		err = recommendedOptions.ApplyTo(serverConfig)
		if serverConfig.SecureServing != nil {
			// Set the pointer from the outer function to allow the outer function to close the listener in case
			// this function returns an error for any reason anywhere below here.
			listener = serverConfig.SecureServing.Listener
		}
		if err != nil {
			return nil, err
		}

		// Loopback authentication to this server does not really make sense since we just proxy everything to
		// the Kube API server, thus we replace loopback connection config with one that does direct connections
		// the Kube API server. Loopback config is mainly used by post start hooks, so this is mostly future proofing.
		serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClientUnsafeForProxying.ProtoConfig) // assume proto is safe (hooks can override)
		// Remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken.
		// See sanity checks at the end of this function.
		serverConfig.LoopbackClientConfig.BearerToken = ""

		// match KAS exactly since our long running operations are just a proxy to it
		// this must be kept in sync with github.com/kubernetes/kubernetes/cmd/kube-apiserver/app/server.go
		// this is nothing to stress about - it has not changed since the beginning of Kube:
		// v1.6 no-op move away from regex to request info https://github.com/kubernetes/kubernetes/pull/38119
		// v1.1 added pods/attach to the list https://github.com/kubernetes/kubernetes/pull/13705
		serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
			sets.NewString("watch", "proxy"),
			sets.NewString("attach", "exec", "proxy", "log", "portforward"),
		)

		// use the custom impersonation proxy service account credentials when reverse proxying to the API server
		kubeClientForProxy, err := getReverseProxyClient(clientOpts)
		if err != nil {
			return nil, fmt.Errorf("failed to build reverse proxy client: %w", err)
		}

		// Assume proto config is safe because transport level configs do not use rest.ContentConfig.
		// Thus if we are interacting with actual APIs, they should be using pre-built clients.
		impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClientForProxy.ProtoConfig))
		if err != nil {
			return nil, err
		}

		defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc
		serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler {
			// We ignore the passed in handler because we never have any REST APIs to delegate to.
			// This means we are ignoring the admission, discovery, REST storage, etc layers.
			doNotDelegate := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})

			// Impersonation proxy business logic with timing information.
			impersonationProxyCompleted := filterlatency.TrackCompleted(doNotDelegate)
			impersonationProxy := impersonationProxyFunc(c)
			handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				defer impersonationProxyCompleted.ServeHTTP(w, r)
				impersonationProxy.ServeHTTP(w, r)
			}))
			handler = filterlatency.TrackStarted(handler, "impersonationproxy")

			// The standard Kube handler chain (authn, authz, impersonation, audit, etc).
			// See the genericapiserver.DefaultBuildHandlerChain func for details.
			handler = defaultBuildHandlerChainFunc(handler, c)

			// we need to grab the bearer token before WithAuthentication deletes it.
			handler = filterlatency.TrackCompleted(handler)
			handler = withBearerTokenPreservation(handler)
			handler = filterlatency.TrackStarted(handler, "bearertokenpreservation")

			// Always set security headers so browsers do the right thing.
			handler = filterlatency.TrackCompleted(handler)
			handler = securityheader.Wrap(handler)
			handler = filterlatency.TrackStarted(handler, "securityheaders")

			return handler
		}

		// wire up a fake audit backend at the metadata level so we can preserve the original user during nested impersonation
		serverConfig.AuditPolicyRuleEvaluator = policy.NewFakePolicyRuleEvaluator(auditinternal.LevelMetadata, nil)
		serverConfig.AuditBackend = &auditfake.Backend{}

		// Probe the API server to figure out if anonymous auth is enabled.
		anonymousAuthEnabled, err := isAnonymousAuthEnabled(kubeClientUnsafeForProxying.JSONConfig)
		if err != nil {
			return nil, fmt.Errorf("could not detect if anonymous authentication is enabled: %w", err)
		}
		plog.Debug("anonymous authentication probed", "anonymousAuthEnabled", anonymousAuthEnabled)

		// if we ever start unioning a TCR bearer token authenticator with serverConfig.Authenticator
		// then we will need to update the related assumption in tokenPassthroughRoundTripper

		delegatingAuthenticator := serverConfig.Authentication.Authenticator
		blockAnonymousAuthenticator := &comparableAuthenticator{
			RequestFunc: func(req *http.Request) (*authenticator.Response, bool, error) {
				resp, ok, err := delegatingAuthenticator.AuthenticateRequest(req)

				// anonymous auth is enabled so no further check is necessary
				if anonymousAuthEnabled {
					return resp, ok, err
				}

				// authentication failed
				if err != nil || !ok {
					return resp, ok, err
				}

				// any other user than anonymous is irrelevant
				if resp.User.GetName() != user.Anonymous {
					return resp, ok, err
				}

				reqInfo, ok := genericapirequest.RequestInfoFrom(req.Context())
				if !ok {
					return nil, false, constable.Error("no RequestInfo found in the context")
				}

				// a TKR is a resource, any request that is not for a resource should not be authenticated
				if !reqInfo.IsResourceRequest {
					return nil, false, nil
				}

				// any resource besides TKR should not be authenticated
				if !isTokenCredReq(reqInfo) {
					return nil, false, nil
				}

				// anonymous authentication is disabled, but we must let an anonymous request
				// to TKR authenticate as this is the only method to retrieve credentials
				return resp, ok, err
			},
		}
		// Set our custom authenticator before calling Compete(), which will use it.
		serverConfig.Authentication.Authenticator = blockAnonymousAuthenticator

		delegatingAuthorizer := serverConfig.Authorization.Authorizer
		customReasonAuthorizer := &comparableAuthorizer{
			AuthorizerFunc: func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
				const baseReason = "decision made by impersonation-proxy.concierge.pinniped.dev"
				switch a.GetVerb() {
				case "":
					// Empty string is disallowed because request info has had bugs in the past where it would leave it empty.
					return authorizer.DecisionDeny, "invalid verb, " + baseReason, nil
				default:
					// Since we authenticate the requesting user, we are in the best position to correctly authorize them.
					// When KAS does the check, it may run the check against our service account and not the requesting user
					// (due to a bug in the code or any other internal SAR checks that the request processing does).
					// This also handles the impersonate verb to allow for nested impersonation.
					decision, reason, err := delegatingAuthorizer.Authorize(ctx, a)

					// make it easier to detect when the impersonation proxy is authorizing a request vs KAS
					switch len(reason) {
					case 0:
						reason = baseReason
					default:
						reason = reason + ", " + baseReason
					}

					return decision, reason, err
				}
			},
		}
		// Set our custom authorizer before calling Compete(), which will use it.
		serverConfig.Authorization.Authorizer = customReasonAuthorizer

		if recConfig != nil {
			recConfig(serverConfig)
		}

		completedConfig := serverConfig.Complete()
		impersonationProxyServer, err := completedConfig.New("impersonation-proxy", genericapiserver.NewEmptyDelegate())
		if err != nil {
			return nil, err
		}

		preparedRun := impersonationProxyServer.PrepareRun()

		// Sanity check. Make sure that our custom authenticator is still in place and did not get changed or wrapped.
		if completedConfig.Authentication.Authenticator != blockAnonymousAuthenticator {
			return nil, fmt.Errorf("invalid mutation of anonymous authenticator detected: %#v", completedConfig.Authentication.Authenticator)
		}

		// Sanity check. Make sure that our custom authorizer is still in place and did not get changed or wrapped.
		if preparedRun.Authorizer != customReasonAuthorizer {
			return nil, fmt.Errorf("invalid mutation of impersonation authorizer detected: %#v", preparedRun.Authorizer)
		}

		// Sanity check. Assert that we have a functioning token file to use and no bearer token.
		if len(preparedRun.LoopbackClientConfig.BearerToken) != 0 || len(preparedRun.LoopbackClientConfig.BearerTokenFile) == 0 {
			return nil, constable.Error("invalid impersonator loopback rest config has wrong bearer token semantics")
		}

		return preparedRun.Run, nil
	}

	result, err := constructServer()
	// If there was any error during construction, then we would like to close the listener to free up the port.
	if err != nil {
		errs := []error{err}
		if listener != nil {
			errs = append(errs, listener.Close())
		}
		return nil, errors.NewAggregate(errs)
	}
	return result, nil
}

func getReverseProxyClient(clientOpts []kubeclient.Option) (*kubeclient.Client, error) {
	// just use the overrides given during unit tests
	if len(clientOpts) != 0 {
		return kubeclient.New(clientOpts...)
	}

	// this is the magic path where the impersonation proxy SA token is mounted
	const tokenFile = "/var/run/secrets/impersonation-proxy.concierge.pinniped.dev/serviceaccount/token" //nolint:gosec // this is not a credential

	// make sure the token file we need exists before trying to use it
	if _, err := os.Stat(tokenFile); err != nil {
		return nil, err
	}

	// build an in cluster config that uses the impersonation proxy token file
	impersonationProxyRestConfig, err := rest.InClusterConfig()
	if err != nil {
		return nil, err
	}
	impersonationProxyRestConfig = kubeclient.SecureAnonymousClientConfig(impersonationProxyRestConfig)
	impersonationProxyRestConfig.BearerTokenFile = tokenFile

	return kubeclient.New(kubeclient.WithConfig(impersonationProxyRestConfig))
}

func isAnonymousAuthEnabled(config *rest.Config) (bool, error) {
	anonymousConfig := kubeclient.SecureAnonymousClientConfig(config)

	// we do not need either of these but RESTClientFor complains if they are not set
	anonymousConfig.GroupVersion = &schema.GroupVersion{}
	anonymousConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()

	// in case anyone looking at audit logs wants to know who is making the anonymous request
	anonymousConfig.UserAgent = rest.DefaultKubernetesUserAgent()

	rc, err := rest.RESTClientFor(anonymousConfig)
	if err != nil {
		return false, err
	}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	defer cancel()

	_, errHealthz := rc.Get().AbsPath("/healthz").DoRaw(ctx)

	switch {
	// 200 ok on healthz clearly indicates authentication success
	case errHealthz == nil:
		return true, nil

	// we are authenticated but not authorized. anonymous authentication is enabled
	case apierrors.IsForbidden(errHealthz):
		return true, nil

	// failure to authenticate will return unauthorized (http misnomer)
	case apierrors.IsUnauthorized(errHealthz):
		return false, nil

	// any other error is unexpected
	default:
		return false, errHealthz
	}
}

func isTokenCredReq(reqInfo *genericapirequest.RequestInfo) bool {
	if reqInfo.Resource != "tokencredentialrequests" {
		return false
	}

	// pinniped components allow for the group suffix to be customized
	// rather than wiring in the current configured suffix, checking the prefix is sufficient
	if !strings.HasPrefix(reqInfo.APIGroup, "login.concierge.") {
		return false
	}

	return true
}

// No-op wrapping around RequestFunc to allow for comparisons.
type comparableAuthenticator struct {
	authenticator.RequestFunc
}

// No-op wrapping around AuthorizerFunc to allow for comparisons.
type comparableAuthorizer struct {
	authorizer.AuthorizerFunc
}

func withBearerTokenPreservation(delegate http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// this looks a bit hacky but lets us avoid writing any logic for parsing out the bearer token
		var reqToken string
		_, _, _ = bearertoken.New(authenticator.TokenFunc(func(_ context.Context, token string) (*authenticator.Response, bool, error) {
			reqToken = token
			return nil, false, nil
		})).AuthenticateRequest(r)

		// smuggle the token through the context.  this does mean that we need to avoid logging the context.
		if len(reqToken) != 0 {
			ctx := context.WithValue(r.Context(), tokenKey, reqToken)
			r = r.WithContext(ctx)
		}

		delegate.ServeHTTP(w, r)
	})
}

func tokenFrom(ctx context.Context) string {
	token, _ := ctx.Value(tokenKey).(string)
	return token
}

// contextKey type is unexported to prevent collisions.
type contextKey int

const tokenKey contextKey = iota

func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapiserver.Config) http.Handler, error) {
	serverURL, err := url.Parse(restConfig.Host)
	if err != nil {
		return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err)
	}

	http1RoundTripper, err := getTransportForProtocol(restConfig, "http/1.1")
	if err != nil {
		return nil, fmt.Errorf("could not get http/1.1 round tripper: %w", err)
	}
	http1RoundTripperAnonymous, err := getTransportForProtocol(kubeclient.SecureAnonymousClientConfig(restConfig), "http/1.1")
	if err != nil {
		return nil, fmt.Errorf("could not get http/1.1 anonymous round tripper: %w", err)
	}

	http2RoundTripper, err := getTransportForProtocol(restConfig, "h2")
	if err != nil {
		return nil, fmt.Errorf("could not get http/2.0 round tripper: %w", err)
	}
	http2RoundTripperAnonymous, err := getTransportForProtocol(kubeclient.SecureAnonymousClientConfig(restConfig), "h2")
	if err != nil {
		return nil, fmt.Errorf("could not get http/2.0 anonymous round tripper: %w", err)
	}

	return func(c *genericapiserver.Config) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			if len(r.Header.Values("Authorization")) != 0 {
				plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so",
					"url", r.URL.String(),
					"method", r.Method,
				)
				newInternalErrResponse(w, r, c.Serializer, "invalid authorization header")
				return
			}

			if err := ensureNoImpersonationHeaders(r); err != nil {
				plog.Error("unknown impersonation header seen",
					err,
					"url", r.URL.String(),
					"method", r.Method,
				)
				newInternalErrResponse(w, r, c.Serializer, "invalid impersonation")
				return
			}

			userInfo, ok := request.UserFrom(r.Context())
			if !ok {
				plog.Warning("aggregated API server logic did not set user info but it is always supposed to do so",
					"url", r.URL.String(),
					"method", r.Method,
				)
				newInternalErrResponse(w, r, c.Serializer, "invalid user")
				return
			}

			ae := audit.AuditEventFrom(r.Context())
			if ae == nil {
				plog.Warning("aggregated API server logic did not set audit event but it is always supposed to do so",
					"url", r.URL.String(),
					"method", r.Method,
				)
				newInternalErrResponse(w, r, c.Serializer, "invalid audit event")
				return
			}

			// grab the request's bearer token if present.  this is optional and does not fail the request if missing.
			token := tokenFrom(r.Context())

			// KAS only supports upgrades via http/1.1 to websockets/SPDY (upgrades never use http/2.0)
			// Thus we default to using http/2.0 when the request is not an upgrade, otherwise we use http/1.1
			baseRT, baseRTAnonymous := http2RoundTripper, http2RoundTripperAnonymous
			isUpgradeRequest := httpstream.IsUpgradeRequest(r)
			if isUpgradeRequest {
				baseRT, baseRTAnonymous = http1RoundTripper, http1RoundTripperAnonymous
			}

			rt, err := getTransportForUser(r.Context(), userInfo, baseRT, baseRTAnonymous, ae, token, c.Authentication.Authenticator)
			if err != nil {
				plog.WarningErr("rejecting request as we cannot act as the current user", err,
					"url", r.URL.String(),
					"method", r.Method,
					"isUpgradeRequest", isUpgradeRequest,
				)
				newInternalErrResponse(w, r, c.Serializer, "unimplemented functionality - unable to act as current user")
				return
			}

			plog.Debug("impersonation proxy servicing request",
				"url", r.URL.String(),
				"method", r.Method,
				"isUpgradeRequest", isUpgradeRequest,
			)
			plog.Trace("impersonation proxy servicing request was for user",
				"url", r.URL.String(),
				"method", r.Method,
				"isUpgradeRequest", isUpgradeRequest,
				"username", userInfo.GetName(), // this info leak seems fine for trace level logs
			)

			// The proxy library used below will panic when the client disconnects abruptly, so in order to
			// assure that this log message is always printed at the end of this func, it must be deferred.
			defer plog.Debug("impersonation proxy finished servicing request",
				"url", r.URL.String(),
				"method", r.Method,
				"isUpgradeRequest", isUpgradeRequest,
			)

			// do not allow the client to cause log confusion by spoofing this header
			if len(r.Header.Values("X-Forwarded-For")) > 0 {
				r = utilnet.CloneRequest(r)
				r.Header.Del("X-Forwarded-For")
			}

			// the http2 code seems to call Close concurrently which can lead to data races
			if r.Body != nil {
				r = utilnet.CloneRequest(r)
				r.Body = &safeReadWriteCloser{rc: r.Body}
			}

			reverseProxy := httputil.NewSingleHostReverseProxy(serverURL)
			reverseProxy.Transport = rt
			reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line
			reverseProxy.ServeHTTP(w, r)
		})
	}, nil
}

var _ io.ReadWriteCloser = &safeReadWriteCloser{}

type safeReadWriteCloser struct {
	m sync.Mutex // all methods allowed concurrently, this only guards concurrent calls to Close

	rc io.ReadCloser

	once     sync.Once // set up rwc and writeErr
	rwc      io.ReadWriteCloser
	writeErr error
}

func (r *safeReadWriteCloser) Read(p []byte) (int, error) {
	return r.rc.Read(p)
}

func (r *safeReadWriteCloser) Write(p []byte) (int, error) {
	r.once.Do(func() {
		var ok bool
		r.rwc, ok = r.rc.(io.ReadWriteCloser)
		if !ok { // this method should only be caused during flows that switch protocols
			r.writeErr = fmt.Errorf("switching protocols failed: io.ReadCloser %T is not io.ReadWriteCloser", r.rc)
		}
	})
	if r.writeErr != nil {
		return 0, r.writeErr
	}
	return r.rwc.Write(p)
}

func (r *safeReadWriteCloser) Close() error {
	r.m.Lock()
	defer r.m.Unlock()

	return r.rc.Close()
}

func ensureNoImpersonationHeaders(r *http.Request) error {
	for key := range r.Header {
		// even though we have unit tests that try to cover this case, it is hard to tell if Go does
		// client side canonicalization on encode, server side canonicalization on decode, or both
		key := http.CanonicalHeaderKey(key)
		if strings.HasPrefix(key, "Impersonate") {
			return fmt.Errorf("%q header already exists", key)
		}
	}

	return nil
}

func getTransportForUser(ctx context.Context, userInfo user.Info, delegate, delegateAnonymous http.RoundTripper, ae *auditinternal.Event, token string, authenticator authenticator.Request) (http.RoundTripper, error) {
	if canImpersonateFully(userInfo) {
		return standardImpersonationRoundTripper(userInfo, ae, delegate)
	}

	return tokenPassthroughRoundTripper(ctx, delegateAnonymous, ae, token, authenticator)
}

func canImpersonateFully(userInfo user.Info) bool {
	// nolint: gosimple  // this structure is on purpose because we plan to expand this function
	if len(userInfo.GetUID()) == 0 {
		return true
	}

	// once kube supports UID impersonation, add logic to detect if the KAS is
	// new enough to have this functionality and return true in that case as well
	return false
}

func standardImpersonationRoundTripper(userInfo user.Info, ae *auditinternal.Event, delegate http.RoundTripper) (http.RoundTripper, error) {
	extra, err := buildExtra(userInfo.GetExtra(), ae)
	if err != nil {
		return nil, err
	}

	impersonateConfig := transport.ImpersonationConfig{
		UserName: userInfo.GetName(),
		Groups:   userInfo.GetGroups(),
		Extra:    extra,
	}
	// transport.NewImpersonatingRoundTripper clones the request before setting headers
	// thus it will not accidentally mutate the input request (see http.Handler docs)
	return transport.NewImpersonatingRoundTripper(impersonateConfig, delegate), nil
}

func tokenPassthroughRoundTripper(ctx context.Context, delegateAnonymous http.RoundTripper, ae *auditinternal.Event, token string, authenticator authenticator.Request) (http.RoundTripper, error) {
	// all code below assumes KAS does not support UID impersonation because that case is handled in the standard path

	// it also assumes that the TCR API does not issue tokens - if this assumption changes, we will need
	// some way to distinguish a token that is only valid against this impersonation proxy and not against KAS.
	// this code will fail closed because said TCR token would not work against KAS and the request would fail.

	// if we get here we know the final user info had a UID
	// if the original user is also performing a nested impersonation, it means that said nested
	// impersonation is trying to impersonate a UID since final user info == ae.ImpersonatedUser
	// we know this KAS does not support UID impersonation so this request must be rejected
	if ae.ImpersonatedUser != nil {
		return nil, constable.Error("unable to impersonate uid")
	}

	// see what KAS thinks this token translates into
	// this is important because certs have precedence over tokens and we want
	// to make sure that we do not get confused and pass along the wrong token
	tokenUser, err := tokenReview(ctx, token, authenticator)
	if err != nil {
		return nil, err
	}

	// we want to compare the result of the token authentication with the original user that made the request
	// if the user who made the request and the token do not match, we cannot go any further at this point
	if !apiequality.Semantic.DeepEqual(ae.User, tokenUser) {
		// this info leak seems fine for trace level logs
		plog.Trace("failed to passthrough token due to user mismatch",
			"original-username", ae.User.Username,
			"original-uid", ae.User.UID,
			"token-username", tokenUser.Username,
			"token-uid", tokenUser.UID,
		)
		return nil, constable.Error("token authenticated as a different user")
	}

	// now we know that if we send this token to KAS, it will authenticate correctly
	return transport.NewBearerAuthRoundTripper(token, delegateAnonymous), nil
}

func tokenReview(ctx context.Context, token string, authenticator authenticator.Request) (authenticationv1.UserInfo, error) {
	if len(token) == 0 {
		return authenticationv1.UserInfo{}, constable.Error("no token on request")
	}

	// create a header that contains nothing but the token
	// an astute observer may ask "but what about the token's audience?"
	// in this case, we want to leave audiences unset per the token review docs:
	// > If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.
	// i.e. we want to make sure that the given token is valid against KAS
	fakeReq := &http.Request{Header: http.Header{}}
	fakeReq.Header.Set("Authorization", "Bearer "+token)

	// propagate cancellation of parent context (without any values such as audience)
	fakeReq = fakeReq.WithContext(valuelesscontext.New(ctx))

	// this will almost always be a free call that hits our 10 second cache TTL
	resp, ok, err := authenticator.AuthenticateRequest(fakeReq)
	if err != nil {
		return authenticationv1.UserInfo{}, err
	}
	if !ok {
		return authenticationv1.UserInfo{}, constable.Error("token failed to authenticate")
	}

	tokenUser := authenticationv1.UserInfo{
		Username: resp.User.GetName(),
		UID:      resp.User.GetUID(),
		Groups:   resp.User.GetGroups(),
		Extra:    make(map[string]authenticationv1.ExtraValue, len(resp.User.GetExtra())),
	}
	for k, v := range resp.User.GetExtra() {
		tokenUser.Extra[k] = v
	}

	return tokenUser, nil
}

func buildExtra(extra map[string][]string, ae *auditinternal.Event) (map[string][]string, error) {
	const reservedImpersonationProxySuffix = ".impersonation-proxy.concierge.pinniped.dev"

	// always validate that the extra is something we support irregardless of nested impersonation
	for k := range extra {
		if !extraKeyRegexp.MatchString(k) {
			return nil, fmt.Errorf("disallowed extra key seen: %s", k)
		}

		if strings.HasSuffix(k, reservedImpersonationProxySuffix) {
			return nil, fmt.Errorf("disallowed extra key with reserved prefix seen: %s", k)
		}
	}

	if ae.ImpersonatedUser == nil {
		return extra, nil // just return the given extra since nested impersonation is not being used
	}

	// avoid mutating input map, preallocate new map to store original user info
	out := make(map[string][]string, len(extra)+1)

	for k, v := range extra {
		out[k] = v // shallow copy of slice since we are not going to mutate it
	}

	origUserInfoJSON, err := json.Marshal(ae.User)
	if err != nil {
		return nil, err
	}

	out["original-user-info"+reservedImpersonationProxySuffix] = []string{string(origUserInfoJSON)}

	return out, nil
}

// extraKeyRegexp is a very conservative regex to handle impersonation's extra key fidelity limitations such as casing and escaping.
var extraKeyRegexp = regexp.MustCompile(`^[a-z0-9/\-._]+$`)

func newInternalErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, msg string) {
	newStatusErrResponse(w, r, s, apierrors.NewInternalError(constable.Error(msg)))
}

func newStatusErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, err *apierrors.StatusError) {
	requestInfo, ok := genericapirequest.RequestInfoFrom(r.Context())
	if !ok {
		responsewriters.InternalError(w, r, constable.Error("no RequestInfo found in the context"))
		return
	}

	gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}
	responsewriters.ErrorNegotiated(err, s, gv, w, r)
}

func getTransportForProtocol(restConfig *rest.Config, protocol string) (http.RoundTripper, error) {
	transportConfig, err := restConfig.TransportConfig()
	if err != nil {
		return nil, fmt.Errorf("could not get in-cluster transport config: %w", err)
	}
	transportConfig.TLS.NextProtos = []string{protocol}

	rt, err := transport.New(transportConfig)
	if err != nil {
		return nil, fmt.Errorf("could not build transport: %w", err)
	}

	// For clients that support http2, transport.New calls http2.ConfigureTransports,
	// which configures with both h2 and http/1.1,
	// even when you explicitly only ask for h2.
	// Override that change.
	cfg, err := utilnet.TLSClientConfig(rt)
	if err != nil {
		return nil, fmt.Errorf("could not extract TLS config: %w", err)
	}
	cfg.NextProtos = []string{protocol}
	if err := kubeclient.AssertSecureTransport(rt); err != nil {
		return nil, err // make sure we only use a secure TLS config
	}

	return rt, nil
}