ContainerImage.Pinniped/internal/concierge/impersonator/impersonator.go

830 lines
32 KiB
Go
Raw Normal View History

// Copyright 2020-2022 the Pinniped contributors. All Rights Reserved.
// SPDX-License-Identifier: Apache-2.0
package impersonator
import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"regexp"
"strings"
"sync"
"time"
authenticationv1 "k8s.io/api/authentication/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured/unstructuredscheme"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apimachinery/pkg/util/httpstream"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apimachinery/pkg/util/sets"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
Update all deps to latest where possible, bump Kube deps to v0.23.1 Highlights from this dep bump: 1. Made a copy of the v0.4.0 github.com/go-logr/stdr implementation for use in tests. We must bump this dep as Kube code uses a newer version now. We would have to rewrite hundreds of test log assertions without this copy. 2. Use github.com/felixge/httpsnoop to undo the changes made by ory/fosite#636 for CLI based login flows. This is required for backwards compatibility with older versions of our CLI. A separate change after this will update the CLI to be more flexible (it is purposefully not part of this change to confirm that we did not break anything). For all browser login flows, we now redirect using http.StatusSeeOther instead of http.StatusFound. 3. Drop plog.RemoveKlogGlobalFlags as klog no longer mutates global process flags 4. Only bump github.com/ory/x to v0.0.297 instead of the latest v0.0.321 because v0.0.298+ pulls in a newer version of go.opentelemetry.io/otel/semconv which breaks k8s.io/apiserver. We should update k8s.io/apiserver to use the newer code. 5. Migrate all code from k8s.io/apimachinery/pkg/util/clock to k8s.io/utils/clock and k8s.io/utils/clock/testing 6. Delete testutil.NewDeleteOptionsRecorder and migrate to the new kubetesting.NewDeleteActionWithOptions 7. Updated ExpectedAuthorizeCodeSessionJSONFromFuzzing caused by fosite's new rotated_secrets OAuth client field. This new field is currently not relevant to us as we have no private clients. Signed-off-by: Monis Khan <mok@vmware.com>
2021-12-10 22:22:36 +00:00
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/request/bearertoken"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/filterlatency"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/apiserver/pkg/endpoints/request"
genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/apiserver/pkg/server/filters"
genericoptions "k8s.io/apiserver/pkg/server/options"
auditfake "k8s.io/apiserver/plugin/pkg/audit/fake"
"k8s.io/client-go/rest"
"k8s.io/client-go/transport"
"go.pinniped.dev/internal/constable"
"go.pinniped.dev/internal/crypto/ptls"
"go.pinniped.dev/internal/dynamiccert"
"go.pinniped.dev/internal/httputil/securityheader"
"go.pinniped.dev/internal/kubeclient"
"go.pinniped.dev/internal/plog"
"go.pinniped.dev/internal/valuelesscontext"
)
// FactoryFunc is a function which can create an impersonator server.
// It returns a function which will start the impersonator server.
// That start function takes a stopCh which can be used to stop the server.
// Once a server has been stopped, don't start it again using the start function.
// Instead, call the factory function again to get a new start function.
type FactoryFunc func(
port int,
dynamicCertProvider dynamiccert.Private,
impersonationProxySignerCA dynamiccert.Public,
) (func(stopCh <-chan struct{}) error, error)
func New(
port int,
dynamicCertProvider dynamiccert.Private,
impersonationProxySignerCA dynamiccert.Public,
) (func(stopCh <-chan struct{}) error, error) {
return newInternal(port, dynamicCertProvider, impersonationProxySignerCA, kubeclient.Secure, nil, nil, nil)
}
func newInternal( //nolint:funlen // yeah, it's kind of long.
port int,
dynamicCertProvider dynamiccert.Private,
impersonationProxySignerCA dynamiccert.Public,
restConfigFunc ptls.RestConfigFunc, // for unit testing, should always be kubeclient.Secure in production
clientOpts []kubeclient.Option, // for unit testing, should always be nil in production
recOpts func(*genericoptions.RecommendedOptions), // for unit testing, should always be nil in production
recConfig func(*genericapiserver.RecommendedConfig), // for unit testing, should always be nil in production
) (func(stopCh <-chan struct{}) error, error) {
var listener net.Listener
constructServer := func() (func(stopCh <-chan struct{}) error, error) {
// Bare minimum server side scheme to allow for status messages to be encoded.
scheme := runtime.NewScheme()
metav1.AddToGroupVersion(scheme, metav1.Unversioned)
codecs := serializer.NewCodecFactory(scheme)
// This is unused for now but it is a safe value that we could use in the future.
defaultEtcdPathPrefix := "/pinniped-impersonation-proxy-registry"
recommendedOptions := genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
codecs.LegacyCodec(),
)
recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet
recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider // serving certs (end user facing)
recommendedOptions.SecureServing.BindPort = port
// secure TLS for connections coming from external clients and going to the Kube API server
// this is best effort because not all options provide the right hooks to override TLS config
// since any client could connect to the impersonation proxy, this uses the default TLS config
if err := ptls.DefaultRecommendedOptions(recommendedOptions, restConfigFunc); err != nil {
return nil, fmt.Errorf("failed to secure recommended options: %w", err)
}
// Wire up the impersonation proxy signer CA as another valid authenticator for client cert auth,
// along with the Kube API server's CA.
// Note: any changes to the Authentication stack need to be kept in sync with any assumptions made
// by getTransportForUser, especially if we ever update the TCR API to start returning bearer tokens.
kubeClientUnsafeForProxying, err := kubeclient.New(clientOpts...)
if err != nil {
return nil, err
}
kubeClientCA, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
"client-ca", metav1.NamespaceSystem, "extension-apiserver-authentication", "client-ca-file", kubeClientUnsafeForProxying.Kubernetes,
)
if err != nil {
return nil, err
}
recommendedOptions.Authentication.ClientCert.CAContentProvider = dynamiccertificates.NewUnionCAContentProvider(
impersonationProxySignerCA, kubeClientCA,
)
if recOpts != nil {
recOpts(recommendedOptions)
}
serverConfig := genericapiserver.NewRecommendedConfig(codecs)
// Note that ApplyTo is going to create a network listener and bind to the requested port.
// It puts this listener into serverConfig.SecureServing.Listener.
err = recommendedOptions.ApplyTo(serverConfig)
if serverConfig.SecureServing != nil {
// Set the pointer from the outer function to allow the outer function to close the listener in case
// this function returns an error for any reason anywhere below here.
listener = serverConfig.SecureServing.Listener
}
if err != nil {
return nil, err
}
// Loopback authentication to this server does not really make sense since we just proxy everything to
// the Kube API server, thus we replace loopback connection config with one that does direct connections
// the Kube API server. Loopback config is mainly used by post start hooks, so this is mostly future proofing.
serverConfig.LoopbackClientConfig = rest.CopyConfig(kubeClientUnsafeForProxying.ProtoConfig) // assume proto is safe (hooks can override)
// Remove the bearer token so our authorizer does not get stomped on by AuthorizeClientBearerToken.
// See sanity checks at the end of this function.
serverConfig.LoopbackClientConfig.BearerToken = ""
// match KAS exactly since our long running operations are just a proxy to it
// this must be kept in sync with github.com/kubernetes/kubernetes/cmd/kube-apiserver/app/server.go
// this is nothing to stress about - it has not changed since the beginning of Kube:
// v1.6 no-op move away from regex to request info https://github.com/kubernetes/kubernetes/pull/38119
// v1.1 added pods/attach to the list https://github.com/kubernetes/kubernetes/pull/13705
serverConfig.LongRunningFunc = filters.BasicLongRunningRequestCheck(
sets.NewString("watch", "proxy"),
sets.NewString("attach", "exec", "proxy", "log", "portforward"),
)
// use the custom impersonation proxy service account credentials when reverse proxying to the API server
kubeClientForProxy, err := getReverseProxyClient(clientOpts)
if err != nil {
return nil, fmt.Errorf("failed to build reverse proxy client: %w", err)
}
// Assume proto config is safe because transport level configs do not use rest.ContentConfig.
// Thus if we are interacting with actual APIs, they should be using pre-built clients.
impersonationProxyFunc, err := newImpersonationReverseProxyFunc(rest.CopyConfig(kubeClientForProxy.ProtoConfig))
if err != nil {
return nil, err
}
defaultBuildHandlerChainFunc := serverConfig.BuildHandlerChainFunc
serverConfig.BuildHandlerChainFunc = func(_ http.Handler, c *genericapiserver.Config) http.Handler {
// We ignore the passed in handler because we never have any REST APIs to delegate to.
// This means we are ignoring the admission, discovery, REST storage, etc layers.
doNotDelegate := http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})
// Impersonation proxy business logic with timing information.
impersonationProxyCompleted := filterlatency.TrackCompleted(doNotDelegate)
impersonationProxy := impersonationProxyFunc(c)
handler := http.Handler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer impersonationProxyCompleted.ServeHTTP(w, r)
impersonationProxy.ServeHTTP(w, r)
}))
handler = filterlatency.TrackStarted(handler, "impersonationproxy")
// The standard Kube handler chain (authn, authz, impersonation, audit, etc).
// See the genericapiserver.DefaultBuildHandlerChain func for details.
handler = defaultBuildHandlerChainFunc(handler, c)
// we need to grab the bearer token before WithAuthentication deletes it.
handler = filterlatency.TrackCompleted(handler)
handler = withBearerTokenPreservation(handler)
handler = filterlatency.TrackStarted(handler, "bearertokenpreservation")
// Always set security headers so browsers do the right thing.
handler = filterlatency.TrackCompleted(handler)
handler = securityheader.Wrap(handler)
handler = filterlatency.TrackStarted(handler, "securityheaders")
return handler
}
// wire up a fake audit backend at the metadata level so we can preserve the original user during nested impersonation
Update all deps to latest where possible, bump Kube deps to v0.23.1 Highlights from this dep bump: 1. Made a copy of the v0.4.0 github.com/go-logr/stdr implementation for use in tests. We must bump this dep as Kube code uses a newer version now. We would have to rewrite hundreds of test log assertions without this copy. 2. Use github.com/felixge/httpsnoop to undo the changes made by ory/fosite#636 for CLI based login flows. This is required for backwards compatibility with older versions of our CLI. A separate change after this will update the CLI to be more flexible (it is purposefully not part of this change to confirm that we did not break anything). For all browser login flows, we now redirect using http.StatusSeeOther instead of http.StatusFound. 3. Drop plog.RemoveKlogGlobalFlags as klog no longer mutates global process flags 4. Only bump github.com/ory/x to v0.0.297 instead of the latest v0.0.321 because v0.0.298+ pulls in a newer version of go.opentelemetry.io/otel/semconv which breaks k8s.io/apiserver. We should update k8s.io/apiserver to use the newer code. 5. Migrate all code from k8s.io/apimachinery/pkg/util/clock to k8s.io/utils/clock and k8s.io/utils/clock/testing 6. Delete testutil.NewDeleteOptionsRecorder and migrate to the new kubetesting.NewDeleteActionWithOptions 7. Updated ExpectedAuthorizeCodeSessionJSONFromFuzzing caused by fosite's new rotated_secrets OAuth client field. This new field is currently not relevant to us as we have no private clients. Signed-off-by: Monis Khan <mok@vmware.com>
2021-12-10 22:22:36 +00:00
serverConfig.AuditPolicyRuleEvaluator = policy.NewFakePolicyRuleEvaluator(auditinternal.LevelMetadata, nil)
serverConfig.AuditBackend = &auditfake.Backend{}
// Probe the API server to figure out if anonymous auth is enabled.
anonymousAuthEnabled, err := isAnonymousAuthEnabled(kubeClientUnsafeForProxying.JSONConfig)
if err != nil {
return nil, fmt.Errorf("could not detect if anonymous authentication is enabled: %w", err)
}
plog.Debug("anonymous authentication probed", "anonymousAuthEnabled", anonymousAuthEnabled)
// if we ever start unioning a TCR bearer token authenticator with serverConfig.Authenticator
// then we will need to update the related assumption in tokenPassthroughRoundTripper
delegatingAuthenticator := serverConfig.Authentication.Authenticator
blockAnonymousAuthenticator := &comparableAuthenticator{
RequestFunc: func(req *http.Request) (*authenticator.Response, bool, error) {
resp, ok, err := delegatingAuthenticator.AuthenticateRequest(req)
// anonymous auth is enabled so no further check is necessary
if anonymousAuthEnabled {
return resp, ok, err
}
// authentication failed
if err != nil || !ok {
return resp, ok, err
}
// any other user than anonymous is irrelevant
if resp.User.GetName() != user.Anonymous {
return resp, ok, err
}
reqInfo, ok := genericapirequest.RequestInfoFrom(req.Context())
if !ok {
return nil, false, constable.Error("no RequestInfo found in the context")
}
// a TKR is a resource, any request that is not for a resource should not be authenticated
if !reqInfo.IsResourceRequest {
return nil, false, nil
}
// any resource besides TKR should not be authenticated
if !isTokenCredReq(reqInfo) {
return nil, false, nil
}
// anonymous authentication is disabled, but we must let an anonymous request
// to TKR authenticate as this is the only method to retrieve credentials
return resp, ok, err
},
}
// Set our custom authenticator before calling Compete(), which will use it.
serverConfig.Authentication.Authenticator = blockAnonymousAuthenticator
delegatingAuthorizer := serverConfig.Authorization.Authorizer
customReasonAuthorizer := &comparableAuthorizer{
AuthorizerFunc: func(ctx context.Context, a authorizer.Attributes) (authorizer.Decision, string, error) {
const baseReason = "decision made by impersonation-proxy.concierge.pinniped.dev"
switch a.GetVerb() {
case "":
// Empty string is disallowed because request info has had bugs in the past where it would leave it empty.
return authorizer.DecisionDeny, "invalid verb, " + baseReason, nil
default:
// Since we authenticate the requesting user, we are in the best position to correctly authorize them.
// When KAS does the check, it may run the check against our service account and not the requesting user
// (due to a bug in the code or any other internal SAR checks that the request processing does).
// This also handles the impersonate verb to allow for nested impersonation.
decision, reason, err := delegatingAuthorizer.Authorize(ctx, a)
// make it easier to detect when the impersonation proxy is authorizing a request vs KAS
switch len(reason) {
case 0:
reason = baseReason
default:
reason = reason + ", " + baseReason
}
return decision, reason, err
}
},
}
// Set our custom authorizer before calling Compete(), which will use it.
serverConfig.Authorization.Authorizer = customReasonAuthorizer
if recConfig != nil {
recConfig(serverConfig)
}
completedConfig := serverConfig.Complete()
impersonationProxyServer, err := completedConfig.New("impersonation-proxy", genericapiserver.NewEmptyDelegate())
if err != nil {
return nil, err
}
preparedRun := impersonationProxyServer.PrepareRun()
// Sanity check. Make sure that our custom authenticator is still in place and did not get changed or wrapped.
if completedConfig.Authentication.Authenticator != blockAnonymousAuthenticator {
return nil, fmt.Errorf("invalid mutation of anonymous authenticator detected: %#v", completedConfig.Authentication.Authenticator)
}
// Sanity check. Make sure that our custom authorizer is still in place and did not get changed or wrapped.
if preparedRun.Authorizer != customReasonAuthorizer {
return nil, fmt.Errorf("invalid mutation of impersonation authorizer detected: %#v", preparedRun.Authorizer)
}
// Sanity check. Assert that we have a functioning token file to use and no bearer token.
if len(preparedRun.LoopbackClientConfig.BearerToken) != 0 || len(preparedRun.LoopbackClientConfig.BearerTokenFile) == 0 {
return nil, constable.Error("invalid impersonator loopback rest config has wrong bearer token semantics")
}
return preparedRun.Run, nil
}
result, err := constructServer()
// If there was any error during construction, then we would like to close the listener to free up the port.
if err != nil {
errs := []error{err}
if listener != nil {
errs = append(errs, listener.Close())
}
return nil, errors.NewAggregate(errs)
}
return result, nil
}
func getReverseProxyClient(clientOpts []kubeclient.Option) (*kubeclient.Client, error) {
// just use the overrides given during unit tests
if len(clientOpts) != 0 {
return kubeclient.New(clientOpts...)
}
// this is the magic path where the impersonation proxy SA token is mounted
const tokenFile = "/var/run/secrets/impersonation-proxy.concierge.pinniped.dev/serviceaccount/token" //nolint:gosec // this is not a credential
// make sure the token file we need exists before trying to use it
if _, err := os.Stat(tokenFile); err != nil {
return nil, err
}
// build an in cluster config that uses the impersonation proxy token file
impersonationProxyRestConfig, err := rest.InClusterConfig()
if err != nil {
return nil, err
}
impersonationProxyRestConfig = kubeclient.SecureAnonymousClientConfig(impersonationProxyRestConfig)
impersonationProxyRestConfig.BearerTokenFile = tokenFile
return kubeclient.New(kubeclient.WithConfig(impersonationProxyRestConfig))
}
func isAnonymousAuthEnabled(config *rest.Config) (bool, error) {
anonymousConfig := kubeclient.SecureAnonymousClientConfig(config)
// we do not need either of these but RESTClientFor complains if they are not set
anonymousConfig.GroupVersion = &schema.GroupVersion{}
anonymousConfig.NegotiatedSerializer = unstructuredscheme.NewUnstructuredNegotiatedSerializer()
// in case anyone looking at audit logs wants to know who is making the anonymous request
anonymousConfig.UserAgent = rest.DefaultKubernetesUserAgent()
rc, err := rest.RESTClientFor(anonymousConfig)
if err != nil {
return false, err
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
_, errHealthz := rc.Get().AbsPath("/healthz").DoRaw(ctx)
switch {
// 200 ok on healthz clearly indicates authentication success
case errHealthz == nil:
return true, nil
// we are authenticated but not authorized. anonymous authentication is enabled
case apierrors.IsForbidden(errHealthz):
return true, nil
// failure to authenticate will return unauthorized (http misnomer)
case apierrors.IsUnauthorized(errHealthz):
return false, nil
// any other error is unexpected
default:
return false, errHealthz
}
}
func isTokenCredReq(reqInfo *genericapirequest.RequestInfo) bool {
if reqInfo.Resource != "tokencredentialrequests" {
return false
}
// pinniped components allow for the group suffix to be customized
// rather than wiring in the current configured suffix, checking the prefix is sufficient
if !strings.HasPrefix(reqInfo.APIGroup, "login.concierge.") {
return false
}
return true
}
// No-op wrapping around RequestFunc to allow for comparisons.
type comparableAuthenticator struct {
authenticator.RequestFunc
}
// No-op wrapping around AuthorizerFunc to allow for comparisons.
type comparableAuthorizer struct {
authorizer.AuthorizerFunc
}
func withBearerTokenPreservation(delegate http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// this looks a bit hacky but lets us avoid writing any logic for parsing out the bearer token
var reqToken string
_, _, _ = bearertoken.New(authenticator.TokenFunc(func(_ context.Context, token string) (*authenticator.Response, bool, error) {
reqToken = token
return nil, false, nil
})).AuthenticateRequest(r)
// smuggle the token through the context. this does mean that we need to avoid logging the context.
if len(reqToken) != 0 {
ctx := context.WithValue(r.Context(), tokenKey, reqToken)
r = r.WithContext(ctx)
}
delegate.ServeHTTP(w, r)
})
}
func tokenFrom(ctx context.Context) string {
token, _ := ctx.Value(tokenKey).(string)
return token
}
// contextKey type is unexported to prevent collisions.
type contextKey int
const tokenKey contextKey = iota
func newImpersonationReverseProxyFunc(restConfig *rest.Config) (func(*genericapiserver.Config) http.Handler, error) {
serverURL, err := url.Parse(restConfig.Host)
if err != nil {
return nil, fmt.Errorf("could not parse host URL from in-cluster config: %w", err)
}
http1RoundTripper, err := getTransportForProtocol(restConfig, "http/1.1")
if err != nil {
return nil, fmt.Errorf("could not get http/1.1 round tripper: %w", err)
}
http1RoundTripperAnonymous, err := getTransportForProtocol(kubeclient.SecureAnonymousClientConfig(restConfig), "http/1.1")
if err != nil {
return nil, fmt.Errorf("could not get http/1.1 anonymous round tripper: %w", err)
}
http2RoundTripper, err := getTransportForProtocol(restConfig, "h2")
if err != nil {
return nil, fmt.Errorf("could not get http/2.0 round tripper: %w", err)
}
http2RoundTripperAnonymous, err := getTransportForProtocol(kubeclient.SecureAnonymousClientConfig(restConfig), "h2")
if err != nil {
return nil, fmt.Errorf("could not get http/2.0 anonymous round tripper: %w", err)
}
return func(c *genericapiserver.Config) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if len(r.Header.Values("Authorization")) != 0 {
plog.Warning("aggregated API server logic did not delete authorization header but it is always supposed to do so",
"url", r.URL.String(),
"method", r.Method,
)
newInternalErrResponse(w, r, c.Serializer, "invalid authorization header")
return
}
if err := ensureNoImpersonationHeaders(r); err != nil {
plog.Error("unknown impersonation header seen",
err,
"url", r.URL.String(),
"method", r.Method,
)
newInternalErrResponse(w, r, c.Serializer, "invalid impersonation")
return
}
userInfo, ok := request.UserFrom(r.Context())
if !ok {
plog.Warning("aggregated API server logic did not set user info but it is always supposed to do so",
"url", r.URL.String(),
"method", r.Method,
)
newInternalErrResponse(w, r, c.Serializer, "invalid user")
return
}
Update all deps to latest where possible, bump Kube deps to v0.23.1 Highlights from this dep bump: 1. Made a copy of the v0.4.0 github.com/go-logr/stdr implementation for use in tests. We must bump this dep as Kube code uses a newer version now. We would have to rewrite hundreds of test log assertions without this copy. 2. Use github.com/felixge/httpsnoop to undo the changes made by ory/fosite#636 for CLI based login flows. This is required for backwards compatibility with older versions of our CLI. A separate change after this will update the CLI to be more flexible (it is purposefully not part of this change to confirm that we did not break anything). For all browser login flows, we now redirect using http.StatusSeeOther instead of http.StatusFound. 3. Drop plog.RemoveKlogGlobalFlags as klog no longer mutates global process flags 4. Only bump github.com/ory/x to v0.0.297 instead of the latest v0.0.321 because v0.0.298+ pulls in a newer version of go.opentelemetry.io/otel/semconv which breaks k8s.io/apiserver. We should update k8s.io/apiserver to use the newer code. 5. Migrate all code from k8s.io/apimachinery/pkg/util/clock to k8s.io/utils/clock and k8s.io/utils/clock/testing 6. Delete testutil.NewDeleteOptionsRecorder and migrate to the new kubetesting.NewDeleteActionWithOptions 7. Updated ExpectedAuthorizeCodeSessionJSONFromFuzzing caused by fosite's new rotated_secrets OAuth client field. This new field is currently not relevant to us as we have no private clients. Signed-off-by: Monis Khan <mok@vmware.com>
2021-12-10 22:22:36 +00:00
ae := audit.AuditEventFrom(r.Context())
if ae == nil {
plog.Warning("aggregated API server logic did not set audit event but it is always supposed to do so",
"url", r.URL.String(),
"method", r.Method,
)
newInternalErrResponse(w, r, c.Serializer, "invalid audit event")
return
}
// grab the request's bearer token if present. this is optional and does not fail the request if missing.
token := tokenFrom(r.Context())
// KAS only supports upgrades via http/1.1 to websockets/SPDY (upgrades never use http/2.0)
// Thus we default to using http/2.0 when the request is not an upgrade, otherwise we use http/1.1
baseRT, baseRTAnonymous := http2RoundTripper, http2RoundTripperAnonymous
isUpgradeRequest := httpstream.IsUpgradeRequest(r)
if isUpgradeRequest {
baseRT, baseRTAnonymous = http1RoundTripper, http1RoundTripperAnonymous
}
rt, err := getTransportForUser(r.Context(), userInfo, baseRT, baseRTAnonymous, ae, token, c.Authentication.Authenticator)
if err != nil {
plog.WarningErr("rejecting request as we cannot act as the current user", err,
"url", r.URL.String(),
"method", r.Method,
"isUpgradeRequest", isUpgradeRequest,
)
newInternalErrResponse(w, r, c.Serializer, "unimplemented functionality - unable to act as current user")
return
}
plog.Debug("impersonation proxy servicing request",
"url", r.URL.String(),
"method", r.Method,
"isUpgradeRequest", isUpgradeRequest,
)
plog.Trace("impersonation proxy servicing request was for user",
"url", r.URL.String(),
"method", r.Method,
"isUpgradeRequest", isUpgradeRequest,
"username", userInfo.GetName(), // this info leak seems fine for trace level logs
)
2021-03-18 17:00:06 +00:00
// The proxy library used below will panic when the client disconnects abruptly, so in order to
// assure that this log message is always printed at the end of this func, it must be deferred.
defer plog.Debug("impersonation proxy finished servicing request",
"url", r.URL.String(),
"method", r.Method,
"isUpgradeRequest", isUpgradeRequest,
)
2021-03-18 17:00:06 +00:00
// do not allow the client to cause log confusion by spoofing this header
if len(r.Header.Values("X-Forwarded-For")) > 0 {
r = utilnet.CloneRequest(r)
r.Header.Del("X-Forwarded-For")
}
// the http2 code seems to call Close concurrently which can lead to data races
if r.Body != nil {
r = utilnet.CloneRequest(r)
r.Body = &safeReadWriteCloser{rc: r.Body}
}
reverseProxy := httputil.NewSingleHostReverseProxy(serverURL)
reverseProxy.Transport = rt
reverseProxy.FlushInterval = 200 * time.Millisecond // the "watch" verb will not work without this line
reverseProxy.ServeHTTP(w, r)
})
}, nil
}
var _ io.ReadWriteCloser = &safeReadWriteCloser{}
type safeReadWriteCloser struct {
m sync.Mutex // all methods allowed concurrently, this only guards concurrent calls to Close
rc io.ReadCloser
once sync.Once // set up rwc and writeErr
rwc io.ReadWriteCloser
writeErr error
}
func (r *safeReadWriteCloser) Read(p []byte) (int, error) {
return r.rc.Read(p)
}
func (r *safeReadWriteCloser) Write(p []byte) (int, error) {
r.once.Do(func() {
var ok bool
r.rwc, ok = r.rc.(io.ReadWriteCloser)
if !ok { // this method should only be caused during flows that switch protocols
r.writeErr = fmt.Errorf("switching protocols failed: io.ReadCloser %T is not io.ReadWriteCloser", r.rc)
}
})
if r.writeErr != nil {
return 0, r.writeErr
}
return r.rwc.Write(p)
}
func (r *safeReadWriteCloser) Close() error {
r.m.Lock()
defer r.m.Unlock()
return r.rc.Close()
}
func ensureNoImpersonationHeaders(r *http.Request) error {
for key := range r.Header {
// even though we have unit tests that try to cover this case, it is hard to tell if Go does
// client side canonicalization on encode, server side canonicalization on decode, or both
key := http.CanonicalHeaderKey(key)
if strings.HasPrefix(key, "Impersonate") {
return fmt.Errorf("%q header already exists", key)
}
}
return nil
}
func getTransportForUser(ctx context.Context, userInfo user.Info, delegate, delegateAnonymous http.RoundTripper, ae *auditinternal.Event, token string, authenticator authenticator.Request) (http.RoundTripper, error) {
if canImpersonateFully(userInfo) {
return standardImpersonationRoundTripper(userInfo, ae, delegate)
}
return tokenPassthroughRoundTripper(ctx, delegateAnonymous, ae, token, authenticator)
}
func canImpersonateFully(userInfo user.Info) bool {
//nolint:gosimple // this structure is on purpose because we plan to expand this function
if len(userInfo.GetUID()) == 0 {
return true
}
// once kube supports UID impersonation, add logic to detect if the KAS is
// new enough to have this functionality and return true in that case as well
return false
}
func standardImpersonationRoundTripper(userInfo user.Info, ae *auditinternal.Event, delegate http.RoundTripper) (http.RoundTripper, error) {
extra, err := buildExtra(userInfo.GetExtra(), ae)
if err != nil {
return nil, err
}
impersonateConfig := transport.ImpersonationConfig{
UserName: userInfo.GetName(),
Groups: userInfo.GetGroups(),
Extra: extra,
}
// transport.NewImpersonatingRoundTripper clones the request before setting headers
// thus it will not accidentally mutate the input request (see http.Handler docs)
return transport.NewImpersonatingRoundTripper(impersonateConfig, delegate), nil
}
func tokenPassthroughRoundTripper(ctx context.Context, delegateAnonymous http.RoundTripper, ae *auditinternal.Event, token string, authenticator authenticator.Request) (http.RoundTripper, error) {
// all code below assumes KAS does not support UID impersonation because that case is handled in the standard path
// it also assumes that the TCR API does not issue tokens - if this assumption changes, we will need
// some way to distinguish a token that is only valid against this impersonation proxy and not against KAS.
// this code will fail closed because said TCR token would not work against KAS and the request would fail.
// if we get here we know the final user info had a UID
// if the original user is also performing a nested impersonation, it means that said nested
// impersonation is trying to impersonate a UID since final user info == ae.ImpersonatedUser
// we know this KAS does not support UID impersonation so this request must be rejected
if ae.ImpersonatedUser != nil {
return nil, constable.Error("unable to impersonate uid")
}
// see what KAS thinks this token translates into
// this is important because certs have precedence over tokens and we want
// to make sure that we do not get confused and pass along the wrong token
tokenUser, err := tokenReview(ctx, token, authenticator)
if err != nil {
return nil, err
}
// we want to compare the result of the token authentication with the original user that made the request
// if the user who made the request and the token do not match, we cannot go any further at this point
if !apiequality.Semantic.DeepEqual(ae.User, tokenUser) {
// this info leak seems fine for trace level logs
plog.Trace("failed to passthrough token due to user mismatch",
"original-username", ae.User.Username,
"original-uid", ae.User.UID,
"token-username", tokenUser.Username,
"token-uid", tokenUser.UID,
)
return nil, constable.Error("token authenticated as a different user")
}
// now we know that if we send this token to KAS, it will authenticate correctly
return transport.NewBearerAuthRoundTripper(token, delegateAnonymous), nil
}
func tokenReview(ctx context.Context, token string, authenticator authenticator.Request) (authenticationv1.UserInfo, error) {
if len(token) == 0 {
return authenticationv1.UserInfo{}, constable.Error("no token on request")
}
// create a header that contains nothing but the token
// an astute observer may ask "but what about the token's audience?"
// in this case, we want to leave audiences unset per the token review docs:
// > If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver.
// i.e. we want to make sure that the given token is valid against KAS
fakeReq := &http.Request{Header: http.Header{}}
fakeReq.Header.Set("Authorization", "Bearer "+token)
// propagate cancellation of parent context (without any values such as audience)
fakeReq = fakeReq.WithContext(valuelesscontext.New(ctx))
// this will almost always be a free call that hits our 10 second cache TTL
resp, ok, err := authenticator.AuthenticateRequest(fakeReq)
if err != nil {
return authenticationv1.UserInfo{}, err
}
if !ok {
return authenticationv1.UserInfo{}, constable.Error("token failed to authenticate")
}
tokenUser := authenticationv1.UserInfo{
Username: resp.User.GetName(),
UID: resp.User.GetUID(),
Groups: resp.User.GetGroups(),
Extra: make(map[string]authenticationv1.ExtraValue, len(resp.User.GetExtra())),
}
for k, v := range resp.User.GetExtra() {
tokenUser.Extra[k] = v
}
return tokenUser, nil
}
func buildExtra(extra map[string][]string, ae *auditinternal.Event) (map[string][]string, error) {
const reservedImpersonationProxySuffix = ".impersonation-proxy.concierge.pinniped.dev"
// always validate that the extra is something we support irregardless of nested impersonation
for k := range extra {
if !extraKeyRegexp.MatchString(k) {
return nil, fmt.Errorf("disallowed extra key seen: %s", k)
}
if strings.HasSuffix(k, reservedImpersonationProxySuffix) {
return nil, fmt.Errorf("disallowed extra key with reserved prefix seen: %s", k)
}
}
if ae.ImpersonatedUser == nil {
return extra, nil // just return the given extra since nested impersonation is not being used
}
// avoid mutating input map, preallocate new map to store original user info
out := make(map[string][]string, len(extra)+1)
for k, v := range extra {
out[k] = v // shallow copy of slice since we are not going to mutate it
}
origUserInfoJSON, err := json.Marshal(ae.User)
if err != nil {
return nil, err
}
out["original-user-info"+reservedImpersonationProxySuffix] = []string{string(origUserInfoJSON)}
return out, nil
}
// extraKeyRegexp is a very conservative regex to handle impersonation's extra key fidelity limitations such as casing and escaping.
var extraKeyRegexp = regexp.MustCompile(`^[a-z0-9/\-._]+$`)
func newInternalErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, msg string) {
newStatusErrResponse(w, r, s, apierrors.NewInternalError(constable.Error(msg)))
}
func newStatusErrResponse(w http.ResponseWriter, r *http.Request, s runtime.NegotiatedSerializer, err *apierrors.StatusError) {
requestInfo, ok := genericapirequest.RequestInfoFrom(r.Context())
if !ok {
responsewriters.InternalError(w, r, constable.Error("no RequestInfo found in the context"))
return
}
gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion}
responsewriters.ErrorNegotiated(err, s, gv, w, r)
}
func getTransportForProtocol(restConfig *rest.Config, protocol string) (http.RoundTripper, error) {
transportConfig, err := restConfig.TransportConfig()
if err != nil {
return nil, fmt.Errorf("could not get in-cluster transport config: %w", err)
}
transportConfig.TLS.NextProtos = []string{protocol}
rt, err := transport.New(transportConfig)
if err != nil {
return nil, fmt.Errorf("could not build transport: %w", err)
}
// For clients that support http2, transport.New calls http2.ConfigureTransports,
// which configures with both h2 and http/1.1,
// even when you explicitly only ask for h2.
// Override that change.
cfg, err := utilnet.TLSClientConfig(rt)
if err != nil {
return nil, fmt.Errorf("could not extract TLS config: %w", err)
}
cfg.NextProtos = []string{protocol}
if err := kubeclient.AssertSecureTransport(rt); err != nil {
return nil, err // make sure we only use a secure TLS config
}
return rt, nil
}