From b00cec954ef6a02e67488502ba5a96e50507eea2 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Fri, 7 Aug 2020 14:49:04 -0700 Subject: [PATCH 1/6] Pre-factor server.go - No functional changes - Move all the stuff about clients and controllers into the controller package - Add more comments and organize the code more into more helper functions to make each function smaller --- internal/controller/prepare_controllers.go | 139 +++++++++++++ internal/server/server.go | 222 +++++++-------------- internal/server/server_test.go | 2 +- 3 files changed, 210 insertions(+), 153 deletions(-) create mode 100644 internal/controller/prepare_controllers.go diff --git a/internal/controller/prepare_controllers.go b/internal/controller/prepare_controllers.go new file mode 100644 index 00000000..96d41ac5 --- /dev/null +++ b/internal/controller/prepare_controllers.go @@ -0,0 +1,139 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package controller + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/runtime" + k8sinformers "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + + "github.com/suzerain-io/controller-go" + "github.com/suzerain-io/placeholder-name/internal/autoregistration" + "github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery" + placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" + placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions" +) + +const ( + singletonWorker = 1 + defaultResyncInterval = 3 * time.Minute +) + +// Prepare the controllers and their informers and return a function that will start them when called. +func PrepareControllers( + ctx context.Context, + caBundle []byte, + serverInstallationNamespace string, + discoveryURLOverride *string, +) (func(ctx context.Context), error) { + // Create k8s clients. + k8sClient, aggregationClient, placeholderClient, err := createClients() + if err != nil { + return nil, fmt.Errorf("could not create clients for the controllers: %w", err) + } + + // TODO Putting this here temporarily on the way toward moving it elsewhere. + // When it moves elsewhere then PrepareControllers() will not need to take ctx and caBundle parameters. + if err := autoregistration.UpdateAPIService(ctx, aggregationClient, caBundle); err != nil { + return nil, fmt.Errorf("could not update the API service: %w", err) + } + + // Create informers. + k8sInformers, placeholderInformers := createInformers(serverInstallationNamespace, k8sClient, placeholderClient) + + // Create controller manager. + controllerManager := controller. + NewManager(). + WithController( + logindiscovery.NewPublisherController( + serverInstallationNamespace, + discoveryURLOverride, + placeholderClient, + k8sInformers.Core().V1().ConfigMaps(), + placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(), + controller.WithInformer, + ), + singletonWorker, + ) + + // Return a function which starts the informers and controllers. + return func(ctx context.Context) { + k8sInformers.Start(ctx.Done()) + placeholderInformers.Start(ctx.Done()) + go controllerManager.Start(ctx) + }, nil +} + +// Create the k8s clients that will be used by the controllers. +func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *placeholderclientset.Clientset, error) { + // Load the Kubernetes client configuration (kubeconfig), + kubeConfig, err := restclient.InClusterConfig() + if err != nil { + return nil, nil, nil, fmt.Errorf("could not load in-cluster configuration: %w", err) + } + + // explicitly use protobuf when talking to built-in kube APIs + protoKubeConfig := createProtoKubeConfig(kubeConfig) + + // Connect to the core Kubernetes API. + k8sClient, err := kubernetes.NewForConfig(protoKubeConfig) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) + } + + // Connect to the Kubernetes aggregation API. + aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) + } + + // Connect to the placeholder API. + // I think we can't use protobuf encoding here because we are using CRDs + // (for which protobuf encoding is not supported). + placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig) + if err != nil { + return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err) + } + + return k8sClient, aggregationClient, placeholderClient, nil +} + +// Create the informers that will be used by the controllers. +func createInformers( + serverInstallationNamespace string, + k8sClient *kubernetes.Clientset, + placeholderClient *placeholderclientset.Clientset, +) (k8sinformers.SharedInformerFactory, placeholderinformers.SharedInformerFactory) { + k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions( + k8sClient, + defaultResyncInterval, + k8sinformers.WithNamespace( + logindiscovery.ClusterInfoNamespace, + ), + ) + placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions( + placeholderClient, + defaultResyncInterval, + placeholderinformers.WithNamespace(serverInstallationNamespace), + ) + return k8sInformers, placeholderInformers +} + +// Returns a copy of the input config with the ContentConfig set to use protobuf. +// Do not use this config to communicate with any CRD based APIs. +func createProtoKubeConfig(kubeConfig *restclient.Config) *restclient.Config { + protoKubeConfig := restclient.CopyConfig(kubeConfig) + const protoThenJSON = runtime.ContentTypeProtobuf + "," + runtime.ContentTypeJSON + protoKubeConfig.AcceptContentTypes = protoThenJSON + protoKubeConfig.ContentType = runtime.ContentTypeProtobuf + return protoKubeConfig +} diff --git a/internal/server/server.go b/internal/server/server.go index c4360763..6ec828ce 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -17,44 +17,28 @@ import ( "time" "github.com/spf13/cobra" - "k8s.io/apimachinery/pkg/runtime" genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/apiserver/pkg/server/dynamiccertificates" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" - k8sinformers "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" - aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" - "github.com/suzerain-io/controller-go" "github.com/suzerain-io/placeholder-name/internal/apiserver" - "github.com/suzerain-io/placeholder-name/internal/autoregistration" "github.com/suzerain-io/placeholder-name/internal/certauthority" - "github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery" + "github.com/suzerain-io/placeholder-name/internal/controller" "github.com/suzerain-io/placeholder-name/internal/downward" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" - placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" - placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions" "github.com/suzerain-io/placeholder-name/pkg/config" ) -const ( - singletonWorker = 1 - defaultResyncInterval = 3 * time.Minute -) - // App is an object that represents the placeholder-name-server application. type App struct { - cmd *cobra.Command + serverCommand *cobra.Command // CLI flags configPath string downwardAPIPath string clusterSigningCertFilePath string clusterSigningKeyFilePath string - - recommendedOptions *genericoptions.RecommendedOptions } // This is ignored for now because we turn off etcd storage below, but this is @@ -63,61 +47,39 @@ const defaultEtcdPathPrefix = "/registry/" + placeholderv1alpha1.GroupName // New constructs a new App with command line args, stdout and stderr. func New(ctx context.Context, args []string, stdout, stderr io.Writer) *App { - a := &App{ - recommendedOptions: genericoptions.NewRecommendedOptions( - defaultEtcdPathPrefix, - apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion), - // TODO we should check to see if all the other default settings are acceptable for us - ), - } - a.recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet + app := &App{} + app.addServerCommand(ctx, args, stdout, stderr) + return app +} +// Run the server. +func (app *App) Run() error { + return app.serverCommand.Execute() +} + +// Create the server command and save it into the App. +func (app *App) addServerCommand(ctx context.Context, args []string, stdout, stderr io.Writer) { cmd := &cobra.Command{ Use: `placeholder-name-server`, - Long: `placeholder-name-server provides a generic API for mapping an external -credential from somewhere to an internal credential to be used for -authenticating to the Kubernetes API.`, - RunE: func(cmd *cobra.Command, args []string) error { - // Load the Kubernetes client configuration (kubeconfig), - kubeConfig, err := restclient.InClusterConfig() - if err != nil { - return fmt.Errorf("could not load in-cluster configuration: %w", err) - } - - // explicitly use protobuf when talking to built-in kube APIs - protoKubeConfig := createProtoKubeConfig(kubeConfig) - - // Connect to the core Kubernetes API. - k8sClient, err := kubernetes.NewForConfig(protoKubeConfig) - if err != nil { - return fmt.Errorf("could not initialize Kubernetes client: %w", err) - } - - // Connect to the Kubernetes aggregation API. - aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig) - if err != nil { - return fmt.Errorf("could not initialize Kubernetes client: %w", err) - } - - // Connect to the placeholder API. - // I think we can't use protobuf encoding here because we are using CRDs - // (for which protobuf encoding is not supported). - placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig) - if err != nil { - return fmt.Errorf("could not initialize placeholder client: %w", err) - } - - return a.run(ctx, k8sClient, aggregationClient, placeholderClient) - }, + Long: "placeholder-name-server provides a generic API for mapping an external\n" + + "credential from somewhere to an internal credential to be used for\n" + + "authenticating to the Kubernetes API.", + RunE: func(cmd *cobra.Command, args []string) error { return app.runServer(ctx) }, Args: cobra.NoArgs, } cmd.SetArgs(args) cmd.SetOut(stdout) cmd.SetErr(stderr) + addCommandlineFlagsToCommand(cmd, app) + app.serverCommand = cmd +} + +// Define the app's commandline flags. +func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) { cmd.Flags().StringVarP( - &a.configPath, + &app.configPath, "config", "c", "placeholder-name.yaml", @@ -125,73 +87,64 @@ authenticating to the Kubernetes API.`, ) cmd.Flags().StringVar( - &a.downwardAPIPath, + &app.downwardAPIPath, "downward-api-path", "/etc/podinfo", "path to Downward API volume mount", ) cmd.Flags().StringVar( - &a.clusterSigningCertFilePath, + &app.clusterSigningCertFilePath, "cluster-signing-cert-file", "", "path to cluster signing certificate", ) cmd.Flags().StringVar( - &a.clusterSigningKeyFilePath, + &app.clusterSigningKeyFilePath, "cluster-signing-key-file", "", "path to cluster signing private key", ) - - a.cmd = cmd - - return a } -func (a *App) Run() error { - return a.cmd.Execute() -} - -func (a *App) run( - ctx context.Context, - k8sClient kubernetes.Interface, - aggregationClient aggregationv1client.Interface, - placeholderClient placeholderclientset.Interface, -) error { - cfg, err := config.FromPath(a.configPath) +// Boot the aggregated API server, which will in turn boot the controllers. +func (app *App) runServer(ctx context.Context) error { + // Read the server config file. + cfg, err := config.FromPath(app.configPath) if err != nil { return fmt.Errorf("could not load config: %w", err) } // Load the Kubernetes cluster signing CA. - clientCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath) + k8sClusterCA, err := certauthority.Load(app.clusterSigningCertFilePath, app.clusterSigningKeyFilePath) if err != nil { return fmt.Errorf("could not load cluster signing CA: %w", err) } + // Create a WebhookTokenAuthenticator. webhookTokenAuthenticator, err := config.NewWebhook(cfg.WebhookConfig) if err != nil { return fmt.Errorf("could not create webhook client: %w", err) } - podinfo, err := downward.Load(a.downwardAPIPath) + // Discover in which namespace we are installed. + podInfo, err := downward.Load(app.downwardAPIPath) if err != nil { return fmt.Errorf("could not read pod metadata: %w", err) } - serverInstallationNamespace := podinfo.Namespace - - // TODO use the postStart hook to generate certs? + serverInstallationNamespace := podInfo.Namespace + // Create a CA. aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"}) if err != nil { return fmt.Errorf("could not initialize CA: %w", err) } + // This string must match the name of the Service declared in the deployment yaml. const serviceName = "placeholder-name-api" - - cert, err := aggregatedAPIServerCA.Issue( + // Using the CA from above, create a TLS server cert for the aggregated API server to use. + aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( pkix.Name{CommonName: serviceName + "." + serverInstallationNamespace + ".svc"}, []string{}, 24*365*time.Hour, @@ -200,35 +153,41 @@ func (a *App) run( return fmt.Errorf("could not issue serving certificate: %w", err) } - if err := autoregistration.UpdateAPIService(ctx, aggregationClient, aggregatedAPIServerCA.Bundle()); err != nil { - return fmt.Errorf("could not register API service: %w", err) - } - - cmrf := wireControllerManagerRunFunc( + // Prepare to start the controllers, but defer actually starting them until the + // post start hook of the aggregated API server. + startControllersFunc, err := controller.PrepareControllers( + ctx, + aggregatedAPIServerCA.Bundle(), serverInstallationNamespace, cfg.DiscoveryConfig.URL, - k8sClient, - placeholderClient, ) - apiServerConfig, err := a.configServer( - cert, + if err != nil { + return fmt.Errorf("could not prepare controllers: %w", err) + } + + // Get the aggregated API server config. + aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig( + aggregatedAPIServerTLSCert, webhookTokenAuthenticator, - clientCA, - cmrf, + k8sClusterCA, + startControllersFunc, ) if err != nil { - return err + return fmt.Errorf("could not configure aggregated API server: %w", err) } - server, err := apiServerConfig.Complete().New() + // Complete the aggregated API server config and make a server instance. + server, err := aggregatedAPIServerConfig.Complete().New() if err != nil { - return fmt.Errorf("could not issue serving certificate: %w", err) + return fmt.Errorf("could not create aggregated API server: %w", err) } + // Run the server. Its post-start hook will start the controllers. return server.GenericAPIServer.PrepareRun().Run(ctx.Done()) } -func (a *App) configServer( +// Create a configuration for the aggregated API server. +func getAggregatedAPIServerConfig( cert *tls.Certificate, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, ca *certauthority.CA, @@ -238,10 +197,17 @@ func (a *App) configServer( if err != nil { return nil, fmt.Errorf("could not create static cert key provider: %w", err) } - a.recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider + + recommendedOptions := genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion), + // TODO we should check to see if all the other default settings are acceptable for us + ) + recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet + recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) - if err := a.recommendedOptions.ApplyTo(serverConfig); err != nil { + if err := recommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } @@ -256,16 +222,6 @@ func (a *App) configServer( return apiServerConfig, nil } -// createProtoKubeConfig returns a copy of the input config with the ContentConfig set to use protobuf. -// do not use this config to communicate with any CRD based APIs. -func createProtoKubeConfig(kubeConfig *restclient.Config) *restclient.Config { - protoKubeConfig := restclient.CopyConfig(kubeConfig) - const protoThenJSON = runtime.ContentTypeProtobuf + "," + runtime.ContentTypeJSON - protoKubeConfig.AcceptContentTypes = protoThenJSON - protoKubeConfig.ContentType = runtime.ContentTypeProtobuf - return protoKubeConfig -} - func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.CertKeyContentProvider, error) { privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) if err != nil { @@ -289,41 +245,3 @@ func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.Cer return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM) } - -func wireControllerManagerRunFunc( - serverInstallationNamespace string, - discoveryURLOverride *string, - k8s kubernetes.Interface, - placeholder placeholderclientset.Interface, -) func(ctx context.Context) { - k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions( - k8s, - defaultResyncInterval, - k8sinformers.WithNamespace( - logindiscovery.ClusterInfoNamespace, - ), - ) - placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions( - placeholder, - defaultResyncInterval, - placeholderinformers.WithNamespace(serverInstallationNamespace), - ) - cm := controller. - NewManager(). - WithController( - logindiscovery.NewPublisherController( - serverInstallationNamespace, - discoveryURLOverride, - placeholder, - k8sInformers.Core().V1().ConfigMaps(), - placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(), - controller.WithInformer, - ), - singletonWorker, - ) - return func(ctx context.Context) { - k8sInformers.Start(ctx.Done()) - placeholderInformers.Start(ctx.Done()) - go cm.Start(ctx) - } -} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 5ae9abe8..fc31dc4e 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -78,7 +78,7 @@ func TestCommand(t *testing.T) { stderr := bytes.NewBuffer([]byte{}) a := New(context.Background(), test.args, stdout, stderr) - a.cmd.RunE = func(cmd *cobra.Command, args []string) error { + a.serverCommand.RunE = func(cmd *cobra.Command, args []string) error { return nil } err := a.Run() From 86c3f89b2e13074ba88f0a02a275c58c222bd480 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Sun, 9 Aug 2020 10:04:05 -0700 Subject: [PATCH 2/6] First draft of moving API server TLS cert generation to controllers - Refactors the existing cert generation code into controllers which read and write a Secret containing the certs - Does not add any new functionality yet, e.g. no new handling for cert expiration, and no leader election to allow for multiple servers running simultaneously - This commit also doesn't add new tests for the cert generation code, but it should be more unit testable now as controllers --- Dockerfile | 3 + README.md | 4 +- deploy/rbac.yaml | 17 +- internal/autoregistration/autoregistration.go | 6 +- internal/controller/apicerts/certs_manager.go | 164 ++++++++++++++++++ .../controller/apicerts/certs_observer.go | 69 ++++++++ .../controller/logindiscovery/publisher.go | 26 +-- internal/controller/utils.go | 31 ++++ .../prepare_controllers.go | 89 ++++++---- .../dynamic_tls_serving_cert_provider.go | 19 ++ internal/server/server.go | 81 +++------ 11 files changed, 391 insertions(+), 118 deletions(-) create mode 100644 internal/controller/apicerts/certs_manager.go create mode 100644 internal/controller/apicerts/certs_observer.go create mode 100644 internal/controller/utils.go rename internal/{controller => controllermanager}/prepare_controllers.go (56%) create mode 100644 internal/provider/dynamic_tls_serving_cert_provider.go diff --git a/Dockerfile b/Dockerfile index f03e2545..00eaa5d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,6 @@ +# Copyright 2020 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 + FROM golang:1.14-alpine as build-env # It is important that these ARG's are defined after the FROM statement diff --git a/README.md b/README.md index a84c0d2c..49b59c26 100644 --- a/README.md +++ b/README.md @@ -1 +1,3 @@ -# placeholder-name \ No newline at end of file +# placeholder-name + +Copyright 2020 VMware, Inc. diff --git a/deploy/rbac.yaml b/deploy/rbac.yaml index 98caf1c3..75b920d8 100644 --- a/deploy/rbac.yaml +++ b/deploy/rbac.yaml @@ -1,5 +1,6 @@ #@ load("@ytt:data", "data") +#! Give permission to various cluster-scoped objects --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -28,6 +29,8 @@ roleRef: kind: ClusterRole name: #@ data.values.app_name + "-aggregated-api-server-cluster-role" apiGroup: rbac.authorization.k8s.io + +#! Give permission to various objects within the app's own namespace --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -38,6 +41,9 @@ rules: - apiGroups: [""] resources: [services] verbs: [create, get, list, patch, update, watch] + - apiGroups: [""] + resources: [secrets] + verbs: [create, get, list, patch, update, watch] - apiGroups: [crds.placeholder.suzerain-io.github.io] resources: [logindiscoveryconfigs] verbs: [create, get, list, update, watch] @@ -55,6 +61,8 @@ roleRef: kind: Role name: #@ data.values.app_name + "-aggregated-api-server-role" apiGroup: rbac.authorization.k8s.io + +#! Allow both authenticated and unauthenticated LoginRequests (i.e. allow all requests) --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -70,7 +78,6 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: #@ data.values.app_name + "-loginrequests-cluster-role-binding" subjects: - #! both authenticated and unauthenticated requests (i.e. all requests) should be allowed - kind: Group name: system:authenticated apiGroup: rbac.authorization.k8s.io @@ -81,6 +88,8 @@ roleRef: kind: ClusterRole name: #@ data.values.app_name + "-loginrequests-cluster-role" apiGroup: rbac.authorization.k8s.io + +#! Give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -93,9 +102,10 @@ subjects: namespace: #@ data.values.namespace roleRef: kind: ClusterRole - #! give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers name: system:auth-delegator apiGroup: rbac.authorization.k8s.io + +#! Give permissions for a special configmap of CA bundles that is needed by aggregated api servers --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -108,9 +118,10 @@ subjects: namespace: #@ data.values.namespace roleRef: kind: Role - #! give permissions for a special configmap of CA bundles that is needed by aggregated api servers name: extension-apiserver-authentication-reader apiGroup: rbac.authorization.k8s.io + +#! Give permission to list and watch ConfigMaps in kube-public --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 diff --git a/internal/autoregistration/autoregistration.go b/internal/autoregistration/autoregistration.go index a729ed70..7033abe4 100644 --- a/internal/autoregistration/autoregistration.go +++ b/internal/autoregistration/autoregistration.go @@ -12,14 +12,14 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/retry" - aggregatationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" ) // UpdateAPIService updates the APIService's CA bundle. -func UpdateAPIService(ctx context.Context, aggregationV1 aggregatationv1client.Interface, aggregatedAPIServerCA []byte) error { - apiServices := aggregationV1.ApiregistrationV1().APIServices() +func UpdateAPIService(ctx context.Context, aggregatorClient aggregatorclient.Interface, aggregatedAPIServerCA []byte) error { + apiServices := aggregatorClient.ApiregistrationV1().APIServices() apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go new file mode 100644 index 00000000..a6efe81f --- /dev/null +++ b/internal/controller/apicerts/certs_manager.go @@ -0,0 +1,164 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package apicerts + +import ( + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/klog/v2" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + + "github.com/suzerain-io/controller-go" + "github.com/suzerain-io/placeholder-name/internal/autoregistration" + "github.com/suzerain-io/placeholder-name/internal/certauthority" + placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" + placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" +) + +const ( + //nolint: gosec + certsSecretName = "api-serving-cert" + caCertificateSecretKey = "caCertificate" + tlsPrivateKeySecretKey = "tlsPrivateKey" + tlsCertificateChainSecretKey = "tlsCertificateChain" +) + +type certsManagerController struct { + namespace string + apiServiceName string + k8sClient kubernetes.Interface + aggregatorClient *aggregatorclient.Clientset + secretInformer corev1informers.SecretInformer +} + +func NewCertsManagerController( + namespace string, + k8sClient kubernetes.Interface, + aggregationClient *aggregatorclient.Clientset, + secretInformer corev1informers.SecretInformer, + withInformer placeholdernamecontroller.WithInformerOptionFunc, +) controller.Controller { + apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName + return controller.New( + controller.Config{ + Name: "certs-manager-controller", + Syncer: &certsManagerController{ + apiServiceName: apiServiceName, + namespace: namespace, + k8sClient: k8sClient, + aggregatorClient: aggregationClient, + secretInformer: secretInformer, + }, + }, + withInformer( + secretInformer, + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace), + controller.InformerOption{}, + ), + // Be sure to run once even if the Secret that the informer is watching doesn't exist. + controller.WithInitialEvent(controller.Key{ + Namespace: namespace, + Name: certsSecretName, + }), + ) +} + +func (c *certsManagerController) Sync(ctx controller.Context) error { + // Try to get the secret from the informer cache. + _, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err) + } + if !notFound { + // The secret already exists, so nothing to do. + return nil + } + + // Create a CA. + aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"}) + if err != nil { + return fmt.Errorf("could not initialize CA: %w", err) + } + + // This string must match the name of the Service declared in the deployment yaml. + const serviceName = "placeholder-name-api" + + // Using the CA from above, create a TLS server cert for the aggregated API server to use. + aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( + pkix.Name{CommonName: serviceName + "." + c.namespace + ".svc"}, + []string{}, + 24*365*time.Hour, + ) + if err != nil { + return fmt.Errorf("could not issue serving certificate: %w", err) + } + + // Write the CA's public key bundle and the serving certs to a secret. + tlsPrivateKeyPEM, tlsCertChainPEM, err := pemEncode(aggregatedAPIServerTLSCert) + if err != nil { + return fmt.Errorf("could not PEM encode serving certificate: %w", err) + } + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: certsSecretName, + Namespace: c.namespace, + }, + StringData: map[string]string{ + caCertificateSecretKey: string(aggregatedAPIServerCA.Bundle()), + tlsPrivateKeySecretKey: string(tlsPrivateKeyPEM), + tlsCertificateChainSecretKey: string(tlsCertChainPEM), + }, + } + _, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("could not create secret: %w", err) + } + + // Update the APIService to give it the new CA bundle. + if err := autoregistration.UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil { + return fmt.Errorf("could not update the API service: %w", err) + } + + klog.Info("certsManagerController Sync successfully created secret and updated API service") + return nil +} + +// Encode a tls.Certificate into a private key PEM and a cert chain PEM. +func pemEncode(cert *tls.Certificate) ([]byte, []byte, error) { + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) + if err != nil { + return nil, nil, fmt.Errorf("error marshalling private key: %w", err) + } + privateKeyPEM := pem.EncodeToMemory(&pem.Block{ + Type: "PRIVATE KEY", + Headers: nil, + Bytes: privateKeyDER, + }) + + certChainPEM := make([]byte, 0) + for _, certFromChain := range cert.Certificate { + certPEMBytes := pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Headers: nil, + Bytes: certFromChain, + }) + certChainPEM = append(certChainPEM, certPEMBytes...) + } + + return privateKeyPEM, certChainPEM, nil +} diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go new file mode 100644 index 00000000..41595c91 --- /dev/null +++ b/internal/controller/apicerts/certs_observer.go @@ -0,0 +1,69 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package apicerts + +import ( + "fmt" + + k8serrors "k8s.io/apimachinery/pkg/api/errors" + corev1informers "k8s.io/client-go/informers/core/v1" + "k8s.io/klog/v2" + + "github.com/suzerain-io/controller-go" + placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" + "github.com/suzerain-io/placeholder-name/internal/provider" +) + +type certsObserverController struct { + namespace string + dynamicCertProvider *provider.DynamicTLSServingCertProvider + secretInformer corev1informers.SecretInformer +} + +func NewCertsObserverController( + namespace string, + dynamicCertProvider *provider.DynamicTLSServingCertProvider, + secretInformer corev1informers.SecretInformer, + withInformer placeholdernamecontroller.WithInformerOptionFunc, +) controller.Controller { + return controller.New( + controller.Config{ + Name: "certs-observer-controller", + Syncer: &certsObserverController{ + namespace: namespace, + dynamicCertProvider: dynamicCertProvider, + secretInformer: secretInformer, + }, + }, + withInformer( + secretInformer, + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace), + controller.InformerOption{}, + ), + ) +} + +func (c *certsObserverController) Sync(_ controller.Context) error { + // Try to get the secret from the informer cache. + certSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName) + notFound := k8serrors.IsNotFound(err) + if err != nil && !notFound { + return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err) + } + if notFound { + klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted") + // The secret does not exist yet or was deleted. + c.dynamicCertProvider.CertPEM = nil + c.dynamicCertProvider.KeyPEM = nil + return nil + } + + // Mutate the in-memory cert provider to update with the latest cert values. + c.dynamicCertProvider.CertPEM = certSecret.Data[tlsCertificateChainSecretKey] + c.dynamicCertProvider.KeyPEM = certSecret.Data[tlsPrivateKeySecretKey] + klog.Info("certsObserverController Sync updated certs in the dynamic cert provider") + return nil +} diff --git a/internal/controller/logindiscovery/publisher.go b/internal/controller/logindiscovery/publisher.go index 4151a90c..54026c44 100644 --- a/internal/controller/logindiscovery/publisher.go +++ b/internal/controller/logindiscovery/publisher.go @@ -17,6 +17,7 @@ import ( "k8s.io/klog/v2" "github.com/suzerain-io/controller-go" + placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1" placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" crdsplaceholderv1alpha1informers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions/crdsplaceholder/v1alpha1" @@ -31,25 +32,6 @@ const ( configName = "placeholder-name-config" ) -func nameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs { - objMatchesFunc := func(obj metav1.Object) bool { - return obj.GetName() == name && obj.GetNamespace() == namespace - } - return controller.FilterFuncs{ - AddFunc: objMatchesFunc, - UpdateFunc: func(oldObj, newObj metav1.Object) bool { - return objMatchesFunc(oldObj) || objMatchesFunc(newObj) - }, - DeleteFunc: objMatchesFunc, - } -} - -// Same signature as controller.WithInformer(). -type withInformerOptionFunc func( - getter controller.InformerGetter, - filter controller.Filter, - opt controller.InformerOption) controller.Option - type publisherController struct { namespace string serverOverride *string @@ -64,7 +46,7 @@ func NewPublisherController( placeholderClient placeholderclientset.Interface, configMapInformer corev1informers.ConfigMapInformer, loginDiscoveryConfigInformer crdsplaceholderv1alpha1informers.LoginDiscoveryConfigInformer, - withInformer withInformerOptionFunc, + withInformer placeholdernamecontroller.WithInformerOptionFunc, ) controller.Controller { return controller.New( controller.Config{ @@ -79,12 +61,12 @@ func NewPublisherController( }, withInformer( configMapInformer, - nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace), + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace), controller.InformerOption{}, ), withInformer( loginDiscoveryConfigInformer, - nameAndNamespaceExactMatchFilterFactory(configName, namespace), + placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(configName, namespace), controller.InformerOption{}, ), ) diff --git a/internal/controller/utils.go b/internal/controller/utils.go new file mode 100644 index 00000000..1b47be1d --- /dev/null +++ b/internal/controller/utils.go @@ -0,0 +1,31 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package controller + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/suzerain-io/controller-go" +) + +func NameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs { + objMatchesFunc := func(obj metav1.Object) bool { + return obj.GetName() == name && obj.GetNamespace() == namespace + } + return controller.FilterFuncs{ + AddFunc: objMatchesFunc, + UpdateFunc: func(oldObj, newObj metav1.Object) bool { + return objMatchesFunc(oldObj) || objMatchesFunc(newObj) + }, + DeleteFunc: objMatchesFunc, + } +} + +// Same signature as controller.WithInformer(). +type WithInformerOptionFunc func( + getter controller.InformerGetter, + filter controller.Filter, + opt controller.InformerOption) controller.Option diff --git a/internal/controller/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go similarity index 56% rename from internal/controller/prepare_controllers.go rename to internal/controllermanager/prepare_controllers.go index 96d41ac5..8ab8eefa 100644 --- a/internal/controller/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -3,7 +3,7 @@ Copyright 2020 VMware, Inc. SPDX-License-Identifier: Apache-2.0 */ -package controller +package controllermanager import ( "context" @@ -14,11 +14,12 @@ import ( k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" restclient "k8s.io/client-go/rest" - aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "github.com/suzerain-io/controller-go" - "github.com/suzerain-io/placeholder-name/internal/autoregistration" + "github.com/suzerain-io/placeholder-name/internal/controller/apicerts" "github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery" + "github.com/suzerain-io/placeholder-name/internal/provider" placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions" ) @@ -30,25 +31,19 @@ const ( // Prepare the controllers and their informers and return a function that will start them when called. func PrepareControllers( - ctx context.Context, - caBundle []byte, serverInstallationNamespace string, discoveryURLOverride *string, + dynamicCertProvider *provider.DynamicTLSServingCertProvider, ) (func(ctx context.Context), error) { // Create k8s clients. - k8sClient, aggregationClient, placeholderClient, err := createClients() + k8sClient, aggregatorClient, placeholderClient, err := createClients() if err != nil { return nil, fmt.Errorf("could not create clients for the controllers: %w", err) } - // TODO Putting this here temporarily on the way toward moving it elsewhere. - // When it moves elsewhere then PrepareControllers() will not need to take ctx and caBundle parameters. - if err := autoregistration.UpdateAPIService(ctx, aggregationClient, caBundle); err != nil { - return nil, fmt.Errorf("could not update the API service: %w", err) - } - - // Create informers. - k8sInformers, placeholderInformers := createInformers(serverInstallationNamespace, k8sClient, placeholderClient) + // Create informers. Don't forget to make sure they get started in the function returned below. + kubePublicNamespaceK8sInformers, installationNamespaceK8sInformers, installationNamespacePlaceholderInformers := + createInformers(serverInstallationNamespace, k8sClient, placeholderClient) // Create controller manager. controllerManager := controller. @@ -58,8 +53,27 @@ func PrepareControllers( serverInstallationNamespace, discoveryURLOverride, placeholderClient, - k8sInformers.Core().V1().ConfigMaps(), - placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(), + kubePublicNamespaceK8sInformers.Core().V1().ConfigMaps(), + installationNamespacePlaceholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(), + controller.WithInformer, + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsManagerController( + serverInstallationNamespace, + k8sClient, + aggregatorClient, + installationNamespaceK8sInformers.Core().V1().Secrets(), + controller.WithInformer, + ), + singletonWorker, + ). + WithController( + apicerts.NewCertsObserverController( + serverInstallationNamespace, + dynamicCertProvider, + installationNamespaceK8sInformers.Core().V1().Secrets(), controller.WithInformer, ), singletonWorker, @@ -67,14 +81,21 @@ func PrepareControllers( // Return a function which starts the informers and controllers. return func(ctx context.Context) { - k8sInformers.Start(ctx.Done()) - placeholderInformers.Start(ctx.Done()) + kubePublicNamespaceK8sInformers.Start(ctx.Done()) + installationNamespaceK8sInformers.Start(ctx.Done()) + installationNamespacePlaceholderInformers.Start(ctx.Done()) + go controllerManager.Start(ctx) }, nil } // Create the k8s clients that will be used by the controllers. -func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *placeholderclientset.Clientset, error) { +func createClients() ( + k8sClient *kubernetes.Clientset, + aggregatorClient *aggregatorclient.Clientset, + placeholderClient *placeholderclientset.Clientset, + err error, +) { // Load the Kubernetes client configuration (kubeconfig), kubeConfig, err := restclient.InClusterConfig() if err != nil { @@ -85,13 +106,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl protoKubeConfig := createProtoKubeConfig(kubeConfig) // Connect to the core Kubernetes API. - k8sClient, err := kubernetes.NewForConfig(protoKubeConfig) + k8sClient, err = kubernetes.NewForConfig(protoKubeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) } // Connect to the Kubernetes aggregation API. - aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig) + aggregatorClient, err = aggregatorclient.NewForConfig(protoKubeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err) } @@ -99,12 +120,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl // Connect to the placeholder API. // I think we can't use protobuf encoding here because we are using CRDs // (for which protobuf encoding is not supported). - placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig) + placeholderClient, err = placeholderclientset.NewForConfig(kubeConfig) if err != nil { return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err) } - return k8sClient, aggregationClient, placeholderClient, nil + //nolint: nakedret + return } // Create the informers that will be used by the controllers. @@ -112,20 +134,27 @@ func createInformers( serverInstallationNamespace string, k8sClient *kubernetes.Clientset, placeholderClient *placeholderclientset.Clientset, -) (k8sinformers.SharedInformerFactory, placeholderinformers.SharedInformerFactory) { - k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions( +) ( + kubePublicNamespaceK8sInformers k8sinformers.SharedInformerFactory, + installationNamespaceK8sInformers k8sinformers.SharedInformerFactory, + installationNamespacePlaceholderInformers placeholderinformers.SharedInformerFactory, +) { + kubePublicNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions( k8sClient, defaultResyncInterval, - k8sinformers.WithNamespace( - logindiscovery.ClusterInfoNamespace, - ), + k8sinformers.WithNamespace(logindiscovery.ClusterInfoNamespace), ) - placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions( + installationNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions( + k8sClient, + defaultResyncInterval, + k8sinformers.WithNamespace(serverInstallationNamespace), + ) + installationNamespacePlaceholderInformers = placeholderinformers.NewSharedInformerFactoryWithOptions( placeholderClient, defaultResyncInterval, placeholderinformers.WithNamespace(serverInstallationNamespace), ) - return k8sInformers, placeholderInformers + return } // Returns a copy of the input config with the ContentConfig set to use protobuf. diff --git a/internal/provider/dynamic_tls_serving_cert_provider.go b/internal/provider/dynamic_tls_serving_cert_provider.go new file mode 100644 index 00000000..7807802e --- /dev/null +++ b/internal/provider/dynamic_tls_serving_cert_provider.go @@ -0,0 +1,19 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package provider + +type DynamicTLSServingCertProvider struct { + CertPEM []byte + KeyPEM []byte +} + +func (*DynamicTLSServingCertProvider) Name() string { + return "DynamicTLSServingCertProvider" +} + +func (p *DynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) { + return p.CertPEM, p.KeyPEM +} diff --git a/internal/server/server.go b/internal/server/server.go index 6ec828ce..68cf2a1e 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -8,24 +8,19 @@ package server import ( "context" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" "fmt" "io" - "time" "github.com/spf13/cobra" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/dynamiccertificates" genericoptions "k8s.io/apiserver/pkg/server/options" "k8s.io/apiserver/plugin/pkg/authenticator/token/webhook" "github.com/suzerain-io/placeholder-name/internal/apiserver" "github.com/suzerain-io/placeholder-name/internal/certauthority" - "github.com/suzerain-io/placeholder-name/internal/controller" + "github.com/suzerain-io/placeholder-name/internal/controllermanager" "github.com/suzerain-io/placeholder-name/internal/downward" + "github.com/suzerain-io/placeholder-name/internal/provider" placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" "github.com/suzerain-io/placeholder-name/pkg/config" ) @@ -135,31 +130,19 @@ func (app *App) runServer(ctx context.Context) error { } serverInstallationNamespace := podInfo.Namespace - // Create a CA. - aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"}) - if err != nil { - return fmt.Errorf("could not initialize CA: %w", err) - } - - // This string must match the name of the Service declared in the deployment yaml. - const serviceName = "placeholder-name-api" - // Using the CA from above, create a TLS server cert for the aggregated API server to use. - aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue( - pkix.Name{CommonName: serviceName + "." + serverInstallationNamespace + ".svc"}, - []string{}, - 24*365*time.Hour, - ) - if err != nil { - return fmt.Errorf("could not issue serving certificate: %w", err) - } + // This cert provider will provide certs to the API server and will + // be mutated by a controller to keep the certs up to date with what + // is stored in a k8s Secret. Therefore it also effectively acting as + // an in-memory cache of what is stored in the k8s Secret, helping to + // keep incoming requests fast. + dynamicCertProvider := &provider.DynamicTLSServingCertProvider{} // Prepare to start the controllers, but defer actually starting them until the // post start hook of the aggregated API server. - startControllersFunc, err := controller.PrepareControllers( - ctx, - aggregatedAPIServerCA.Bundle(), + startControllersFunc, err := controllermanager.PrepareControllers( serverInstallationNamespace, cfg.DiscoveryConfig.URL, + dynamicCertProvider, ) if err != nil { return fmt.Errorf("could not prepare controllers: %w", err) @@ -167,7 +150,7 @@ func (app *App) runServer(ctx context.Context) error { // Get the aggregated API server config. aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig( - aggregatedAPIServerTLSCert, + dynamicCertProvider, webhookTokenAuthenticator, k8sClusterCA, startControllersFunc, @@ -188,25 +171,29 @@ func (app *App) runServer(ctx context.Context) error { // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( - cert *tls.Certificate, + dynamicCertProvider *provider.DynamicTLSServingCertProvider, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, ca *certauthority.CA, startControllersPostStartHook func(context.Context), ) (*apiserver.Config, error) { - provider, err := createStaticCertKeyProvider(cert) - if err != nil { - return nil, fmt.Errorf("could not create static cert key provider: %w", err) - } - recommendedOptions := genericoptions.NewRecommendedOptions( defaultEtcdPathPrefix, apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion), // TODO we should check to see if all the other default settings are acceptable for us ) recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet - recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider + recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs) + // Note that among other things, this ApplyTo() function copies + // `recommendedOptions.SecureServing.ServerCert.GeneratedCert` into + // `serverConfig.SecureServing.Cert` thus making `dynamicCertProvider` + // the cert provider for the running server. The provider will be called + // by the API machinery periodically. When the provider returns nil certs, + // the API server will return "the server is currently unable to + // handle the request" error responses for all incoming requests. + // If the provider later starts returning certs, then the API server + // will use them to handle the incoming requests successfully. if err := recommendedOptions.ApplyTo(serverConfig); err != nil { return nil, err } @@ -221,27 +208,3 @@ func getAggregatedAPIServerConfig( } return apiServerConfig, nil } - -func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.CertKeyContentProvider, error) { - privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) - if err != nil { - return nil, fmt.Errorf("error marshalling private key: %w", err) - } - privateKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "PRIVATE KEY", - Headers: nil, - Bytes: privateKeyDER, - }) - - certChainPEM := make([]byte, 0) - for _, certFromChain := range cert.Certificate { - certPEMBytes := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Headers: nil, - Bytes: certFromChain, - }) - certChainPEM = append(certChainPEM, certPEMBytes...) - } - - return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM) -} From cc9ae23a0c24f9fc4e3faeb4e40abfe2c7a44579 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 10 Aug 2020 18:53:53 -0700 Subject: [PATCH 3/6] Add tests for the new cert controllers and some other small refactorings - Add a unit test for each cert controller - Make DynamicTLSServingCertProvider an interface and use a mutex internally - Create a shared ToPEM function instead of having two very similar functions - Move the ObservableWithInformerOption test helper to testutils - Rename some variables and imports --- .../autoregistration/autoregistration_test.go | 16 +- internal/certauthority/certauthority.go | 10 + internal/controller/apicerts/certs_manager.go | 40 +-- .../controller/apicerts/certs_manager_test.go | 333 ++++++++++++++++++ .../controller/apicerts/certs_observer.go | 10 +- .../apicerts/certs_observer_test.go | 232 ++++++++++++ .../logindiscovery/publisher_test.go | 27 +- .../controllermanager/prepare_controllers.go | 2 +- .../dynamic_tls_serving_cert_provider.go | 37 +- internal/server/server.go | 26 +- internal/server/server_test.go | 2 +- .../observable_with_informer_option.go | 30 ++ 12 files changed, 672 insertions(+), 93 deletions(-) create mode 100644 internal/controller/apicerts/certs_manager_test.go create mode 100644 internal/controller/apicerts/certs_observer_test.go create mode 100644 internal/testutil/observable_with_informer_option.go diff --git a/internal/autoregistration/autoregistration_test.go b/internal/autoregistration/autoregistration_test.go index 7ea4a200..6ac4e731 100644 --- a/internal/autoregistration/autoregistration_test.go +++ b/internal/autoregistration/autoregistration_test.go @@ -17,7 +17,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" kubetesting "k8s.io/client-go/testing" apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" - aggregationv1fake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake" + aggregatorv1fake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake" ) func TestUpdateAPIService(t *testing.T) { @@ -25,14 +25,14 @@ func TestUpdateAPIService(t *testing.T) { tests := []struct { name string - mocks func(*aggregationv1fake.Clientset) + mocks func(*aggregatorv1fake.Clientset) caInput []byte wantObjects []apiregistrationv1.APIService wantErr string }{ { name: "happy path update when the pre-existing APIService did not already have a CA bundle", - mocks: func(c *aggregationv1fake.Clientset) { + mocks: func(c *aggregatorv1fake.Clientset) { _ = c.Tracker().Add(&apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistrationv1.APIServiceSpec{ @@ -52,7 +52,7 @@ func TestUpdateAPIService(t *testing.T) { }, { name: "happy path update when the pre-existing APIService already had a CA bundle", - mocks: func(c *aggregationv1fake.Clientset) { + mocks: func(c *aggregatorv1fake.Clientset) { _ = c.Tracker().Add(&apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistrationv1.APIServiceSpec{ @@ -72,7 +72,7 @@ func TestUpdateAPIService(t *testing.T) { }, { name: "error on update", - mocks: func(c *aggregationv1fake.Clientset) { + mocks: func(c *aggregatorv1fake.Clientset) { _ = c.Tracker().Add(&apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistrationv1.APIServiceSpec{}, @@ -85,7 +85,7 @@ func TestUpdateAPIService(t *testing.T) { }, { name: "error on get", - mocks: func(c *aggregationv1fake.Clientset) { + mocks: func(c *aggregatorv1fake.Clientset) { _ = c.Tracker().Add(&apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistrationv1.APIServiceSpec{}, @@ -99,7 +99,7 @@ func TestUpdateAPIService(t *testing.T) { }, { name: "conflict error on update, followed by successful retry", - mocks: func(c *aggregationv1fake.Clientset) { + mocks: func(c *aggregatorv1fake.Clientset) { _ = c.Tracker().Add(&apiregistrationv1.APIService{ ObjectMeta: metav1.ObjectMeta{Name: apiServiceName}, Spec: apiregistrationv1.APIServiceSpec{ @@ -148,7 +148,7 @@ func TestUpdateAPIService(t *testing.T) { t.Run(tt.name, func(t *testing.T) { ctx := context.Background() - client := aggregationv1fake.NewSimpleClientset() + client := aggregatorv1fake.NewSimpleClientset() if tt.mocks != nil { tt.mocks(client) } diff --git a/internal/certauthority/certauthority.go b/internal/certauthority/certauthority.go index a95c67b2..5e6249ad 100644 --- a/internal/certauthority/certauthority.go +++ b/internal/certauthority/certauthority.go @@ -195,6 +195,16 @@ func toPEM(cert *tls.Certificate, err error) ([]byte, []byte, error) { return nil, nil, err } + certPEM, keyPEM, err := ToPEM(cert) + if err != nil { + return nil, nil, err + } + + return certPEM, keyPEM, nil +} + +// Encode a tls.Certificate into a private key PEM and a cert chain PEM. +func ToPEM(cert *tls.Certificate) ([]byte, []byte, error) { // Encode the certificate(s) to PEM. certPEMBlocks := make([][]byte, 0, len(cert.Certificate)) for _, c := range cert.Certificate { diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index a6efe81f..5c499e0c 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -6,10 +6,7 @@ SPDX-License-Identifier: Apache-2.0 package apicerts import ( - "crypto/tls" - "crypto/x509" "crypto/x509/pkix" - "encoding/pem" "fmt" "time" @@ -25,7 +22,6 @@ import ( "github.com/suzerain-io/placeholder-name/internal/autoregistration" "github.com/suzerain-io/placeholder-name/internal/certauthority" placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" - placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" ) const ( @@ -38,28 +34,25 @@ const ( type certsManagerController struct { namespace string - apiServiceName string k8sClient kubernetes.Interface - aggregatorClient *aggregatorclient.Clientset + aggregatorClient aggregatorclient.Interface secretInformer corev1informers.SecretInformer } func NewCertsManagerController( namespace string, k8sClient kubernetes.Interface, - aggregationClient *aggregatorclient.Clientset, + aggregatorClient aggregatorclient.Interface, secretInformer corev1informers.SecretInformer, withInformer placeholdernamecontroller.WithInformerOptionFunc, ) controller.Controller { - apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName return controller.New( controller.Config{ Name: "certs-manager-controller", Syncer: &certsManagerController{ - apiServiceName: apiServiceName, namespace: namespace, k8sClient: k8sClient, - aggregatorClient: aggregationClient, + aggregatorClient: aggregatorClient, secretInformer: secretInformer, }, }, @@ -108,7 +101,7 @@ func (c *certsManagerController) Sync(ctx controller.Context) error { } // Write the CA's public key bundle and the serving certs to a secret. - tlsPrivateKeyPEM, tlsCertChainPEM, err := pemEncode(aggregatedAPIServerTLSCert) + tlsPrivateKeyPEM, tlsCertChainPEM, err := certauthority.ToPEM(aggregatedAPIServerTLSCert) if err != nil { return fmt.Errorf("could not PEM encode serving certificate: %w", err) } @@ -137,28 +130,3 @@ func (c *certsManagerController) Sync(ctx controller.Context) error { klog.Info("certsManagerController Sync successfully created secret and updated API service") return nil } - -// Encode a tls.Certificate into a private key PEM and a cert chain PEM. -func pemEncode(cert *tls.Certificate) ([]byte, []byte, error) { - privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey) - if err != nil { - return nil, nil, fmt.Errorf("error marshalling private key: %w", err) - } - privateKeyPEM := pem.EncodeToMemory(&pem.Block{ - Type: "PRIVATE KEY", - Headers: nil, - Bytes: privateKeyDER, - }) - - certChainPEM := make([]byte, 0) - for _, certFromChain := range cert.Certificate { - certPEMBytes := pem.EncodeToMemory(&pem.Block{ - Type: "CERTIFICATE", - Headers: nil, - Bytes: certFromChain, - }) - certChainPEM = append(certChainPEM, certPEMBytes...) - } - - return privateKeyPEM, certChainPEM, nil -} diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go new file mode 100644 index 00000000..a600b5d9 --- /dev/null +++ b/internal/controller/apicerts/certs_manager_test.go @@ -0,0 +1,333 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package apicerts + +import ( + "context" + "crypto/x509" + "encoding/pem" + "errors" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + kubeinformers "k8s.io/client-go/informers" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + coretesting "k8s.io/client-go/testing" + apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1" + aggregatorfake "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/fake" + + "github.com/suzerain-io/controller-go" + "github.com/suzerain-io/placeholder-name/internal/testutil" + placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" +) + +func TestManagerControllerInformerFilters(t *testing.T) { + spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + + var r *require.Assertions + var observableWithInformerOption *testutil.ObservableWithInformerOption + var secretsInformerFilter controller.Filter + + it.Before(func() { + r = require.New(t) + observableWithInformerOption = testutil.NewObservableWithInformerOption() + secretsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Secrets() + _ = NewCertsManagerController( + installedInNamespace, + nil, + nil, + secretsInformer, + observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters + ) + secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) + }) + + when("watching Secret objects", func() { + var subject controller.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.Secret + + it.Before(func() { + subject = secretsInformerFilter + target = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: installedInNamespace}} + wrongNamespace = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: "wrong-namespace"}} + wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target Secret changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a Secret from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a Secret with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a Secret with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} + +func TestManagerControllerSync(t *testing.T) { + spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + + var r *require.Assertions + + var subject controller.Controller + var kubeAPIClient *kubernetesfake.Clientset + var aggregatorAPIClient *aggregatorfake.Clientset + var kubeInformerClient *kubernetesfake.Clientset + var kubeInformers kubeinformers.SharedInformerFactory + var timeoutContext context.Context + var timeoutContextCancel context.CancelFunc + var syncContext *controller.Context + + // Defer starting the informers until the last possible moment so that the + // nested Before's can keep adding things to the informer caches. + var startInformersAndController = func() { + // Set this at the last second to allow for injection of server override. + subject = NewCertsManagerController( + installedInNamespace, + kubeAPIClient, + aggregatorAPIClient, + kubeInformers.Core().V1().Secrets(), + controller.WithInformer, + ) + + // Set this at the last second to support calling subject.Name(). + syncContext = &controller.Context{ + Context: timeoutContext, + Name: subject.Name(), + Key: controller.Key{ + Namespace: installedInNamespace, + Name: "api-serving-cert", + }, + } + + // Must start informers before calling TestRunSynchronously() + kubeInformers.Start(timeoutContext.Done()) + controller.TestRunSynchronously(t, subject) + } + + it.Before(func() { + r = require.New(t) + + timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + + kubeInformerClient = kubernetesfake.NewSimpleClientset() + kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) + kubeAPIClient = kubernetesfake.NewSimpleClientset() + aggregatorAPIClient = aggregatorfake.NewSimpleClientset() + }) + + it.After(func() { + timeoutContextCancel() + }) + + when("there is not yet an api-serving-cert Secret in the installation namespace or it was deleted", func() { + it.Before(func() { + unrelatedSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some other secret", + Namespace: installedInNamespace, + }, + } + err := kubeInformerClient.Tracker().Add(unrelatedSecret) + r.NoError(err) + }) + + when("the APIService exists", func() { + it.Before(func() { + apiService := &apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName, + }, + Spec: apiregistrationv1.APIServiceSpec{ + CABundle: nil, + VersionPriority: 1234, + }, + } + err := aggregatorAPIClient.Tracker().Add(apiService) + r.NoError(err) + }) + + it("creates the api-serving-cert Secret and updates the APIService's ca bundle", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.NoError(err) + + // Check all the relevant fields from the create Secret action + r.Len(kubeAPIClient.Actions(), 1) + actualAction := kubeAPIClient.Actions()[0].(coretesting.CreateActionImpl) + r.Equal(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, actualAction.GetResource()) + r.Equal(installedInNamespace, actualAction.GetNamespace()) + actualSecret := actualAction.GetObject().(*corev1.Secret) + r.Equal("api-serving-cert", actualSecret.Name) + r.Equal(installedInNamespace, actualSecret.Namespace) + actualCACert := actualSecret.StringData["caCertificate"] + actualPrivateKey := actualSecret.StringData["tlsPrivateKey"] + actualCertChain := actualSecret.StringData["tlsCertificateChain"] + r.NotEmpty(actualCACert) + r.NotEmpty(actualPrivateKey) + r.NotEmpty(actualCertChain) + + // Validate the created cert using the CA, and also validate the cert's hostname + roots := x509.NewCertPool() + ok := roots.AppendCertsFromPEM([]byte(actualCACert)) + r.True(ok) + block, _ := pem.Decode([]byte(actualPrivateKey)) + r.NotNil(block) + parsedCert, err := x509.ParseCertificate(block.Bytes) + r.NoError(err) + opts := x509.VerifyOptions{ + DNSName: "placeholder-name-api." + installedInNamespace + ".svc", + Roots: roots, + } + _, err = parsedCert.Verify(opts) + r.NoError(err) + + // Check the created cert's validity bounds + r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2) + r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2) + + // TODO How can we validate the tlsCertificateChain? + + // Make sure we updated the APIService caBundle and left it otherwise unchanged + r.Len(aggregatorAPIClient.Actions(), 2) + r.Equal("get", aggregatorAPIClient.Actions()[0].GetVerb()) + expectedAPIServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName + expectedUpdateAction := coretesting.NewUpdateAction( + schema.GroupVersionResource{ + Group: apiregistrationv1.GroupName, + Version: "v1", + Resource: "apiservices", + }, + "", + &apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{ + Name: expectedAPIServiceName, + Namespace: "", + }, + Spec: apiregistrationv1.APIServiceSpec{ + VersionPriority: 1234, // only the CABundle is updated, this other field is left unchanged + CABundle: []byte(actualCACert), + }, + }, + ) + r.Equal(expectedUpdateAction, aggregatorAPIClient.Actions()[1]) + }) + + when("updating the APIService fails", func() { + it.Before(func() { + aggregatorAPIClient.PrependReactor( + "update", + "apiservices", + func(_ coretesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("update failed") + }, + ) + }) + + it("returns the update error", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.EqualError(err, "could not update the API service: could not update API service: update failed") + }) + }) + }) + + when("the APIService does not exist", func() { + it.Before(func() { + unrelatedAPIService := &apiregistrationv1.APIService{ + ObjectMeta: metav1.ObjectMeta{Name: "some other api service"}, + Spec: apiregistrationv1.APIServiceSpec{}, + } + err := aggregatorAPIClient.Tracker().Add(unrelatedAPIService) + r.NoError(err) + }) + + it("returns an error", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.Error(err) + r.Regexp("could not get existing version of API service: .* not found", err.Error()) + }) + }) + + when("creating the Secret fails", func() { + it.Before(func() { + kubeAPIClient.PrependReactor( + "create", + "secrets", + func(_ coretesting.Action) (bool, runtime.Object, error) { + return true, nil, errors.New("create failed") + }, + ) + }) + + it("returns the create error", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.EqualError(err, "could not create secret: create failed") + }) + }) + }) + + when("there is an api-serving-cert Secret already in the installation namespace", func() { + it.Before(func() { + apiServingCertSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "api-serving-cert", + Namespace: installedInNamespace, + }, + } + err := kubeInformerClient.Tracker().Add(apiServingCertSecret) + r.NoError(err) + }) + + it("does not need to make any API calls with its API clients", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.NoError(err) + r.Empty(kubeAPIClient.Actions()) + r.Empty(aggregatorAPIClient.Actions()) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} diff --git a/internal/controller/apicerts/certs_observer.go b/internal/controller/apicerts/certs_observer.go index 41595c91..b574ef06 100644 --- a/internal/controller/apicerts/certs_observer.go +++ b/internal/controller/apicerts/certs_observer.go @@ -19,13 +19,13 @@ import ( type certsObserverController struct { namespace string - dynamicCertProvider *provider.DynamicTLSServingCertProvider + dynamicCertProvider provider.DynamicTLSServingCertProvider secretInformer corev1informers.SecretInformer } func NewCertsObserverController( namespace string, - dynamicCertProvider *provider.DynamicTLSServingCertProvider, + dynamicCertProvider provider.DynamicTLSServingCertProvider, secretInformer corev1informers.SecretInformer, withInformer placeholdernamecontroller.WithInformerOptionFunc, ) controller.Controller { @@ -56,14 +56,12 @@ func (c *certsObserverController) Sync(_ controller.Context) error { if notFound { klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted") // The secret does not exist yet or was deleted. - c.dynamicCertProvider.CertPEM = nil - c.dynamicCertProvider.KeyPEM = nil + c.dynamicCertProvider.Set(nil, nil) return nil } // Mutate the in-memory cert provider to update with the latest cert values. - c.dynamicCertProvider.CertPEM = certSecret.Data[tlsCertificateChainSecretKey] - c.dynamicCertProvider.KeyPEM = certSecret.Data[tlsPrivateKeySecretKey] + c.dynamicCertProvider.Set(certSecret.Data[tlsCertificateChainSecretKey], certSecret.Data[tlsPrivateKeySecretKey]) klog.Info("certsObserverController Sync updated certs in the dynamic cert provider") return nil } diff --git a/internal/controller/apicerts/certs_observer_test.go b/internal/controller/apicerts/certs_observer_test.go new file mode 100644 index 00000000..c979830b --- /dev/null +++ b/internal/controller/apicerts/certs_observer_test.go @@ -0,0 +1,232 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package apicerts + +import ( + "context" + "testing" + "time" + + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeinformers "k8s.io/client-go/informers" + kubernetesfake "k8s.io/client-go/kubernetes/fake" + + "github.com/suzerain-io/controller-go" + "github.com/suzerain-io/placeholder-name/internal/provider" + "github.com/suzerain-io/placeholder-name/internal/testutil" +) + +func TestObserverControllerInformerFilters(t *testing.T) { + spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + + var r *require.Assertions + var observableWithInformerOption *testutil.ObservableWithInformerOption + var secretsInformerFilter controller.Filter + + it.Before(func() { + r = require.New(t) + observableWithInformerOption = testutil.NewObservableWithInformerOption() + secretsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Secrets() + _ = NewCertsObserverController( + installedInNamespace, + nil, + secretsInformer, + observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters + ) + secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) + }) + + when("watching Secret objects", func() { + var subject controller.Filter + var target, wrongNamespace, wrongName, unrelated *corev1.Secret + + it.Before(func() { + subject = secretsInformerFilter + target = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: installedInNamespace}} + wrongNamespace = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api-serving-cert", Namespace: "wrong-namespace"}} + wrongName = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: installedInNamespace}} + unrelated = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "wrong-name", Namespace: "wrong-namespace"}} + }) + + when("the target Secret changes", func() { + it("returns true to trigger the sync method", func() { + r.True(subject.Add(target)) + r.True(subject.Update(target, unrelated)) + r.True(subject.Update(unrelated, target)) + r.True(subject.Delete(target)) + }) + }) + + when("a Secret from another namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongNamespace)) + r.False(subject.Update(wrongNamespace, unrelated)) + r.False(subject.Update(unrelated, wrongNamespace)) + r.False(subject.Delete(wrongNamespace)) + }) + }) + + when("a Secret with a different name changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(wrongName)) + r.False(subject.Update(wrongName, unrelated)) + r.False(subject.Update(unrelated, wrongName)) + r.False(subject.Delete(wrongName)) + }) + }) + + when("a Secret with a different name and a different namespace changes", func() { + it("returns false to avoid triggering the sync method", func() { + r.False(subject.Add(unrelated)) + r.False(subject.Update(unrelated, unrelated)) + r.False(subject.Delete(unrelated)) + }) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} + +func TestObserverControllerSync(t *testing.T) { + spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) { + const installedInNamespace = "some-namespace" + + var r *require.Assertions + + var subject controller.Controller + var kubeInformerClient *kubernetesfake.Clientset + var kubeInformers kubeinformers.SharedInformerFactory + var timeoutContext context.Context + var timeoutContextCancel context.CancelFunc + var syncContext *controller.Context + var dynamicCertProvider provider.DynamicTLSServingCertProvider + + // Defer starting the informers until the last possible moment so that the + // nested Before's can keep adding things to the informer caches. + var startInformersAndController = func() { + // Set this at the last second to allow for injection of server override. + subject = NewCertsObserverController( + installedInNamespace, + dynamicCertProvider, + kubeInformers.Core().V1().Secrets(), + controller.WithInformer, + ) + + // Set this at the last second to support calling subject.Name(). + syncContext = &controller.Context{ + Context: timeoutContext, + Name: subject.Name(), + Key: controller.Key{ + Namespace: installedInNamespace, + Name: "api-serving-cert", + }, + } + + // Must start informers before calling TestRunSynchronously() + kubeInformers.Start(timeoutContext.Done()) + controller.TestRunSynchronously(t, subject) + } + + it.Before(func() { + r = require.New(t) + + timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + + kubeInformerClient = kubernetesfake.NewSimpleClientset() + kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) + dynamicCertProvider = provider.NewDynamicTLSServingCertProvider() + }) + + it.After(func() { + timeoutContextCancel() + }) + + when("there is not yet an api-serving-cert Secret in the installation namespace or it was deleted", func() { + it.Before(func() { + unrelatedSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some other secret", + Namespace: installedInNamespace, + }, + } + err := kubeInformerClient.Tracker().Add(unrelatedSecret) + r.NoError(err) + + dynamicCertProvider.Set([]byte("some cert"), []byte("some private key")) + }) + + it("sets the dynamicCertProvider's cert and key to nil", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.NoError(err) + + actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() + r.Nil(actualCertChain) + r.Nil(actualKey) + }) + }) + + when("there is an api-serving-cert Secret with the expected keys already in the installation namespace", func() { + it.Before(func() { + apiServingCertSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "api-serving-cert", + Namespace: installedInNamespace, + }, + Data: map[string][]byte{ + "caCertificate": []byte("fake cert"), + "tlsPrivateKey": []byte("fake private key"), + "tlsCertificateChain": []byte("fake cert chain"), + }, + } + err := kubeInformerClient.Tracker().Add(apiServingCertSecret) + r.NoError(err) + + dynamicCertProvider.Set(nil, nil) + }) + + it("updates the dynamicCertProvider's cert and key", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.NoError(err) + + actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() + r.Equal("fake cert chain", string(actualCertChain)) + r.Equal("fake private key", string(actualKey)) + }) + }) + + when("the api-serving-cert Secret exists but is missing the expected keys", func() { + it.Before(func() { + apiServingCertSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "api-serving-cert", + Namespace: installedInNamespace, + }, + Data: map[string][]byte{}, + } + err := kubeInformerClient.Tracker().Add(apiServingCertSecret) + r.NoError(err) + + dynamicCertProvider.Set(nil, nil) + }) + + it("set the missing values in the dynamicCertProvider as nil", func() { + startInformersAndController() + err := controller.TestSync(t, subject, *syncContext) + r.NoError(err) + + actualCertChain, actualKey := dynamicCertProvider.CurrentCertKeyContent() + r.Nil(actualCertChain) + r.Nil(actualKey) + }) + }) + }, spec.Parallel(), spec.Report(report.Terminal{})) +} diff --git a/internal/controller/logindiscovery/publisher_test.go b/internal/controller/logindiscovery/publisher_test.go index adb6a07a..0672cfad 100644 --- a/internal/controller/logindiscovery/publisher_test.go +++ b/internal/controller/logindiscovery/publisher_test.go @@ -24,41 +24,24 @@ import ( coretesting "k8s.io/client-go/testing" "github.com/suzerain-io/controller-go" + "github.com/suzerain-io/placeholder-name/internal/testutil" crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1" placeholderfake "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned/fake" placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions" ) -type ObservableWithInformerOption struct { - InformerToFilterMap map[controller.InformerGetter]controller.Filter -} - -func NewObservableWithInformerOption() *ObservableWithInformerOption { - return &ObservableWithInformerOption{ - InformerToFilterMap: make(map[controller.InformerGetter]controller.Filter), - } -} - -func (owi *ObservableWithInformerOption) WithInformer( - getter controller.InformerGetter, - filter controller.Filter, - opt controller.InformerOption) controller.Option { - owi.InformerToFilterMap[getter] = filter - return controller.WithInformer(getter, filter, opt) -} - func TestInformerFilters(t *testing.T) { spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" var r *require.Assertions - var observableWithInformerOption *ObservableWithInformerOption + var observableWithInformerOption *testutil.ObservableWithInformerOption var configMapInformerFilter controller.Filter var loginDiscoveryConfigInformerFilter controller.Filter it.Before(func() { r = require.New(t) - observableWithInformerOption = NewObservableWithInformerOption() + observableWithInformerOption = testutil.NewObservableWithInformerOption() configMapInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().ConfigMaps() loginDiscoveryConfigInformer := placeholderinformers.NewSharedInformerFactory(nil, 0).Crds().V1alpha1().LoginDiscoveryConfigs() _ = NewPublisherController( @@ -69,8 +52,8 @@ func TestInformerFilters(t *testing.T) { loginDiscoveryConfigInformer, observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters ) - configMapInformerFilter = observableWithInformerOption.InformerToFilterMap[configMapInformer] - loginDiscoveryConfigInformerFilter = observableWithInformerOption.InformerToFilterMap[loginDiscoveryConfigInformer] + configMapInformerFilter = observableWithInformerOption.GetFilterForInformer(configMapInformer) + loginDiscoveryConfigInformerFilter = observableWithInformerOption.GetFilterForInformer(loginDiscoveryConfigInformer) }) when("watching ConfigMap objects", func() { diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 8ab8eefa..78b3caf1 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -33,7 +33,7 @@ const ( func PrepareControllers( serverInstallationNamespace string, discoveryURLOverride *string, - dynamicCertProvider *provider.DynamicTLSServingCertProvider, + dynamicCertProvider provider.DynamicTLSServingCertProvider, ) (func(ctx context.Context), error) { // Create k8s clients. k8sClient, aggregatorClient, placeholderClient, err := createClients() diff --git a/internal/provider/dynamic_tls_serving_cert_provider.go b/internal/provider/dynamic_tls_serving_cert_provider.go index 7807802e..55d8f5d9 100644 --- a/internal/provider/dynamic_tls_serving_cert_provider.go +++ b/internal/provider/dynamic_tls_serving_cert_provider.go @@ -5,15 +5,40 @@ SPDX-License-Identifier: Apache-2.0 package provider -type DynamicTLSServingCertProvider struct { - CertPEM []byte - KeyPEM []byte +import ( + "sync" + + "k8s.io/apiserver/pkg/server/dynamiccertificates" +) + +type DynamicTLSServingCertProvider interface { + dynamiccertificates.CertKeyContentProvider + Set(certPEM, keyPEM []byte) } -func (*DynamicTLSServingCertProvider) Name() string { +type dynamicTLSServingCertProvider struct { + certPEM []byte + keyPEM []byte + mutex sync.RWMutex +} + +func NewDynamicTLSServingCertProvider() DynamicTLSServingCertProvider { + return &dynamicTLSServingCertProvider{} +} + +func (p *dynamicTLSServingCertProvider) Set(certPEM, keyPEM []byte) { + p.mutex.Lock() // acquire a write lock + defer p.mutex.Unlock() + p.certPEM = certPEM + p.keyPEM = keyPEM +} + +func (p *dynamicTLSServingCertProvider) Name() string { return "DynamicTLSServingCertProvider" } -func (p *DynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) { - return p.CertPEM, p.KeyPEM +func (p *dynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) { + p.mutex.RLock() // acquire a read lock + defer p.mutex.RUnlock() + return p.certPEM, p.keyPEM } diff --git a/internal/server/server.go b/internal/server/server.go index 68cf2a1e..9711c6f8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -27,7 +27,7 @@ import ( // App is an object that represents the placeholder-name-server application. type App struct { - serverCommand *cobra.Command + cmd *cobra.Command // CLI flags configPath string @@ -48,27 +48,27 @@ func New(ctx context.Context, args []string, stdout, stderr io.Writer) *App { } // Run the server. -func (app *App) Run() error { - return app.serverCommand.Execute() +func (a *App) Run() error { + return a.cmd.Execute() } // Create the server command and save it into the App. -func (app *App) addServerCommand(ctx context.Context, args []string, stdout, stderr io.Writer) { +func (a *App) addServerCommand(ctx context.Context, args []string, stdout, stderr io.Writer) { cmd := &cobra.Command{ Use: `placeholder-name-server`, Long: "placeholder-name-server provides a generic API for mapping an external\n" + "credential from somewhere to an internal credential to be used for\n" + "authenticating to the Kubernetes API.", - RunE: func(cmd *cobra.Command, args []string) error { return app.runServer(ctx) }, + RunE: func(cmd *cobra.Command, args []string) error { return a.runServer(ctx) }, Args: cobra.NoArgs, } cmd.SetArgs(args) cmd.SetOut(stdout) cmd.SetErr(stderr) - addCommandlineFlagsToCommand(cmd, app) + addCommandlineFlagsToCommand(cmd, a) - app.serverCommand = cmd + a.cmd = cmd } // Define the app's commandline flags. @@ -104,15 +104,15 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) { } // Boot the aggregated API server, which will in turn boot the controllers. -func (app *App) runServer(ctx context.Context) error { +func (a *App) runServer(ctx context.Context) error { // Read the server config file. - cfg, err := config.FromPath(app.configPath) + cfg, err := config.FromPath(a.configPath) if err != nil { return fmt.Errorf("could not load config: %w", err) } // Load the Kubernetes cluster signing CA. - k8sClusterCA, err := certauthority.Load(app.clusterSigningCertFilePath, app.clusterSigningKeyFilePath) + k8sClusterCA, err := certauthority.Load(a.clusterSigningCertFilePath, a.clusterSigningKeyFilePath) if err != nil { return fmt.Errorf("could not load cluster signing CA: %w", err) } @@ -124,7 +124,7 @@ func (app *App) runServer(ctx context.Context) error { } // Discover in which namespace we are installed. - podInfo, err := downward.Load(app.downwardAPIPath) + podInfo, err := downward.Load(a.downwardAPIPath) if err != nil { return fmt.Errorf("could not read pod metadata: %w", err) } @@ -135,7 +135,7 @@ func (app *App) runServer(ctx context.Context) error { // is stored in a k8s Secret. Therefore it also effectively acting as // an in-memory cache of what is stored in the k8s Secret, helping to // keep incoming requests fast. - dynamicCertProvider := &provider.DynamicTLSServingCertProvider{} + dynamicCertProvider := provider.NewDynamicTLSServingCertProvider() // Prepare to start the controllers, but defer actually starting them until the // post start hook of the aggregated API server. @@ -171,7 +171,7 @@ func (app *App) runServer(ctx context.Context) error { // Create a configuration for the aggregated API server. func getAggregatedAPIServerConfig( - dynamicCertProvider *provider.DynamicTLSServingCertProvider, + dynamicCertProvider provider.DynamicTLSServingCertProvider, webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator, ca *certauthority.CA, startControllersPostStartHook func(context.Context), diff --git a/internal/server/server_test.go b/internal/server/server_test.go index fc31dc4e..5ae9abe8 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -78,7 +78,7 @@ func TestCommand(t *testing.T) { stderr := bytes.NewBuffer([]byte{}) a := New(context.Background(), test.args, stdout, stderr) - a.serverCommand.RunE = func(cmd *cobra.Command, args []string) error { + a.cmd.RunE = func(cmd *cobra.Command, args []string) error { return nil } err := a.Run() diff --git a/internal/testutil/observable_with_informer_option.go b/internal/testutil/observable_with_informer_option.go new file mode 100644 index 00000000..4f3f73d0 --- /dev/null +++ b/internal/testutil/observable_with_informer_option.go @@ -0,0 +1,30 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package testutil + +import "github.com/suzerain-io/controller-go" + +type ObservableWithInformerOption struct { + InformerToFilterMap map[controller.InformerGetter]controller.Filter +} + +func NewObservableWithInformerOption() *ObservableWithInformerOption { + return &ObservableWithInformerOption{ + InformerToFilterMap: make(map[controller.InformerGetter]controller.Filter), + } +} + +func (i *ObservableWithInformerOption) WithInformer( + getter controller.InformerGetter, + filter controller.Filter, + opt controller.InformerOption) controller.Option { + i.InformerToFilterMap[getter] = filter + return controller.WithInformer(getter, filter, opt) +} + +func (i *ObservableWithInformerOption) GetFilterForInformer(getter controller.InformerGetter) controller.Filter { + return i.InformerToFilterMap[getter] +} From 8034ef24ffa4fb2df26271fa4ca073a8a54e234b Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Mon, 10 Aug 2020 19:34:45 -0700 Subject: [PATCH 4/6] Fix a mistake from the previous commit - Got the order of multiple return values backwards, which was caught by the integration tests --- internal/controller/apicerts/certs_manager.go | 2 +- internal/controller/apicerts/certs_manager_test.go | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index 5c499e0c..16eda4bd 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -101,7 +101,7 @@ func (c *certsManagerController) Sync(ctx controller.Context) error { } // Write the CA's public key bundle and the serving certs to a secret. - tlsPrivateKeyPEM, tlsCertChainPEM, err := certauthority.ToPEM(aggregatedAPIServerTLSCert) + tlsCertChainPEM, tlsPrivateKeyPEM, err := certauthority.ToPEM(aggregatedAPIServerTLSCert) if err != nil { return fmt.Errorf("could not PEM encode serving certificate: %w", err) } diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index a600b5d9..47bf82da 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -31,6 +31,8 @@ import ( placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" ) +// TODO test that it uses controller.WithInitialEvent correctly + func TestManagerControllerInformerFilters(t *testing.T) { spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" @@ -211,7 +213,7 @@ func TestManagerControllerSync(t *testing.T) { roots := x509.NewCertPool() ok := roots.AppendCertsFromPEM([]byte(actualCACert)) r.True(ok) - block, _ := pem.Decode([]byte(actualPrivateKey)) + block, _ := pem.Decode([]byte(actualCertChain)) r.NotNil(block) parsedCert, err := x509.ParseCertificate(block.Bytes) r.NoError(err) @@ -226,7 +228,8 @@ func TestManagerControllerSync(t *testing.T) { r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2) r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2) - // TODO How can we validate the tlsCertificateChain? + // TODO How can we validate that the actualPrivateKey is correct and works with the other cert values? + // Maybe start a test http server using these certs and then make a request to it using the actualCACert? // Make sure we updated the APIService caBundle and left it otherwise unchanged r.Len(aggregatorAPIClient.Actions(), 2) From fadd718d08fec2010fed307fe564f9420f5c898e Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 11 Aug 2020 10:14:57 -0700 Subject: [PATCH 5/6] Add integration and more unit tests - Add integration test for serving cert auto-generation and rotation - Add unit test for `WithInitialEvent` of the cert manager controller - Move UpdateAPIService() into the `apicerts` package, since that is the only user of the function. --- internal/controller/apicerts/certs_manager.go | 6 +- .../controller/apicerts/certs_manager_test.go | 21 ++- .../apicerts/update_api_service.go} | 3 +- .../apicerts/update_api_service_test.go} | 2 +- internal/controller/utils.go | 3 + .../controllermanager/prepare_controllers.go | 1 + .../observable_with_informer_option.go | 11 +- .../observable_with_initial_event_option.go | 25 +++ test/go.mod | 1 + test/go.sum | 143 ++++++++++++++++++ test/integration/api_serving_certs_test.go | 97 ++++++++++++ test/library/client.go | 7 + 12 files changed, 304 insertions(+), 16 deletions(-) rename internal/{autoregistration/autoregistration.go => controller/apicerts/update_api_service.go} (94%) rename internal/{autoregistration/autoregistration_test.go => controller/apicerts/update_api_service_test.go} (99%) create mode 100644 internal/testutil/observable_with_initial_event_option.go create mode 100644 test/integration/api_serving_certs_test.go diff --git a/internal/controller/apicerts/certs_manager.go b/internal/controller/apicerts/certs_manager.go index 16eda4bd..8cfbcaf9 100644 --- a/internal/controller/apicerts/certs_manager.go +++ b/internal/controller/apicerts/certs_manager.go @@ -19,7 +19,6 @@ import ( aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" "github.com/suzerain-io/controller-go" - "github.com/suzerain-io/placeholder-name/internal/autoregistration" "github.com/suzerain-io/placeholder-name/internal/certauthority" placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller" ) @@ -45,6 +44,7 @@ func NewCertsManagerController( aggregatorClient aggregatorclient.Interface, secretInformer corev1informers.SecretInformer, withInformer placeholdernamecontroller.WithInformerOptionFunc, + withInitialEvent placeholdernamecontroller.WithInitialEventOptionFunc, ) controller.Controller { return controller.New( controller.Config{ @@ -62,7 +62,7 @@ func NewCertsManagerController( controller.InformerOption{}, ), // Be sure to run once even if the Secret that the informer is watching doesn't exist. - controller.WithInitialEvent(controller.Key{ + withInitialEvent(controller.Key{ Namespace: namespace, Name: certsSecretName, }), @@ -123,7 +123,7 @@ func (c *certsManagerController) Sync(ctx controller.Context) error { } // Update the APIService to give it the new CA bundle. - if err := autoregistration.UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil { + if err := UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil { return fmt.Errorf("could not update the API service: %w", err) } diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index 47bf82da..26a76e11 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -31,26 +31,27 @@ import ( placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" ) -// TODO test that it uses controller.WithInitialEvent correctly - -func TestManagerControllerInformerFilters(t *testing.T) { - spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) { +func TestManagerControllerOptions(t *testing.T) { + spec.Run(t, "options", func(t *testing.T, when spec.G, it spec.S) { const installedInNamespace = "some-namespace" var r *require.Assertions var observableWithInformerOption *testutil.ObservableWithInformerOption + var observableWithInitialEventOption *testutil.ObservableWithInitialEventOption var secretsInformerFilter controller.Filter it.Before(func() { r = require.New(t) observableWithInformerOption = testutil.NewObservableWithInformerOption() + observableWithInitialEventOption = testutil.NewObservableWithInitialEventOption() secretsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Secrets() _ = NewCertsManagerController( installedInNamespace, nil, nil, secretsInformer, - observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters + observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters + observableWithInitialEventOption.WithInitialEvent, // make it possible to observe the behavior of the initial event ) secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer) }) @@ -102,6 +103,15 @@ func TestManagerControllerInformerFilters(t *testing.T) { }) }) }) + + when("starting up", func() { + it("asks for an initial event because the Secret may not exist yet and it needs to run anyway", func() { + r.Equal(controller.Key{ + Namespace: installedInNamespace, + Name: "api-serving-cert", + }, observableWithInitialEventOption.GetInitialEventKey()) + }) + }) }, spec.Parallel(), spec.Report(report.Terminal{})) } @@ -130,6 +140,7 @@ func TestManagerControllerSync(t *testing.T) { aggregatorAPIClient, kubeInformers.Core().V1().Secrets(), controller.WithInformer, + controller.WithInitialEvent, ) // Set this at the last second to support calling subject.Name(). diff --git a/internal/autoregistration/autoregistration.go b/internal/controller/apicerts/update_api_service.go similarity index 94% rename from internal/autoregistration/autoregistration.go rename to internal/controller/apicerts/update_api_service.go index 7033abe4..63ceaf90 100644 --- a/internal/autoregistration/autoregistration.go +++ b/internal/controller/apicerts/update_api_service.go @@ -3,8 +3,7 @@ Copyright 2020 VMware, Inc. SPDX-License-Identifier: Apache-2.0 */ -// Package autoregistration updates the pre-registered APIService. -package autoregistration +package apicerts import ( "context" diff --git a/internal/autoregistration/autoregistration_test.go b/internal/controller/apicerts/update_api_service_test.go similarity index 99% rename from internal/autoregistration/autoregistration_test.go rename to internal/controller/apicerts/update_api_service_test.go index 6ac4e731..962d8a95 100644 --- a/internal/autoregistration/autoregistration_test.go +++ b/internal/controller/apicerts/update_api_service_test.go @@ -3,7 +3,7 @@ Copyright 2020 VMware, Inc. SPDX-License-Identifier: Apache-2.0 */ -package autoregistration +package apicerts import ( "context" diff --git a/internal/controller/utils.go b/internal/controller/utils.go index 1b47be1d..25b7fc0e 100644 --- a/internal/controller/utils.go +++ b/internal/controller/utils.go @@ -29,3 +29,6 @@ type WithInformerOptionFunc func( getter controller.InformerGetter, filter controller.Filter, opt controller.InformerOption) controller.Option + +// Same signature as controller.WithInitialEvent(). +type WithInitialEventOptionFunc func(key controller.Key) controller.Option diff --git a/internal/controllermanager/prepare_controllers.go b/internal/controllermanager/prepare_controllers.go index 78b3caf1..eee8d954 100644 --- a/internal/controllermanager/prepare_controllers.go +++ b/internal/controllermanager/prepare_controllers.go @@ -66,6 +66,7 @@ func PrepareControllers( aggregatorClient, installationNamespaceK8sInformers.Core().V1().Secrets(), controller.WithInformer, + controller.WithInitialEvent, ), singletonWorker, ). diff --git a/internal/testutil/observable_with_informer_option.go b/internal/testutil/observable_with_informer_option.go index 4f3f73d0..02536a5d 100644 --- a/internal/testutil/observable_with_informer_option.go +++ b/internal/testutil/observable_with_informer_option.go @@ -8,23 +8,24 @@ package testutil import "github.com/suzerain-io/controller-go" type ObservableWithInformerOption struct { - InformerToFilterMap map[controller.InformerGetter]controller.Filter + informerToFilterMap map[controller.InformerGetter]controller.Filter } func NewObservableWithInformerOption() *ObservableWithInformerOption { return &ObservableWithInformerOption{ - InformerToFilterMap: make(map[controller.InformerGetter]controller.Filter), + informerToFilterMap: make(map[controller.InformerGetter]controller.Filter), } } func (i *ObservableWithInformerOption) WithInformer( getter controller.InformerGetter, filter controller.Filter, - opt controller.InformerOption) controller.Option { - i.InformerToFilterMap[getter] = filter + opt controller.InformerOption, +) controller.Option { + i.informerToFilterMap[getter] = filter return controller.WithInformer(getter, filter, opt) } func (i *ObservableWithInformerOption) GetFilterForInformer(getter controller.InformerGetter) controller.Filter { - return i.InformerToFilterMap[getter] + return i.informerToFilterMap[getter] } diff --git a/internal/testutil/observable_with_initial_event_option.go b/internal/testutil/observable_with_initial_event_option.go new file mode 100644 index 00000000..c16f1c08 --- /dev/null +++ b/internal/testutil/observable_with_initial_event_option.go @@ -0,0 +1,25 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package testutil + +import "github.com/suzerain-io/controller-go" + +type ObservableWithInitialEventOption struct { + key controller.Key +} + +func NewObservableWithInitialEventOption() *ObservableWithInitialEventOption { + return &ObservableWithInitialEventOption{} +} + +func (i *ObservableWithInitialEventOption) WithInitialEvent(key controller.Key) controller.Option { + i.key = key + return controller.WithInitialEvent(key) +} + +func (i *ObservableWithInitialEventOption) GetInitialEventKey() controller.Key { + return i.key +} diff --git a/test/go.mod b/test/go.mod index 9f5d9734..648e8f8e 100644 --- a/test/go.mod +++ b/test/go.mod @@ -11,6 +11,7 @@ require ( k8s.io/api v0.19.0-rc.0 k8s.io/apimachinery v0.19.0-rc.0 k8s.io/client-go v0.19.0-rc.0 + k8s.io/kube-aggregator v0.18.6 ) replace ( diff --git a/test/go.sum b/test/go.sum index 8d31b19d..a97a4bf2 100644 --- a/test/go.sum +++ b/test/go.sum @@ -11,6 +11,7 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= @@ -30,29 +31,56 @@ github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.0.0-20190815234213-e83c0a1c26c8/go.mod h1:pmLOTb3x90VhIKxsA9yeQG5yfOkkKnkk1h+Ql8NDYDw= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -67,14 +95,19 @@ github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -104,28 +137,46 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -135,10 +186,17 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -146,9 +204,12 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -157,29 +218,68 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -209,9 +309,12 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -221,7 +324,9 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -238,34 +343,46 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -283,6 +400,7 @@ golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -315,6 +433,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -327,17 +446,23 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -347,26 +472,44 @@ gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.18.6/go.mod h1:eeyxr+cwCjMdLAmr2W3RyDI0VvTawSg/3RFFBEnmZGI= k8s.io/api v0.19.0-rc.0 h1:K+xi+F3RNAxpFyS1f7uHekMNprjFX7WVZDx2lJE+A3A= k8s.io/api v0.19.0-rc.0/go.mod h1:WBGMHEmngOdQBAvJiYUgP5mGDdCWXM52yDm1gtos8C0= +k8s.io/apimachinery v0.18.6/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= k8s.io/apimachinery v0.19.0-rc.0 h1:IBmRy0elJCGgxtCT0bHT93N+rhx+vF2DD1XXJ3ntLa8= k8s.io/apimachinery v0.19.0-rc.0/go.mod h1:EjWiYOPi+BZennZ5pGa3JLkQ+znhEOodGy/+umjiLDU= +k8s.io/apiserver v0.18.6/go.mod h1:Zt2XvTHuaZjBz6EFYzpp+X4hTmgWGy8AthNVnTdm3Wg= +k8s.io/client-go v0.18.6/go.mod h1:/fwtGLjYMS1MaM5oi+eXhKwG+1UHidUEXRh6cNsdO0Q= k8s.io/client-go v0.19.0-rc.0 h1:6WW8MElhoLeYcLiN4ky1159XG5E39KYdmLCrV/6lNiE= k8s.io/client-go v0.19.0-rc.0/go.mod h1:3kWGD05F7c58atlk7ep9ob1hg2Yu9NSz8gJxCNNTHhc= +k8s.io/code-generator v0.18.6/go.mod h1:TgNEVx9hCyPGpdtCWA34olQYLkh3ok9ar7XfSsr8b6c= +k8s.io/component-base v0.18.6/go.mod h1:knSVsibPR5K6EW2XOjEHik6sdU5nCvKMrzMt2D4In14= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200114144118-36b2048a9120/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-aggregator v0.18.6 h1:xGP3oe0tAWEYnGWTnDPjXiIItekrnwDA2O7w0WqvGoo= +k8s.io/kube-aggregator v0.18.6/go.mod h1:MKm8inLHdeiXQJCl6UdmgMosRrqJgyxO2obTXOkey/s= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= k8s.io/kube-openapi v0.0.0-20200427153329-656914f816f9/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw= k8s.io/kube-openapi v0.0.0-20200615155156-dffdd1682719/go.mod h1:bfCVj+qXcEaE5SCvzBaqpOySr6tuCcpPKqF6HD8nyCw= +k8s.io/utils v0.0.0-20200324210504-a9aa75ae1b89/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19 h1:7Nu2dTj82c6IaWvL7hImJzcXoTPz1MsSCH7r+0m6rfo= k8s.io/utils v0.0.0-20200619165400-6e3d28b6ed19/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.7/go.mod h1:PHgbrJT7lCHcxMU+mDHEm+nx46H4zuuHZkDP6icnhu0= sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= diff --git a/test/integration/api_serving_certs_test.go b/test/integration/api_serving_certs_test.go new file mode 100644 index 00000000..ef35edce --- /dev/null +++ b/test/integration/api_serving_certs_test.go @@ -0,0 +1,97 @@ +/* +Copyright 2020 VMware, Inc. +SPDX-License-Identifier: Apache-2.0 +*/ + +package integration + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1" + "github.com/suzerain-io/placeholder-name/test/library" +) + +func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) { + library.SkipUnlessIntegration(t) + namespaceName := library.Getenv(t, "PLACEHOLDER_NAME_NAMESPACE") + + kubeClient := library.NewClientset(t) + aggregatedClient := library.NewAggregatedClientset(t) + placeholderClient := library.NewPlaceholderNameClientset(t) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + const apiServiceName = "v1alpha1.placeholder.suzerain-io.github.io" + + // Get the initial auto-generated version of the Secret. + secret, err := kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{}) + require.NoError(t, err) + initialCACert := secret.Data["caCertificate"] + initialPrivateKey := secret.Data["tlsPrivateKey"] + initialCertChain := secret.Data["tlsCertificateChain"] + require.NotEmpty(t, initialCACert) + require.NotEmpty(t, initialPrivateKey) + require.NotEmpty(t, initialCertChain) + + // Check that the APIService has the same CA. + apiService, err := aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{}) + require.NoError(t, err) + require.Equal(t, initialCACert, apiService.Spec.CABundle) + + // Delete the Secret, simulating an end user doing `kubectl delete` to manually ask for an immediate rotation. + err = kubeClient.CoreV1().Secrets(namespaceName).Delete(ctx, "api-serving-cert", metav1.DeleteOptions{}) + require.NoError(t, err) + + // Expect that the Secret comes back right away with newly minted certs. + var secretIsRegenerated = func() bool { + secret, err = kubeClient.CoreV1().Secrets(namespaceName).Get(ctx, "api-serving-cert", metav1.GetOptions{}) + return err == nil + } + assert.Eventually(t, secretIsRegenerated, 10*time.Second, 250*time.Millisecond) + require.NoError(t, err) // prints out the error in case of failure + regeneratedCACert := secret.Data["caCertificate"] + regeneratedPrivateKey := secret.Data["tlsPrivateKey"] + regeneratedCertChain := secret.Data["tlsCertificateChain"] + require.NotEmpty(t, regeneratedCACert) + require.NotEmpty(t, regeneratedPrivateKey) + require.NotEmpty(t, regeneratedCertChain) + require.NotEqual(t, initialCACert, regeneratedCACert) + require.NotEqual(t, initialPrivateKey, regeneratedPrivateKey) + require.NotEqual(t, initialCertChain, regeneratedCertChain) + + // Expect that the APIService was also updated with the new CA. + var aggregatedAPIUpdated = func() bool { + apiService, err = aggregatedClient.ApiregistrationV1().APIServices().Get(ctx, apiServiceName, metav1.GetOptions{}) + return err == nil + } + assert.Eventually(t, aggregatedAPIUpdated, 10*time.Second, 250*time.Millisecond) + require.NoError(t, err) // prints out the error in case of failure + require.Equal(t, regeneratedCACert, apiService.Spec.CABundle) + + // Check that we can still make requests to the aggregated API through the kube API server, + // because the kube API server uses these certs when proxying requests to the aggregated API server, + // so this is effectively checking that the aggregated API server is using these new certs. + var aggregatedAPIWorking = func() bool { + _, err = placeholderClient.PlaceholderV1alpha1().LoginRequests().Create(ctx, &v1alpha1.LoginRequest{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{}, + Spec: v1alpha1.LoginRequestSpec{ + Type: v1alpha1.TokenLoginCredentialType, + Token: &v1alpha1.LoginRequestTokenCredential{Value: "not a good token"}, + }, + }, metav1.CreateOptions{}) + // Should have got a success response with an error message inside it complaining about the token value. + return err == nil + } + // Unfortunately, although our code changes all the certs immediately, it seems to take ~1 minute for + // the API machinery to notice that we updated our serving cert, causing 1 minute of downtime for our endpoint. + assert.Eventually(t, aggregatedAPIWorking, 2*time.Minute, 250*time.Millisecond) + require.NoError(t, err) // prints out the error in case of failure +} diff --git a/test/library/client.go b/test/library/client.go index 4ede3098..5524d919 100644 --- a/test/library/client.go +++ b/test/library/client.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" placeholdernameclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned" ) @@ -63,3 +64,9 @@ func NewPlaceholderNameClientset(t *testing.T) placeholdernameclientset.Interfac return placeholdernameclientset.NewForConfigOrDie(NewClientConfig(t)) } + +func NewAggregatedClientset(t *testing.T) aggregatorclient.Interface { + t.Helper() + + return aggregatorclient.NewForConfigOrDie(NewClientConfig(t)) +} From 5ec1fbd1cadf79beb085b4bef89c643ad6cb92b0 Mon Sep 17 00:00:00 2001 From: Ryan Richard Date: Tue, 11 Aug 2020 10:39:50 -0700 Subject: [PATCH 6/6] Add an assertion that the private key and cert chain match in certs_manager_test.go Signed-off-by: Matt Moyer --- internal/controller/apicerts/certs_manager_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/controller/apicerts/certs_manager_test.go b/internal/controller/apicerts/certs_manager_test.go index 26a76e11..c4c68e13 100644 --- a/internal/controller/apicerts/certs_manager_test.go +++ b/internal/controller/apicerts/certs_manager_test.go @@ -7,6 +7,7 @@ package apicerts import ( "context" + "crypto/tls" "crypto/x509" "encoding/pem" "errors" @@ -239,8 +240,9 @@ func TestManagerControllerSync(t *testing.T) { r.WithinDuration(time.Now(), parsedCert.NotBefore, time.Minute*2) r.WithinDuration(time.Now().Add(24*365*time.Hour), parsedCert.NotAfter, time.Minute*2) - // TODO How can we validate that the actualPrivateKey is correct and works with the other cert values? - // Maybe start a test http server using these certs and then make a request to it using the actualCACert? + // Check that the private key and cert chain match + _, err = tls.X509KeyPair([]byte(actualCertChain), []byte(actualPrivateKey)) + r.NoError(err) // Make sure we updated the APIService caBundle and left it otherwise unchanged r.Len(aggregatorAPIClient.Actions(), 2)