Add validation step to "pinniped get kubeconfig".

This adds two new flags to "pinniped get kubeconfig": --skip-validation and --timeout.

By default, at the end of the kubeconfig generation process, we validate that we can reach the configured cluster. In the future this might also validate that the TokenCredentialRequest API is running, but for not it just verifies that the DNS name resolves, and the TLS connection is available on the given port.

If there is an error during this check, we block and retry for up to 10 minutes. This duration can be changed with --timeout an the entire process can be skipped with --skip-validation.

Signed-off-by: Matt Moyer <moyerm@vmware.com>

cmd/pinniped/cmd/kubeconfig.go

@@ -6,11 +6,14 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
 	"strconv"
 	"strings"
@@ -91,6 +94,8 @@ type getKubeconfigConciergeParams struct {
 type getKubeconfigParams struct {
 	kubeconfigPath            string
 	kubeconfigContextOverride string
+	skipValidate              bool
+	timeout                   time.Duration
 	outputPath                string
 	staticToken               string
 	staticTokenEnvName        string
@@ -136,6 +141,8 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command {
 	f.StringVar(&flags.oidc.requestAudience, "oidc-request-audience", "", "Request a token with an alternate audience using RFC8693 token exchange")
 	f.StringVar(&flags.kubeconfigPath, "kubeconfig", os.Getenv("KUBECONFIG"), "Path to kubeconfig file")
 	f.StringVar(&flags.kubeconfigContextOverride, "kubeconfig-context", "", "Kubeconfig context name (default: current active context)")
+	f.BoolVar(&flags.skipValidate, "skip-validation", false, "Skip final validation of the kubeconfig (default: false)")
+	f.DurationVar(&flags.timeout, "timeout", 10*time.Minute, "Timeout for autodiscovery and validation")
 	f.StringVarP(&flags.outputPath, "output", "o", "", "Output file path (default: stdout)")
 
 	mustMarkHidden(cmd, "oidc-debug-session-cache")
@@ -152,13 +159,16 @@ func kubeconfigCommand(deps kubeconfigDeps) *cobra.Command {
 			defer func() { _ = out.Close() }()
 			cmd.SetOut(out)
 		}
-		return runGetKubeconfig(cmd.OutOrStdout(), deps, flags)
+		return runGetKubeconfig(cmd.Context(), cmd.OutOrStdout(), deps, flags)
 	}
 	return cmd
 }
 
 //nolint:funlen
-func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigParams) error {
+func runGetKubeconfig(ctx context.Context, out io.Writer, deps kubeconfigDeps, flags getKubeconfigParams) error {
+	ctx, cancel := context.WithTimeout(ctx, flags.timeout)
+	defer cancel()
+
 	// Validate api group suffix and immediately return an error if it is invalid.
 	if err := groupsuffix.Validate(flags.concierge.apiGroupSuffix); err != nil {
 		return fmt.Errorf("invalid api group suffix: %w", err)
@@ -229,7 +239,12 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar
 		if flags.staticTokenEnvName != "" {
 			execConfig.Args = append(execConfig.Args, "--token-env="+flags.staticTokenEnvName)
 		}
-		return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig))
+
+		kubeconfig := newExecKubeconfig(cluster, &execConfig)
+		if err := validateKubeconfig(ctx, flags, kubeconfig, deps.log); err != nil {
+			return err
+		}
+		return writeConfigAsYAML(out, kubeconfig)
 	}
 
 	// Otherwise continue to parse the OIDC-related flags and output a config that runs `pinniped login oidc`.
@@ -260,7 +275,11 @@ func runGetKubeconfig(out io.Writer, deps kubeconfigDeps, flags getKubeconfigPar
 	if flags.oidc.requestAudience != "" {
 		execConfig.Args = append(execConfig.Args, "--request-audience="+flags.oidc.requestAudience)
 	}
-	return writeConfigAsYAML(out, newExecKubeconfig(cluster, &execConfig))
+	kubeconfig := newExecKubeconfig(cluster, &execConfig)
+	if err := validateKubeconfig(ctx, flags, kubeconfig, deps.log); err != nil {
+		return err
+	}
+	return writeConfigAsYAML(out, kubeconfig)
 }
 
 func configureConcierge(credentialIssuer *configv1alpha1.CredentialIssuer, authenticator metav1.Object, flags *getKubeconfigParams, v1Cluster *clientcmdapi.Cluster, oidcCABundle *string, execConfig *clientcmdapi.ExecConfig, log logr.Logger) error {
@@ -518,3 +537,77 @@ func copyCurrentClusterFromExistingKubeConfig(currentKubeConfig clientcmdapi.Con
 	}
 	return currentKubeConfig.Clusters[ctx.Cluster], nil
 }
+
+func validateKubeconfig(ctx context.Context, flags getKubeconfigParams, kubeconfig clientcmdapi.Config, log logr.Logger) error {
+	if flags.skipValidate {
+		return nil
+	}
+
+	kubeContext := kubeconfig.Contexts[kubeconfig.CurrentContext]
+	if kubeContext == nil {
+		return fmt.Errorf("invalid kubeconfig (no context)")
+	}
+	cluster := kubeconfig.Clusters[kubeContext.Cluster]
+	if cluster == nil {
+		return fmt.Errorf("invalid kubeconfig (no cluster)")
+	}
+
+	kubeconfigCA := x509.NewCertPool()
+	if !kubeconfigCA.AppendCertsFromPEM(cluster.CertificateAuthorityData) {
+		return fmt.Errorf("invalid kubeconfig (no certificateAuthorityData)")
+	}
+
+	httpClient := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				MinVersion: tls.VersionTLS12,
+				RootCAs:    kubeconfigCA,
+			},
+			Proxy:               http.ProxyFromEnvironment,
+			TLSHandshakeTimeout: 10 * time.Second,
+		},
+		Timeout: 10 * time.Second,
+	}
+
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	pingCluster := func() error {
+		reqCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		defer cancel()
+		req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, cluster.Server, nil)
+		if err != nil {
+			return fmt.Errorf("could not form request to validate cluster: %w", err)
+		}
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			return err
+		}
+		_ = resp.Body.Close()
+		if resp.StatusCode >= 500 {
+			return fmt.Errorf("unexpected status code %d", resp.StatusCode)
+		}
+		return nil
+	}
+
+	err := pingCluster()
+	if err == nil {
+		log.Info("validated connection to the cluster")
+		return nil
+	}
+
+	log.Info("could not immediately connect to the cluster but it may be initializing, will retry until timeout")
+	deadline, _ := ctx.Deadline()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+			err := pingCluster()
+			if err == nil {
+				return nil
+			}
+			log.Error(err, "could not connect to cluster, retrying...", "remaining", time.Until(deadline).Round(time.Second).String())
+		}
+	}
+}

cmd/pinniped/cmd/kubeconfig_test.go

@@ -84,8 +84,10 @@ func TestGetKubeconfig(t *testing.T) {
 				      --oidc-session-cache string             Path to OpenID Connect session cache file
 				      --oidc-skip-browser                     During OpenID Connect login, skip opening the browser (just print the URL)
 				  -o, --output string                         Output file path (default: stdout)
+				      --skip-validation                       Skip final validation of the kubeconfig (default: false)
 				      --static-token string                   Instead of doing an OIDC-based login, specify a static token
 				      --static-token-env string               Instead of doing an OIDC-based login, read a static token from the environment
+				      --timeout duration                      Timeout for autodiscovery and validation (default 10m0s)
 			`),
 		},
 		{
@@ -528,6 +530,7 @@ func TestGetKubeconfig(t *testing.T) {
 			args: []string{
 				"--kubeconfig", "./testdata/kubeconfig.yaml",
 				"--static-token", "test-token",
+				"--skip-validation",
 			},
 			conciergeObjects: []runtime.Object{
 				&configv1alpha1.CredentialIssuer{
@@ -588,6 +591,7 @@ func TestGetKubeconfig(t *testing.T) {
 			args: []string{
 				"--kubeconfig", "./testdata/kubeconfig.yaml",
 				"--static-token-env", "TEST_TOKEN",
+				"--skip-validation",
 			},
 			conciergeObjects: []runtime.Object{
 				&configv1alpha1.CredentialIssuer{
@@ -647,6 +651,7 @@ func TestGetKubeconfig(t *testing.T) {
 			name: "autodetect JWT authenticator",
 			args: []string{
 				"--kubeconfig", "./testdata/kubeconfig.yaml",
+				"--skip-validation",
 			},
 			conciergeObjects: []runtime.Object{
 				&configv1alpha1.CredentialIssuer{
@@ -735,6 +740,7 @@ func TestGetKubeconfig(t *testing.T) {
 				"--oidc-session-cache", "/path/to/cache/dir/sessions.yaml",
 				"--oidc-debug-session-cache",
 				"--oidc-request-audience", "test-audience",
+				"--skip-validation",
 			},
 			conciergeObjects: []runtime.Object{
 				&configv1alpha1.CredentialIssuer{
@@ -802,6 +808,7 @@ func TestGetKubeconfig(t *testing.T) {
 				"--concierge-ca-bundle", testConciergeCABundlePath,
 				"--concierge-endpoint", "https://impersonation-proxy-endpoint.test",
 				"--concierge-mode", "ImpersonationProxy",
+				"--skip-validation",
 			},
 			conciergeObjects: []runtime.Object{
 				&configv1alpha1.CredentialIssuer{
@@ -871,6 +878,7 @@ func TestGetKubeconfig(t *testing.T) {
 			name: "autodetect impersonation proxy with autodetected JWT authenticator",
 			args: []string{
 				"--kubeconfig", "./testdata/kubeconfig.yaml",
+				"--skip-validation",
 			},
 			conciergeObjects: []runtime.Object{
 				&configv1alpha1.CredentialIssuer{