From ae498f14b426695587bb928506e2e1ccb812242e Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Tue, 12 Jan 2021 15:55:31 -0500 Subject: [PATCH] test/integration: ensure no pods restart during integration tests Signed-off-by: Andrew Keesler --- test/integration/cli_test.go | 2 + .../concierge_api_serving_certs_test.go | 2 + test/integration/concierge_client_test.go | 2 + .../concierge_credentialissuerconfig_test.go | 2 + .../concierge_credentialrequest_test.go | 18 ++++-- .../concierge_kubecertagent_test.go | 2 + test/integration/e2e_test.go | 3 + test/integration/supervisor_discovery_test.go | 7 +++ test/integration/supervisor_healthz_test.go | 4 +- test/integration/supervisor_login_test.go | 2 + test/integration/supervisor_secrets_test.go | 2 + test/integration/supervisor_upstream_test.go | 2 + test/library/assertions.go | 57 +++++++++++++++++++ 13 files changed, 100 insertions(+), 5 deletions(-) diff --git a/test/integration/cli_test.go b/test/integration/cli_test.go index d31b817d..887c264e 100644 --- a/test/integration/cli_test.go +++ b/test/integration/cli_test.go @@ -35,6 +35,8 @@ import ( func TestCLIGetKubeconfigStaticToken(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + // Create a test webhook configuration to use with the CLI. ctx, cancelFunc := context.WithTimeout(context.Background(), 4*time.Minute) defer cancelFunc() diff --git a/test/integration/concierge_api_serving_certs_test.go b/test/integration/concierge_api_serving_certs_test.go index c7885fb1..2aabe315 100644 --- a/test/integration/concierge_api_serving_certs_test.go +++ b/test/integration/concierge_api_serving_certs_test.go @@ -22,6 +22,8 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) { env := library.IntegrationEnv(t) defaultServingCertResourceName := env.ConciergeAppName + "-api-tls-serving-certificate" + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + tests := []struct { name string forceRotation func(context.Context, kubernetes.Interface, string) error diff --git a/test/integration/concierge_client_test.go b/test/integration/concierge_client_test.go index 67f23083..5098664c 100644 --- a/test/integration/concierge_client_test.go +++ b/test/integration/concierge_client_test.go @@ -57,6 +57,8 @@ var maskKey = func(s string) string { return strings.ReplaceAll(s, "TESTING KEY" func TestClient(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() diff --git a/test/integration/concierge_credentialissuerconfig_test.go b/test/integration/concierge_credentialissuerconfig_test.go index a59080b0..065fcf8c 100644 --- a/test/integration/concierge_credentialissuerconfig_test.go +++ b/test/integration/concierge_credentialissuerconfig_test.go @@ -21,6 +21,8 @@ func TestCredentialIssuer(t *testing.T) { config := library.NewClientConfig(t) client := library.NewConciergeClientset(t) + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() diff --git a/test/integration/concierge_credentialrequest_test.go b/test/integration/concierge_credentialrequest_test.go index 8b1b6c45..29878b9f 100644 --- a/test/integration/concierge_credentialrequest_test.go +++ b/test/integration/concierge_credentialrequest_test.go @@ -23,7 +23,9 @@ import ( ) func TestUnsuccessfulCredentialRequest(t *testing.T) { - library.SkipUnlessIntegration(t) + env := library.IntegrationEnv(t) + + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -42,6 +44,8 @@ func TestUnsuccessfulCredentialRequest(t *testing.T) { func TestSuccessfulCredentialRequest(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute) defer cancel() @@ -127,7 +131,9 @@ func TestSuccessfulCredentialRequest(t *testing.T) { } func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthenticateTheUser(t *testing.T) { - library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{Token: "not a good token"}) @@ -139,7 +145,9 @@ func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthentic } func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T) { - library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{Token: ""}) @@ -158,7 +166,9 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T } func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheClusterIsNotCapable(t *testing.T) { - library.IntegrationEnv(t).WithoutCapability(library.ClusterSigningKeyIsAvailable) + env := library.IntegrationEnv(t).WithoutCapability(library.ClusterSigningKeyIsAvailable) + + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 8d054dce..73d9d4f6 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -27,6 +27,8 @@ const ( func TestKubeCertAgent(t *testing.T) { env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable) + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) defer cancel() diff --git a/test/integration/e2e_test.go b/test/integration/e2e_test.go index cc5f4964..86f21599 100644 --- a/test/integration/e2e_test.go +++ b/test/integration/e2e_test.go @@ -46,6 +46,9 @@ func TestE2EFullIntegration(t *testing.T) { defer library.DumpLogs(t, env.SupervisorNamespace, "") defer library.DumpLogs(t, "dex", "app=proxy") + library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "") + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute) defer cancelFunc() diff --git a/test/integration/supervisor_discovery_test.go b/test/integration/supervisor_discovery_test.go index dc2bf971..906fa1f5 100644 --- a/test/integration/supervisor_discovery_test.go +++ b/test/integration/supervisor_discovery_test.go @@ -44,7 +44,10 @@ func TestSupervisorOIDCDiscovery(t *testing.T) { env := library.IntegrationEnv(t) client := library.NewSupervisorClientset(t) + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ns := env.SupervisorNamespace + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() @@ -148,6 +151,8 @@ func TestSupervisorTLSTerminationWithSNI(t *testing.T) { pinnipedClient := library.NewSupervisorClientset(t) kubeClient := library.NewKubernetesClientset(t) + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ns := env.SupervisorNamespace ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() @@ -215,6 +220,8 @@ func TestSupervisorTLSTerminationWithDefaultCerts(t *testing.T) { pinnipedClient := library.NewSupervisorClientset(t) kubeClient := library.NewKubernetesClientset(t) + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ns := env.SupervisorNamespace ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() diff --git a/test/integration/supervisor_healthz_test.go b/test/integration/supervisor_healthz_test.go index 1691be40..20323aca 100644 --- a/test/integration/supervisor_healthz_test.go +++ b/test/integration/supervisor_healthz_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 the Pinniped contributors. All Rights Reserved. +// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package integration @@ -29,6 +29,8 @@ func TestSupervisorHealthz(t *testing.T) { t.Skip("PINNIPED_TEST_SUPERVISOR_HTTP_ADDRESS not defined") } + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() diff --git a/test/integration/supervisor_login_test.go b/test/integration/supervisor_login_test.go index 6b25408b..d6779c35 100644 --- a/test/integration/supervisor_login_test.go +++ b/test/integration/supervisor_login_test.go @@ -44,6 +44,8 @@ func TestSupervisorLogin(t *testing.T) { defer library.DumpLogs(t, env.SupervisorNamespace, "") defer library.DumpLogs(t, "dex", "app=proxy") + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() diff --git a/test/integration/supervisor_secrets_test.go b/test/integration/supervisor_secrets_test.go index ebd36693..f1b0ff32 100644 --- a/test/integration/supervisor_secrets_test.go +++ b/test/integration/supervisor_secrets_test.go @@ -24,6 +24,8 @@ func TestSupervisorSecrets(t *testing.T) { kubeClient := library.NewKubernetesClientset(t) supervisorClient := library.NewSupervisorClientset(t) + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() diff --git a/test/integration/supervisor_upstream_test.go b/test/integration/supervisor_upstream_test.go index d5182510..31ccc5cb 100644 --- a/test/integration/supervisor_upstream_test.go +++ b/test/integration/supervisor_upstream_test.go @@ -17,6 +17,8 @@ import ( func TestSupervisorUpstreamOIDCDiscovery(t *testing.T) { env := library.IntegrationEnv(t) + library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "") + t.Run("invalid missing secret and bad issuer", func(t *testing.T) { t.Parallel() spec := v1alpha1.OIDCIdentityProviderSpec{ diff --git a/test/library/assertions.go b/test/library/assertions.go index 476e1ff3..4f42f5a7 100644 --- a/test/library/assertions.go +++ b/test/library/assertions.go @@ -4,10 +4,14 @@ package library import ( + "context" + "fmt" "testing" "time" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ) @@ -24,3 +28,56 @@ func RequireEventuallyWithoutError( t.Helper() require.NoError(t, wait.PollImmediate(tick, waitFor, f), msgAndArgs...) } + +// NewRestartAssertion allows a caller to assert that there were no restarts for a Pod in the +// provided namespace with the provided labelSelector during the lifetime of a test. +func AssertNoRestartsDuringTest(t *testing.T, namespace, labelSelector string) { + t.Helper() + + previousRestartCounts := getRestartCounts(t, namespace, labelSelector) + + t.Cleanup(func() { + currentRestartCounts := getRestartCounts(t, namespace, labelSelector) + + for key, previousRestartCount := range previousRestartCounts { + currentRestartCount, ok := currentRestartCounts[key] + if assert.Truef( + t, + ok, + "pod namespace/name/container %s existed at beginning of the test, but not the end", + key, + ) { + assert.Equal( + t, + previousRestartCount, + currentRestartCount, + "pod namespace/name/container %s has restarted %d times (original count was %d)", + key, + currentRestartCount, + previousRestartCount, + ) + } + } + }) +} + +func getRestartCounts(t *testing.T, namespace, labelSelector string) map[string]int32 { + t.Helper() + + kubeClient := NewKubernetesClientset(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + require.NoError(t, err) + + restartCounts := make(map[string]int32) + for _, pod := range pods.Items { + for _, container := range pod.Status.ContainerStatuses { + key := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, container.Name) + restartCounts[key] = container.RestartCount + } + } + + return restartCounts +}