test/integration: ensure no pods restart during integration tests

Signed-off-by: Andrew Keesler <akeesler@vmware.com>

test/integration/cli_test.go

@@ -35,6 +35,8 @@ import (
 func TestCLIGetKubeconfigStaticToken(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+
 	// Create a test webhook configuration to use with the CLI.
 	ctx, cancelFunc := context.WithTimeout(context.Background(), 4*time.Minute)
 	defer cancelFunc()

test/integration/concierge_api_serving_certs_test.go

@@ -22,6 +22,8 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) {
 	env := library.IntegrationEnv(t)
 	defaultServingCertResourceName := env.ConciergeAppName + "-api-tls-serving-certificate"
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+
 	tests := []struct {
 		name          string
 		forceRotation func(context.Context, kubernetes.Interface, string) error

test/integration/concierge_client_test.go

@@ -57,6 +57,8 @@ var maskKey = func(s string) string { return strings.ReplaceAll(s, "TESTING KEY"
 func TestClient(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 	defer cancel()
 

test/integration/concierge_credentialissuerconfig_test.go

@@ -21,6 +21,8 @@ func TestCredentialIssuer(t *testing.T) {
 	config := library.NewClientConfig(t)
 	client := library.NewConciergeClientset(t)
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
 

test/integration/concierge_credentialrequest_test.go

@@ -23,7 +23,9 @@ import (
 )
 
 func TestUnsuccessfulCredentialRequest(t *testing.T) {
-	library.SkipUnlessIntegration(t)
+	env := library.IntegrationEnv(t)
+
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
 
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 	defer cancel()
@@ -42,6 +44,8 @@ func TestUnsuccessfulCredentialRequest(t *testing.T) {
 func TestSuccessfulCredentialRequest(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 6*time.Minute)
 	defer cancel()
 
@@ -127,7 +131,9 @@ func TestSuccessfulCredentialRequest(t *testing.T) {
 }
 
 func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthenticateTheUser(t *testing.T) {
-	library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
+	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
+
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
 
 	response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{Token: "not a good token"})
 
@@ -139,7 +145,9 @@ func TestFailedCredentialRequestWhenTheRequestIsValidButTheTokenDoesNotAuthentic
 }
 
 func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T) {
-	library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
+	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
+
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
 
 	response, err := makeRequest(context.Background(), t, loginv1alpha1.TokenCredentialRequestSpec{Token: ""})
 
@@ -158,7 +166,9 @@ func TestCredentialRequest_ShouldFailWhenRequestDoesNotIncludeToken(t *testing.T
 }
 
 func TestCredentialRequest_OtherwiseValidRequestWithRealTokenShouldFailWhenTheClusterIsNotCapable(t *testing.T) {
-	library.IntegrationEnv(t).WithoutCapability(library.ClusterSigningKeyIsAvailable)
+	env := library.IntegrationEnv(t).WithoutCapability(library.ClusterSigningKeyIsAvailable)
+
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
 
 	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 	defer cancel()

test/integration/concierge_kubecertagent_test.go

@@ -27,6 +27,8 @@ const (
 func TestKubeCertAgent(t *testing.T) {
 	env := library.IntegrationEnv(t).WithCapability(library.ClusterSigningKeyIsAvailable)
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute)
 	defer cancel()
 

test/integration/e2e_test.go

@@ -46,6 +46,9 @@ func TestE2EFullIntegration(t *testing.T) {
 	defer library.DumpLogs(t, env.SupervisorNamespace, "")
 	defer library.DumpLogs(t, "dex", "app=proxy")
 
+	library.AssertNoRestartsDuringTest(t, env.ConciergeNamespace, "")
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancelFunc()
 

test/integration/supervisor_discovery_test.go

@@ -44,7 +44,10 @@ func TestSupervisorOIDCDiscovery(t *testing.T) {
 	env := library.IntegrationEnv(t)
 	client := library.NewSupervisorClientset(t)
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ns := env.SupervisorNamespace
+
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
 
@@ -148,6 +151,8 @@ func TestSupervisorTLSTerminationWithSNI(t *testing.T) {
 	pinnipedClient := library.NewSupervisorClientset(t)
 	kubeClient := library.NewKubernetesClientset(t)
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ns := env.SupervisorNamespace
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()
@@ -215,6 +220,8 @@ func TestSupervisorTLSTerminationWithDefaultCerts(t *testing.T) {
 	pinnipedClient := library.NewSupervisorClientset(t)
 	kubeClient := library.NewKubernetesClientset(t)
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ns := env.SupervisorNamespace
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	defer cancel()

test/integration/supervisor_healthz_test.go

@@ -1,4 +1,4 @@
-// Copyright 2020 the Pinniped contributors. All Rights Reserved.
+// Copyright 2020-2021 the Pinniped contributors. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
 package integration
@@ -29,6 +29,8 @@ func TestSupervisorHealthz(t *testing.T) {
 		t.Skip("PINNIPED_TEST_SUPERVISOR_HTTP_ADDRESS not defined")
 	}
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 	defer cancel()
 

test/integration/supervisor_login_test.go

@@ -44,6 +44,8 @@ func TestSupervisorLogin(t *testing.T) {
 	defer library.DumpLogs(t, env.SupervisorNamespace, "")
 	defer library.DumpLogs(t, "dex", "app=proxy")
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
 	defer cancel()
 

test/integration/supervisor_secrets_test.go

@@ -24,6 +24,8 @@ func TestSupervisorSecrets(t *testing.T) {
 	kubeClient := library.NewKubernetesClientset(t)
 	supervisorClient := library.NewSupervisorClientset(t)
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
 	defer cancel()
 

test/integration/supervisor_upstream_test.go

@@ -17,6 +17,8 @@ import (
 func TestSupervisorUpstreamOIDCDiscovery(t *testing.T) {
 	env := library.IntegrationEnv(t)
 
+	library.AssertNoRestartsDuringTest(t, env.SupervisorNamespace, "")
+
 	t.Run("invalid missing secret and bad issuer", func(t *testing.T) {
 		t.Parallel()
 		spec := v1alpha1.OIDCIdentityProviderSpec{

test/library/assertions.go

@@ -4,10 +4,14 @@
 package library
 
 import (
+	"context"
+	"fmt"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 )
 
@@ -24,3 +28,56 @@ func RequireEventuallyWithoutError(
 	t.Helper()
 	require.NoError(t, wait.PollImmediate(tick, waitFor, f), msgAndArgs...)
 }
+
+// NewRestartAssertion allows a caller to assert that there were no restarts for a Pod in the
+// provided namespace with the provided labelSelector during the lifetime of a test.
+func AssertNoRestartsDuringTest(t *testing.T, namespace, labelSelector string) {
+	t.Helper()
+
+	previousRestartCounts := getRestartCounts(t, namespace, labelSelector)
+
+	t.Cleanup(func() {
+		currentRestartCounts := getRestartCounts(t, namespace, labelSelector)
+
+		for key, previousRestartCount := range previousRestartCounts {
+			currentRestartCount, ok := currentRestartCounts[key]
+			if assert.Truef(
+				t,
+				ok,
+				"pod namespace/name/container %s existed at beginning of the test, but not the end",
+				key,
+			) {
+				assert.Equal(
+					t,
+					previousRestartCount,
+					currentRestartCount,
+					"pod namespace/name/container %s has restarted %d times (original count was %d)",
+					key,
+					currentRestartCount,
+					previousRestartCount,
+				)
+			}
+		}
+	})
+}
+
+func getRestartCounts(t *testing.T, namespace, labelSelector string) map[string]int32 {
+	t.Helper()
+
+	kubeClient := NewKubernetesClientset(t)
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	pods, err := kubeClient.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector})
+	require.NoError(t, err)
+
+	restartCounts := make(map[string]int32)
+	for _, pod := range pods.Items {
+		for _, container := range pod.Status.ContainerStatuses {
+			key := fmt.Sprintf("%s/%s/%s", pod.Namespace, pod.Name, container.Name)
+			restartCounts[key] = container.RestartCount
+		}
+	}
+
+	return restartCounts
+}