Merge pull request #407 from ankeesler/test-flake

test/integration: make TestKubeCertAgent more stable

test/integration/concierge_kubecertagent_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/diff"
+	"k8s.io/apimachinery/pkg/util/wait"
 
 	"go.pinniped.dev/test/library"
 )
@@ -90,6 +91,9 @@ func TestKubeCertAgent(t *testing.T) {
 	}
 
 	t.Run("reconcile on update", func(t *testing.T) {
+		// Ensure that the next test will start from a known state.
+		defer ensureKubeCertAgentSteadyState(t, agentPodsReconciled)
+
 		// Update the image of the first pod. The controller should see it, and flip it back.
 		//
 		// Note that we update the toleration field here because it is the only field, currently, that
@@ -109,6 +113,9 @@ func TestKubeCertAgent(t *testing.T) {
 	})
 
 	t.Run("reconcile on delete", func(t *testing.T) {
+		// Ensure that the next test will start from a known state.
+		defer ensureKubeCertAgentSteadyState(t, agentPodsReconciled)
+
 		// Delete the first pod. The controller should see it, and flip it back.
 		err = kubeClient.
 			CoreV1().
@@ -122,6 +129,21 @@ func TestKubeCertAgent(t *testing.T) {
 	})
 }
 
+func ensureKubeCertAgentSteadyState(t *testing.T, agentPodsReconciled func() bool) {
+	t.Helper()
+
+	const wantSteadyStateSnapshots = 3
+	var steadyStateSnapshots int
+	require.NoError(t, wait.Poll(250*time.Millisecond, 30*time.Second, func() (bool, error) {
+		if agentPodsReconciled() {
+			steadyStateSnapshots++
+		} else {
+			steadyStateSnapshots = 0
+		}
+		return steadyStateSnapshots == wantSteadyStateSnapshots, nil
+	}))
+}
+
 func sortPods(pods *corev1.PodList) {
 	sort.Slice(pods.Items, func(i, j int) bool {
 		return pods.Items[i].Name < pods.Items[j].Name