From 12d5b8959d8aa9faf1317bbc7b4f1e6161a91258 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Wed, 10 Feb 2021 11:25:08 -0500 Subject: [PATCH] test/integration: make TestKubeCertAgent more stable I think the reason we were seeing flakes here is because the kube cert agent pods had not reached a steady state even though our test assertions passed, so the test would proceed immediately and run more assertions on top of a weird state of the kube cert agent pods. Signed-off-by: Andrew Keesler --- .../concierge_kubecertagent_test.go | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/test/integration/concierge_kubecertagent_test.go b/test/integration/concierge_kubecertagent_test.go index 73d9d4f6..37392013 100644 --- a/test/integration/concierge_kubecertagent_test.go +++ b/test/integration/concierge_kubecertagent_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/wait" "go.pinniped.dev/test/library" ) @@ -90,6 +91,9 @@ func TestKubeCertAgent(t *testing.T) { } t.Run("reconcile on update", func(t *testing.T) { + // Ensure that the next test will start from a known state. + defer ensureKubeCertAgentSteadyState(t, agentPodsReconciled) + // Update the image of the first pod. The controller should see it, and flip it back. // // Note that we update the toleration field here because it is the only field, currently, that @@ -109,6 +113,9 @@ func TestKubeCertAgent(t *testing.T) { }) t.Run("reconcile on delete", func(t *testing.T) { + // Ensure that the next test will start from a known state. + defer ensureKubeCertAgentSteadyState(t, agentPodsReconciled) + // Delete the first pod. The controller should see it, and flip it back. err = kubeClient. CoreV1(). @@ -122,6 +129,21 @@ func TestKubeCertAgent(t *testing.T) { }) } +func ensureKubeCertAgentSteadyState(t *testing.T, agentPodsReconciled func() bool) { + t.Helper() + + const wantSteadyStateSnapshots = 3 + var steadyStateSnapshots int + require.NoError(t, wait.Poll(250*time.Millisecond, 30*time.Second, func() (bool, error) { + if agentPodsReconciled() { + steadyStateSnapshots++ + } else { + steadyStateSnapshots = 0 + } + return steadyStateSnapshots == wantSteadyStateSnapshots, nil + })) +} + func sortPods(pods *corev1.PodList) { sort.Slice(pods.Items, func(i, j int) bool { return pods.Items[i].Name < pods.Items[j].Name