djpbessems/ContainerImage.Pinniped
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve the reliability of TestGetPinnipedCategory.

Browse Source 
This test could flake in some rare scenarios. This change adds a bunch of retries, improves the debugging output if the tests fail, and puts all of the subtests in parallel which saves ~10s on my local machine.

Signed-off-by: Matt Moyer <moyerm@vmware.com>
This commit is contained in:
Matt Moyer 2021-03-16 17:22:28 -05:00
parent 4ab3c64b70
commit 30a392b900
No known key found for this signature in database
GPG Key ID: EAE88AD172C5AE2D
1 changed files with 75 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
test/integration/category_test.go
View File

@ -6,118 +6,110 @@ package integration
import ( import (
"bytes" "bytes"
"os/exec" "os/exec"
"strings"
"sync"
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.pinniped.dev/test/library" "go.pinniped.dev/test/library"
) )
func runTestKubectlCommand(t *testing.T, args ...string) (string, string) {
t.Helper()
var lock sync.Mutex
var stdOut, stdErr bytes.Buffer
var err error
start := time.Now()
attempts := 0
if !assert.Eventually(t, func() bool {
lock.Lock()
defer lock.Unlock()
attempts++
stdOut.Reset()
stdErr.Reset()
cmd := exec.Command("kubectl", args...)
cmd.Stdout = &stdOut
cmd.Stderr = &stdErr
err = cmd.Run()
return err == nil
},
120*time.Second,
200*time.Millisecond,
) {
lock.Lock()
defer lock.Unlock()
t.Logf(
"never ran %q successfully even after %d attempts (%s)",
"kubectl "+strings.Join(args, " "),
attempts,
time.Since(start).Round(time.Second),
)
t.Logf("last error: %v", err)
t.Logf("stdout:\n%s\n", stdOut.String())
t.Logf("stderr:\n%s\n", stdErr.String())
t.FailNow()
}
return stdOut.String(), stdErr.String()
}
func TestGetPinnipedCategory(t *testing.T) { func TestGetPinnipedCategory(t *testing.T) {
env := library.IntegrationEnv(t) env := library.IntegrationEnv(t)
dotSuffix := "." + env.APIGroupSuffix dotSuffix := "." + env.APIGroupSuffix
t.Run("category, no special params", func(t *testing.T) { t.Run("category, no special params", func(t *testing.T) {
var stdOut, stdErr bytes.Buffer t.Parallel()
stdout, stderr := runTestKubectlCommand(t, "get", "pinniped", "-A")
var err error require.Empty(t, stderr)
require.Eventuallyf(t, func() bool { require.NotContains(t, stdout, "MethodNotAllowed")
cmd := exec.Command("kubectl", "get", "pinniped", "-A") require.Contains(t, stdout, dotSuffix)
cmd.Stdout = &stdOut
cmd.Stderr = &stdErr
err = cmd.Run()
return err == nil
},
60*time.Second,
1*time.Second,
"never ran 'kubectl get pinniped -A' successfully:\n%s\n\n%s",
stdErr.String(),
stdOut.String(),
)
require.Empty(t, stdErr.String())
require.NotContains(t, stdOut.String(), "MethodNotAllowed")
require.Contains(t, stdOut.String(), dotSuffix)
}) })
t.Run("category, table params", func(t *testing.T) { t.Run("category, table params", func(t *testing.T) {
var stdOut, stdErr bytes.Buffer t.Parallel()
stdout, stderr := runTestKubectlCommand(t, "get", "pinniped", "-A", "-o", "wide", "-v", "10")
cmd := exec.Command("kubectl", "get", "pinniped", "-A", "-o", "wide", "-v", "10") require.NotContains(t, stdout, "MethodNotAllowed")
cmd.Stdout = &stdOut require.Contains(t, stdout, dotSuffix)
cmd.Stderr = &stdErr require.Contains(t, stderr, `"kind":"Table"`)
err := cmd.Run() require.Contains(t, stderr, `"resourceVersion":"0"`)
require.NoError(t, err, stdErr.String(), stdOut.String()) require.Contains(t, stderr, `/v1alpha1/tokencredentialrequests`)
require.Contains(t, stderr, `/v1alpha1/whoamirequests`)
require.NotContains(t, stdOut.String(), "MethodNotAllowed")
require.Contains(t, stdOut.String(), dotSuffix)
require.Contains(t, stdErr.String(), `"kind":"Table"`)
require.Contains(t, stdErr.String(), `"resourceVersion":"0"`)
require.Contains(t, stdErr.String(), `/v1alpha1/tokencredentialrequests`)
require.Contains(t, stdErr.String(), `/v1alpha1/whoamirequests`)
}) })
t.Run("list, no special params", func(t *testing.T) { t.Run("list, no special params", func(t *testing.T) {
var stdOut, stdErr bytes.Buffer t.Parallel()
stdout, stderr := runTestKubectlCommand(t, "get", "tokencredentialrequests.login.concierge"+dotSuffix, "-A")
//nolint: gosec // input is part of test env require.Empty(t, stdout)
cmd := exec.Command("kubectl", "get", "tokencredentialrequests.login.concierge"+dotSuffix, "-A") require.NotContains(t, stderr, "MethodNotAllowed")
cmd.Stdout = &stdOut require.Contains(t, stderr, `No resources found`)
cmd.Stderr = &stdErr
err := cmd.Run()
require.NoError(t, err, stdErr.String(), stdOut.String())
require.Empty(t, stdOut.String())
require.NotContains(t, stdErr.String(), "MethodNotAllowed")
require.Contains(t, stdErr.String(), `No resources found`)
}) })
t.Run("list, table params", func(t *testing.T) { t.Run("list, table params", func(t *testing.T) {
var stdOut, stdErr bytes.Buffer t.Parallel()
stdout, stderr := runTestKubectlCommand(t, "get", "tokencredentialrequests.login.concierge"+dotSuffix, "-A", "-o", "wide", "-v", "10")
//nolint: gosec // input is part of test env require.Empty(t, stdout)
cmd := exec.Command("kubectl", "get", "tokencredentialrequests.login.concierge"+dotSuffix, "-A", "-o", "wide", "-v", "10") require.NotContains(t, stderr, "MethodNotAllowed")
cmd.Stdout = &stdOut require.Contains(t, stderr, `"kind":"Table"`)
cmd.Stderr = &stdErr require.Contains(t, stderr, `"resourceVersion":"0"`)
err := cmd.Run()
require.NoError(t, err, stdErr.String(), stdOut.String())
require.Empty(t, stdOut.String())
require.NotContains(t, stdErr.String(), "MethodNotAllowed")
require.Contains(t, stdErr.String(), `"kind":"Table"`)
require.Contains(t, stdErr.String(), `"resourceVersion":"0"`)
}) })
t.Run("raw request to see body, token cred", func(t *testing.T) { t.Run("raw request to see body, token cred", func(t *testing.T) {
var stdOut, stdErr bytes.Buffer t.Parallel()
stdout, stderr := runTestKubectlCommand(t, "get", "--raw", "/apis/login.concierge"+dotSuffix+"/v1alpha1/tokencredentialrequests")
//nolint: gosec // input is part of test env require.NotContains(t, stdout, "MethodNotAllowed")
cmd := exec.Command("kubectl", "get", "--raw", "/apis/login.concierge"+dotSuffix+"/v1alpha1/tokencredentialrequests") require.Contains(t, stdout, `{"kind":"TokenCredentialRequestList","apiVersion":"login.concierge`+
cmd.Stdout = &stdOut
cmd.Stderr = &stdErr
err := cmd.Run()
require.NoError(t, err, stdErr.String(), stdOut.String())
require.Empty(t, stdErr.String())
require.NotContains(t, stdOut.String(), "MethodNotAllowed")
require.Contains(t, stdOut.String(), `{"kind":"TokenCredentialRequestList","apiVersion":"login.concierge`+
dotSuffix+`/v1alpha1","metadata":{"resourceVersion":"0"},"items":[]}`) dotSuffix+`/v1alpha1","metadata":{"resourceVersion":"0"},"items":[]}`)
require.Empty(t, stderr)
}) })
t.Run("raw request to see body, whoami", func(t *testing.T) { t.Run("raw request to see body, whoami", func(t *testing.T) {
var stdOut, stdErr bytes.Buffer t.Parallel()
stdout, stderr := runTestKubectlCommand(t, "get", "--raw", "/apis/identity.concierge"+dotSuffix+"/v1alpha1/whoamirequests")
//nolint: gosec // input is part of test env require.NotContains(t, stdout, "MethodNotAllowed")
cmd := exec.Command("kubectl", "get", "--raw", "/apis/identity.concierge"+dotSuffix+"/v1alpha1/whoamirequests") require.Contains(t, stdout, `{"kind":"WhoAmIRequestList","apiVersion":"identity.concierge`+
cmd.Stdout = &stdOut
cmd.Stderr = &stdErr
err := cmd.Run()
require.NoError(t, err, stdErr.String(), stdOut.String())
require.Empty(t, stdErr.String())
require.NotContains(t, stdOut.String(), "MethodNotAllowed")
require.Contains(t, stdOut.String(), `{"kind":"WhoAmIRequestList","apiVersion":"identity.concierge`+
dotSuffix+`/v1alpha1","metadata":{"resourceVersion":"0"},"items":[]}`) dotSuffix+`/v1alpha1","metadata":{"resourceVersion":"0"},"items":[]}`)
require.Empty(t, stderr)
}) })
} }