// Copyright 2020-2023 the Pinniped contributors. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 package integration import ( "bytes" "context" "errors" "fmt" "os/exec" "regexp" "strings" "testing" "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/discovery" "go.pinniped.dev/test/testlib" ) func TestGetAPIResourceList(t *testing.T) { //nolint:gocyclo // each t.Run is pretty simple, but there are many env := testlib.IntegrationEnv(t) client := testlib.NewKubernetesClientset(t) groups, resources, err := client.Discovery().ServerGroupsAndResources() // discovery can have partial failures when an API service is unavailable (i.e. because of TestAPIServingCertificateAutoCreationAndRotation) // we ignore failures for groups that are not relevant to this test if err != nil { discoveryFailed := &discovery.ErrGroupDiscoveryFailed{} isDiscoveryFailed := errors.As(err, &discoveryFailed) require.True(t, isDiscoveryFailed, err) for gv, gvErr := range discoveryFailed.Groups { if strings.HasSuffix(gv.Group, "."+env.APIGroupSuffix) { require.NoError(t, gvErr) } } } makeGV := func(firstSegment, secondSegment string) schema.GroupVersion { return schema.GroupVersion{ Group: fmt.Sprintf("%s.%s.%s", firstSegment, secondSegment, env.APIGroupSuffix), Version: "v1alpha1", } } loginConciergeGV := makeGV("login", "concierge") identityConciergeGV := makeGV("identity", "concierge") authenticationConciergeGV := makeGV("authentication", "concierge") configConciergeGV := makeGV("config", "concierge") idpSupervisorGV := makeGV("idp", "supervisor") configSupervisorGV := makeGV("config", "supervisor") clientSecretSupervisorGV := makeGV("clientsecret", "supervisor") tests := []struct { group metav1.APIGroup resourceByVersion map[string][]metav1.APIResource }{ { group: metav1.APIGroup{ Name: loginConciergeGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: loginConciergeGV.String(), Version: loginConciergeGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: loginConciergeGV.String(), Version: loginConciergeGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ loginConciergeGV.String(): { { Name: "tokencredentialrequests", SingularName: "tokencredentialrequest", Kind: "TokenCredentialRequest", Verbs: []string{"create", "list"}, Namespaced: false, Categories: []string{"pinniped"}, }, }, }, }, { group: metav1.APIGroup{ Name: identityConciergeGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: identityConciergeGV.String(), Version: identityConciergeGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: identityConciergeGV.String(), Version: identityConciergeGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ identityConciergeGV.String(): { { Name: "whoamirequests", SingularName: "whoamirequest", Kind: "WhoAmIRequest", Verbs: []string{"create", "list"}, Namespaced: false, Categories: []string{"pinniped"}, }, }, }, }, { group: metav1.APIGroup{ Name: clientSecretSupervisorGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: clientSecretSupervisorGV.String(), Version: clientSecretSupervisorGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: clientSecretSupervisorGV.String(), Version: clientSecretSupervisorGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ clientSecretSupervisorGV.String(): { { Name: "oidcclientsecretrequests", SingularName: "oidcclientsecretrequest", Kind: "OIDCClientSecretRequest", Verbs: []string{"create", "list"}, Namespaced: true, Categories: []string{"pinniped"}, }, }, }, }, { group: metav1.APIGroup{ Name: configSupervisorGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: configSupervisorGV.String(), Version: configSupervisorGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: configSupervisorGV.String(), Version: configSupervisorGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ configSupervisorGV.String(): { { Name: "federationdomains", SingularName: "federationdomain", Namespaced: true, Kind: "FederationDomain", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped"}, }, { Name: "federationdomains/status", Namespaced: true, Kind: "FederationDomain", Verbs: []string{"get", "patch", "update"}, }, { Name: "oidcclients", SingularName: "oidcclient", Namespaced: true, Kind: "OIDCClient", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped"}, }, { Name: "oidcclients/status", Namespaced: true, Kind: "OIDCClient", Verbs: []string{"get", "patch", "update"}, }, }, }, }, { group: metav1.APIGroup{ Name: idpSupervisorGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: idpSupervisorGV.String(), Version: idpSupervisorGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: idpSupervisorGV.String(), Version: idpSupervisorGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ idpSupervisorGV.String(): { { Name: "oidcidentityproviders", SingularName: "oidcidentityprovider", Namespaced: true, Kind: "OIDCIdentityProvider", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped", "pinniped-idp", "pinniped-idps"}, }, { Name: "oidcidentityproviders/status", Namespaced: true, Kind: "OIDCIdentityProvider", Verbs: []string{"get", "patch", "update"}, }, { Name: "ldapidentityproviders", SingularName: "ldapidentityprovider", Namespaced: true, Kind: "LDAPIdentityProvider", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped", "pinniped-idp", "pinniped-idps"}, }, { Name: "ldapidentityproviders/status", Namespaced: true, Kind: "LDAPIdentityProvider", Verbs: []string{"get", "patch", "update"}, }, { Name: "activedirectoryidentityproviders", SingularName: "activedirectoryidentityprovider", Namespaced: true, Kind: "ActiveDirectoryIdentityProvider", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped", "pinniped-idp", "pinniped-idps"}, }, { Name: "activedirectoryidentityproviders/status", Namespaced: true, Kind: "ActiveDirectoryIdentityProvider", Verbs: []string{"get", "patch", "update"}, }, }, }, }, { group: metav1.APIGroup{ Name: configConciergeGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: configConciergeGV.String(), Version: configConciergeGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: configConciergeGV.String(), Version: configConciergeGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ configConciergeGV.String(): { { Name: "credentialissuers", SingularName: "credentialissuer", Namespaced: false, Kind: "CredentialIssuer", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped"}, }, { Name: "credentialissuers/status", Namespaced: false, Kind: "CredentialIssuer", Verbs: []string{"get", "patch", "update"}, }, }, }, }, { group: metav1.APIGroup{ Name: authenticationConciergeGV.Group, Versions: []metav1.GroupVersionForDiscovery{ { GroupVersion: authenticationConciergeGV.String(), Version: authenticationConciergeGV.Version, }, }, PreferredVersion: metav1.GroupVersionForDiscovery{ GroupVersion: authenticationConciergeGV.String(), Version: authenticationConciergeGV.Version, }, }, resourceByVersion: map[string][]metav1.APIResource{ authenticationConciergeGV.String(): { { Name: "webhookauthenticators", SingularName: "webhookauthenticator", Namespaced: false, Kind: "WebhookAuthenticator", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped", "pinniped-authenticator", "pinniped-authenticators"}, }, { Name: "webhookauthenticators/status", Namespaced: false, Kind: "WebhookAuthenticator", Verbs: []string{"get", "patch", "update"}, }, { Name: "jwtauthenticators", SingularName: "jwtauthenticator", Namespaced: false, Kind: "JWTAuthenticator", Verbs: []string{"delete", "deletecollection", "get", "list", "patch", "create", "update", "watch"}, Categories: []string{"pinniped", "pinniped-authenticator", "pinniped-authenticators"}, }, { Name: "jwtauthenticators/status", Namespaced: false, Kind: "JWTAuthenticator", Verbs: []string{"get", "patch", "update"}, }, }, }, }, } t.Run("every Pinniped API has explicit test coverage", func(t *testing.T) { t.Parallel() testedGroups := map[string]bool{} for _, tt := range tests { testedGroups[tt.group.Name] = true } foundPinnipedGroups := 0 for _, g := range groups { if !strings.Contains(g.Name, env.APIGroupSuffix) { continue } foundPinnipedGroups++ assert.Truef(t, testedGroups[g.Name], "expected group %q to have assertions defined", g.Name) } require.Equal(t, len(testedGroups), foundPinnipedGroups) }) t.Run("every API categorized appropriately", func(t *testing.T) { t.Parallel() for _, r := range resources { if !strings.Contains(r.GroupVersion, env.APIGroupSuffix) { continue } for _, a := range r.APIResources { if strings.HasSuffix(a.Name, "/status") { continue } assert.Containsf(t, a.Categories, "pinniped", "expected resource %q to be in the 'pinniped' category", a.Name) assert.NotContainsf(t, a.Categories, "all", "expected resource %q not to be in the 'all' category", a.Name) } } }) t.Run("every concierge API is cluster scoped", func(t *testing.T) { t.Parallel() for _, r := range resources { if !strings.Contains(r.GroupVersion, env.APIGroupSuffix) { continue } if !strings.Contains(r.GroupVersion, ".concierge.") { continue } for _, a := range r.APIResources { assert.False(t, a.Namespaced, "concierge APIs must be cluster scoped: %#v", a) } } }) t.Run("every API has a status subresource", func(t *testing.T) { t.Parallel() aggregatedAPIs := sets.NewString("tokencredentialrequests", "whoamirequests", "oidcclientsecretrequests") var regular, status []string for _, r := range resources { if !strings.Contains(r.GroupVersion, env.APIGroupSuffix) { continue } for _, a := range r.APIResources { if aggregatedAPIs.Has(a.Name) { continue // skip our special aggregated APIs with their own magical properties } if strings.HasSuffix(a.Name, "/status") { status = append(status, strings.TrimSuffix(a.Name, "/status")) } else { regular = append(regular, a.Name) } } } assert.Equal(t, regular, status) }) t.Run("every API can show its docs to the user via kubectl explain, including aggregated APIs, and everything has a description", func(t *testing.T) { t.Parallel() for _, r := range resources { if !strings.Contains(r.GroupVersion, env.APIGroupSuffix) { continue } for _, a := range r.APIResources { if strings.HasSuffix(a.Name, "/status") { // skip status subresources for this test, as they don't work with `kubectl explain` continue } // Note that this test might indirectly depend on the kubectl discovery cache, found in $HOME/.kube/cache/discovery. // If you are working on changing API type struct comments, then you may need to clear your discovery cache // (or wait ~10 minutes for the cache to expire) for the new comments to appear in the `kubectl explain` results. requireKubectlExplainShowsDescriptionForResource(t, a.Name, a.Kind, r.GroupVersion) } } }) t.Run("Pinniped resources do not have short names", func(t *testing.T) { t.Parallel() for _, r := range resources { if !strings.Contains(r.GroupVersion, env.APIGroupSuffix) { continue } for _, a := range r.APIResources { assert.Empty(t, a.ShortNames, "expected resource %q not to have any short names", a.Name) } } }) for _, tt := range tests { tt := tt t.Run(tt.group.Name, func(t *testing.T) { t.Parallel() require.Contains(t, groups, &tt.group) for groupVersion, expectedResources := range tt.resourceByVersion { // Find the actual resource list and make a copy. var actualResourceList *metav1.APIResourceList for _, resource := range resources { if resource.GroupVersion == groupVersion { actualResourceList = resource.DeepCopy() } } require.NotNilf(t, actualResourceList, "could not find groupVersion %s", groupVersion) for i := range actualResourceList.APIResources { // Because its hard to predict the storage version hash (e.g. "t/+v41y+3e4="), we just don't // worry about comparing that field. actualResourceList.APIResources[i].StorageVersionHash = "" // These fields were empty for a long time but started to be non-empty at some Kubernetes version. // The filled-in fields were first noticed when CI tested against a 1.27 pre-release. // To make this test pass on all versions of Kube, just ignore these fields for now. actualResourceList.APIResources[i].Group = "" actualResourceList.APIResources[i].Version = "" if strings.HasSuffix(actualResourceList.APIResources[i].Name, "/status") { actualResourceList.APIResources[i].SingularName = "" } } require.ElementsMatch(t, expectedResources, actualResourceList.APIResources, "unexpected API resources") } }) } } // safe to run in parallel with serial tests since it only reads CRDs, see main_test.go. func TestCRDAdditionalPrinterColumns_Parallel(t *testing.T) { // AdditionalPrinterColumns can be set on a CRD to make `kubectl get` return those columns in its table output. // The main purpose of this test is to fail when we add a new CRD without considering which // AdditionalPrinterColumns to set on it. This test will force us to consider it and make an explicit choice. env := testlib.IntegrationEnv(t) ctx, cancelFunc := context.WithTimeout(context.Background(), time.Minute) defer cancelFunc() // AdditionalPrinterColumns are not returned by the Kube discovery endpoints, // so "discover" them in the CRD definitions instead. apiExtensionsV1Client := testlib.NewAPIExtensionsV1Client(t) crdList, err := apiExtensionsV1Client.CustomResourceDefinitions().List(ctx, metav1.ListOptions{}) require.NoError(t, err) addSuffix := func(base string) string { return base + "." + env.APIGroupSuffix } // Since we're checking that AdditionalPrinterColumns exists on every CRD then we might as well also // assert which fields are set as AdditionalPrinterColumns. // Ideally, every CRD should show some kind of identifying info, some kind of status, and Age. expectedColumnsPerCRDVersion := map[string]map[string][]apiextensionsv1.CustomResourceColumnDefinition{ addSuffix("credentialissuers.config.concierge"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "ProxyMode", Type: "string", JSONPath: ".spec.impersonationProxy.mode"}, // CredentialIssuer status is a list of strategies, each with its own status. Unfortunately, // AdditionalPrinterColumns cannot show multiple results, e.g. a list of strategy types where // the status is equal to Successful. See https://github.com/kubernetes/kubernetes/issues/67268. // Although this selector can evaluate to multiple results, the Kube CRD implementation of JSONPath // will always only show the first result. Thus, this column will show the first successful strategy // type, which is the same thing that `pinniped get kubeconfig` looks for, so the value of this // column represents the current default strategy that will be used by `pinniped get kubeconfig`. {Name: "DefaultStrategy", Type: "string", JSONPath: `.status.strategies[?(@.status == "Success")].type`}, {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("webhookauthenticators.authentication.concierge"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Endpoint", Type: "string", JSONPath: ".spec.endpoint"}, // Note that WebhookAuthenticators have a status type, but no controller currently sets the status, so we don't show it. {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("jwtauthenticators.authentication.concierge"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Issuer", Type: "string", JSONPath: ".spec.issuer"}, {Name: "Audience", Type: "string", JSONPath: ".spec.audience"}, // Note that JWTAuthenticators have a status type, but no controller currently sets the status, so we don't show it. {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("activedirectoryidentityproviders.idp.supervisor"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Host", Type: "string", JSONPath: ".spec.host"}, {Name: "Status", Type: "string", JSONPath: ".status.phase"}, {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("federationdomains.config.supervisor"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Issuer", Type: "string", JSONPath: ".spec.issuer"}, {Name: "Status", Type: "string", JSONPath: ".status.status"}, {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("ldapidentityproviders.idp.supervisor"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Host", Type: "string", JSONPath: ".spec.host"}, {Name: "Status", Type: "string", JSONPath: ".status.phase"}, {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("oidcidentityproviders.idp.supervisor"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Issuer", Type: "string", JSONPath: ".spec.issuer"}, {Name: "Status", Type: "string", JSONPath: ".status.phase"}, {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, addSuffix("oidcclients.config.supervisor"): { "v1alpha1": []apiextensionsv1.CustomResourceColumnDefinition{ {Name: "Privileged Scopes", Type: "string", JSONPath: `.spec.allowedScopes[?(@ == "pinniped:request-audience")]`}, {Name: "Client Secrets", Type: "integer", JSONPath: ".status.totalClientSecrets"}, {Name: "Status", Type: "string", JSONPath: ".status.phase"}, {Name: "Age", Type: "date", JSONPath: ".metadata.creationTimestamp"}, }, }, } actualPinnipedCRDCount := 0 expectedPinnipedCRDCount := 8 // the current number of CRDs that we ship as part of Pinniped for _, crd := range crdList.Items { if !strings.Contains(crd.Spec.Group, env.APIGroupSuffix) { continue // skip non-Pinniped CRDs } // Found a Pinniped CRD, so let's check it for AdditionalPrinterColumns. actualPinnipedCRDCount++ for _, version := range crd.Spec.Versions { expectedColumns, ok := expectedColumnsPerCRDVersion[crd.Name][version.Name] assert.Truef(t, ok, "should have found an expected AdditionalPrinterColumns for CRD %q version %q: "+ "please make sure that some useful AdditionalPrinterColumns are defined on the CRD and update this test's expectations", crd.Name, version.Name) assert.Equalf(t, expectedColumns, version.AdditionalPrinterColumns, "CRD %q version %q had unexpected AdditionalPrinterColumns", crd.Name, version.Name) } } // Make sure that the logic of this test did not accidentally skip a CRD that it should have interrogated. require.Equal(t, expectedPinnipedCRDCount, actualPinnipedCRDCount, "did not find expected number of Pinniped CRDs to check for additionalPrinterColumns") } func requireKubectlExplainShowsDescriptionForResource(t *testing.T, resourceName string, resourceKind string, resourceGroupVersion string) { // Run kubectl explain on the resource. output := runKubectlExplain(t, resourceName, resourceGroupVersion) // Check that the output is as expected. require.Regexp(t, `(?m)^KIND:\s+`+regexp.QuoteMeta(resourceKind)+`$`, output) require.Regexp(t, `(?m)^VERSION:\s+`+regexp.QuoteMeta(resourceGroupVersion)+`$`, output) require.Regexp(t, `(?m)^DESCRIPTION:$`, output) // Use assert here so that the test keeps running when a description is empty, so we can find all the empty descriptions. assert.NotRegexp(t, `(?m)^\s*\s*$`, output, "resource or field should not have an empty description in kubectl explain") if strings.Contains(output, "\nFIELD: ") { // We must have explained a leaf field, which has no children fields. return } if resourceName == "whoamirequests.spec" { // This is an exception because this field is declared to be an empty struct in its type definition. It is // not a leaf field because it is a struct, but it also has no children because the struct contains no fields. // So it has neither the `FIELD:` section nor the `FIELDS:` section in the output. return } // Otherwise, we must have explained a resource or field which has children fields, so it should have a fields list. require.Contains(t, output, "\nFIELDS:\n") // Grab everything after the line that says `FIELDS:`. fieldsSectionMatches := regexp.MustCompile(`(?s).+\nFIELDS:\n(.+)`).FindStringSubmatch(output) require.Len(t, fieldsSectionMatches, 2) allFieldsDescribedText := fieldsSectionMatches[1] // Grab the names of all the fields from the fields description. fieldNames := []string{} for _, line := range strings.Split(allFieldsDescribedText, "\n") { if strings.HasPrefix(line, " ") { // Field names are indented by exactly three spaces. // Skip lines that are indented deeper, which are field descriptions. continue } if len(strings.TrimSpace(line)) == 0 { // Ignore empty lines. continue } // Field name lines start with 3 spaces, then the field name, then some tabs/spaces, then the field type. // Grab just the field name. fieldsNameMatches := regexp.MustCompile(`^ {3}(\S+)\s+`).FindStringSubmatch(line) require.Len(t, fieldsNameMatches, 2, fmt.Sprintf("field name line which did not match: %s", line)) fieldNames = append(fieldNames, fieldsNameMatches[1]) } require.Greater(t, len(fieldNames), 0, "should have found some field names in the kubectl explain output, but didn't find any") // For each field, check to see that docs were provided for that field by making a recursive call to this function. for _, fieldName := range fieldNames { if fieldName == "kind" || fieldName == "metadata" || fieldName == "apiVersion" { // Skip these since the docs are implemented by k8s packages, so we can assume that they are correct. continue } requireKubectlExplainShowsDescriptionForResource(t, fmt.Sprintf("%s.%s", resourceName, fieldName), resourceKind, resourceGroupVersion) } } func runKubectlExplain(t *testing.T, resourceName string, apiVersion string) string { t.Helper() var stdOut, stdErr bytes.Buffer cmd := exec.Command("kubectl", "explain", resourceName, "--api-version", apiVersion) t.Log("Running:", cmd.String()) cmd.Stdout = &stdOut cmd.Stderr = &stdErr err := cmd.Run() var exitErr *exec.ExitError if errors.As(err, &exitErr) { t.Logf("Running kubectl explain had non-zero exit code."+ "\nkubectl explain stdout: %s\nkubectl explain stderr: %s", stdOut.String(), stdErr.String()) } require.NoError(t, err) return stdOut.String() }