1056cef384
- Only sync on add/update of secrets in the same namespace which have the "storage.pinniped.dev/garbage-collect-after" annotation, and also during a full resync of the informer whenever secrets in the same namespace with that annotation exist. - Ignore deleted secrets to avoid having this controller trigger itself unnecessarily when it deletes a secret. This controller is never interested in deleted secrets, since its only job is to delete existing secrets. - No change to the self-imposed rate limit logic. That still applies because secrets with this annotation will be created and updated regularly while the system is running (not just during rare system configuration steps).
384 lines
14 KiB
Go
384 lines
14 KiB
Go
// Copyright 2020 the Pinniped contributors. All Rights Reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
package supervisorstorage
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/sclevine/spec"
|
|
"github.com/sclevine/spec/report"
|
|
"github.com/stretchr/testify/require"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/util/clock"
|
|
kubeinformers "k8s.io/client-go/informers"
|
|
kubernetesfake "k8s.io/client-go/kubernetes/fake"
|
|
kubetesting "k8s.io/client-go/testing"
|
|
|
|
"go.pinniped.dev/internal/controllerlib"
|
|
"go.pinniped.dev/internal/testutil"
|
|
)
|
|
|
|
func TestGarbageCollectorControllerInformerFilters(t *testing.T) {
|
|
spec.Run(t, "informer filters", func(t *testing.T, when spec.G, it spec.S) {
|
|
var (
|
|
r *require.Assertions
|
|
observableWithInformerOption *testutil.ObservableWithInformerOption
|
|
secretsInformerFilter controllerlib.Filter
|
|
)
|
|
|
|
it.Before(func() {
|
|
r = require.New(t)
|
|
observableWithInformerOption = testutil.NewObservableWithInformerOption()
|
|
secretsInformer := kubeinformers.NewSharedInformerFactory(nil, 0).Core().V1().Secrets()
|
|
_ = GarbageCollectorController(
|
|
clock.RealClock{},
|
|
nil,
|
|
secretsInformer,
|
|
observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters
|
|
)
|
|
secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer)
|
|
})
|
|
|
|
when("watching Secret objects", func() {
|
|
var (
|
|
subject controllerlib.Filter
|
|
secretWithAnnotation, otherSecret *corev1.Secret
|
|
)
|
|
|
|
it.Before(func() {
|
|
subject = secretsInformerFilter
|
|
secretWithAnnotation = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "any-name", Namespace: "any-namespace", Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": "some timestamp",
|
|
}}}
|
|
otherSecret = &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "any-other-name", Namespace: "any-namespace"}}
|
|
})
|
|
|
|
when("any Secret with the required annotation is added or updated", func() {
|
|
it("returns true to trigger the sync function", func() {
|
|
r.True(subject.Add(secretWithAnnotation))
|
|
r.True(subject.Update(secretWithAnnotation, otherSecret))
|
|
r.True(subject.Update(otherSecret, secretWithAnnotation))
|
|
})
|
|
})
|
|
|
|
when("any Secret with the required annotation is deleted", func() {
|
|
it("returns false to skip the sync function because it does not need to worry about secrets that are already gone", func() {
|
|
r.False(subject.Delete(secretWithAnnotation))
|
|
})
|
|
})
|
|
|
|
when("any Secret without the required annotation changes", func() {
|
|
it("returns false to skip the sync function", func() {
|
|
r.False(subject.Add(otherSecret))
|
|
r.False(subject.Update(otherSecret, otherSecret))
|
|
r.False(subject.Delete(otherSecret))
|
|
})
|
|
})
|
|
|
|
when("any other type is passed", func() {
|
|
it("returns false to skip the sync function", func() {
|
|
wrongType := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "some-ns", Namespace: "some-ns"}}
|
|
|
|
r.False(subject.Add(wrongType))
|
|
r.False(subject.Update(wrongType, wrongType))
|
|
r.False(subject.Delete(wrongType))
|
|
})
|
|
})
|
|
})
|
|
}, spec.Parallel(), spec.Report(report.Terminal{}))
|
|
}
|
|
|
|
func TestGarbageCollectorControllerSync(t *testing.T) {
|
|
secretsGVR := schema.GroupVersionResource{
|
|
Group: "",
|
|
Version: "v1",
|
|
Resource: "secrets",
|
|
}
|
|
|
|
spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) {
|
|
const (
|
|
installedInNamespace = "some-namespace"
|
|
)
|
|
|
|
var (
|
|
r *require.Assertions
|
|
subject controllerlib.Controller
|
|
kubeInformerClient *kubernetesfake.Clientset
|
|
kubeClient *kubernetesfake.Clientset
|
|
kubeInformers kubeinformers.SharedInformerFactory
|
|
timeoutContext context.Context
|
|
timeoutContextCancel context.CancelFunc
|
|
syncContext *controllerlib.Context
|
|
fakeClock *clock.FakeClock
|
|
frozenNow time.Time
|
|
)
|
|
|
|
// Defer starting the informers until the last possible moment so that the
|
|
// nested Before's can keep adding things to the informer caches.
|
|
var startInformersAndController = func() {
|
|
// Set this at the last second to allow for injection of server override.
|
|
subject = GarbageCollectorController(
|
|
fakeClock,
|
|
kubeClient,
|
|
kubeInformers.Core().V1().Secrets(),
|
|
controllerlib.WithInformer,
|
|
)
|
|
|
|
// Set this at the last second to support calling subject.Name().
|
|
syncContext = &controllerlib.Context{
|
|
Context: timeoutContext,
|
|
Name: subject.Name(),
|
|
Key: controllerlib.Key{
|
|
Namespace: "",
|
|
Name: "",
|
|
},
|
|
}
|
|
|
|
// Must start informers before calling TestRunSynchronously()
|
|
kubeInformers.Start(timeoutContext.Done())
|
|
controllerlib.TestRunSynchronously(t, subject)
|
|
}
|
|
|
|
it.Before(func() {
|
|
r = require.New(t)
|
|
|
|
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
|
|
|
|
kubeInformerClient = kubernetesfake.NewSimpleClientset()
|
|
kubeClient = kubernetesfake.NewSimpleClientset()
|
|
kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
|
|
frozenNow = time.Now().UTC()
|
|
fakeClock = clock.NewFakeClock(frozenNow)
|
|
|
|
unrelatedSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "some other unrelated secret",
|
|
Namespace: installedInNamespace,
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(unrelatedSecret))
|
|
r.NoError(kubeClient.Tracker().Add(unrelatedSecret))
|
|
})
|
|
|
|
it.After(func() {
|
|
timeoutContextCancel()
|
|
})
|
|
|
|
when("there are secrets without the garbage-collect-after annotation", func() {
|
|
it("does not delete those secrets", func() {
|
|
startInformersAndController()
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
|
|
require.Empty(t, kubeClient.Actions())
|
|
list, err := kubeClient.CoreV1().Secrets(installedInNamespace).List(context.Background(), metav1.ListOptions{})
|
|
r.NoError(err)
|
|
r.Len(list.Items, 1)
|
|
r.Equal("some other unrelated secret", list.Items[0].Name)
|
|
})
|
|
})
|
|
|
|
when("there are secrets with the garbage-collect-after annotation", func() {
|
|
it.Before(func() {
|
|
firstExpiredSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "first expired secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(-time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(firstExpiredSecret))
|
|
r.NoError(kubeClient.Tracker().Add(firstExpiredSecret))
|
|
secondExpiredSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "second expired secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(-2 * time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(secondExpiredSecret))
|
|
r.NoError(kubeClient.Tracker().Add(secondExpiredSecret))
|
|
unexpiredSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "unexpired secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(unexpiredSecret))
|
|
r.NoError(kubeClient.Tracker().Add(unexpiredSecret))
|
|
})
|
|
|
|
it("should delete any that are past their expiration", func() {
|
|
startInformersAndController()
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
|
|
r.ElementsMatch(
|
|
[]kubetesting.Action{
|
|
kubetesting.NewDeleteAction(secretsGVR, installedInNamespace, "first expired secret"),
|
|
kubetesting.NewDeleteAction(secretsGVR, installedInNamespace, "second expired secret"),
|
|
},
|
|
kubeClient.Actions(),
|
|
)
|
|
list, err := kubeClient.CoreV1().Secrets(installedInNamespace).List(context.Background(), metav1.ListOptions{})
|
|
r.NoError(err)
|
|
r.Len(list.Items, 2)
|
|
r.ElementsMatch([]string{"unexpired secret", "some other unrelated secret"}, []string{list.Items[0].Name, list.Items[1].Name})
|
|
})
|
|
})
|
|
|
|
when("very little time has passed since the previous sync call", func() {
|
|
it.Before(func() {
|
|
// Add a secret that will expire in 20 seconds.
|
|
expiredSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "expired secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(20 * time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(expiredSecret))
|
|
r.NoError(kubeClient.Tracker().Add(expiredSecret))
|
|
})
|
|
|
|
it("should do nothing to avoid being super chatty since it is called for every change to any Secret, until more time has passed", func() {
|
|
startInformersAndController()
|
|
require.Empty(t, kubeClient.Actions())
|
|
|
|
// Run sync once with the current time set to frozenTime.
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
require.Empty(t, kubeClient.Actions())
|
|
|
|
// Run sync again when not enough time has passed since the most recent run, so no delete
|
|
// operations should happen even though there is a expired secret now.
|
|
fakeClock.Step(29 * time.Second)
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
require.Empty(t, kubeClient.Actions())
|
|
|
|
// Step to the exact threshold and run Sync again. Now we are past the rate limiting period.
|
|
fakeClock.Step(1*time.Second + 1*time.Millisecond)
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
|
|
// It should have deleted the expired secret.
|
|
r.ElementsMatch(
|
|
[]kubetesting.Action{
|
|
kubetesting.NewDeleteAction(secretsGVR, installedInNamespace, "expired secret"),
|
|
},
|
|
kubeClient.Actions(),
|
|
)
|
|
list, err := kubeClient.CoreV1().Secrets(installedInNamespace).List(context.Background(), metav1.ListOptions{})
|
|
r.NoError(err)
|
|
r.Len(list.Items, 1)
|
|
r.Equal("some other unrelated secret", list.Items[0].Name)
|
|
})
|
|
})
|
|
|
|
when("there is a secret with a malformed garbage-collect-after date", func() {
|
|
it.Before(func() {
|
|
malformedSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "malformed secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": "not-a-real-date-string",
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(malformedSecret))
|
|
r.NoError(kubeClient.Tracker().Add(malformedSecret))
|
|
expiredSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "expired secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(-time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(expiredSecret))
|
|
r.NoError(kubeClient.Tracker().Add(expiredSecret))
|
|
})
|
|
|
|
it("does not delete that secret", func() {
|
|
startInformersAndController()
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
|
|
r.ElementsMatch(
|
|
[]kubetesting.Action{
|
|
kubetesting.NewDeleteAction(secretsGVR, installedInNamespace, "expired secret"),
|
|
},
|
|
kubeClient.Actions(),
|
|
)
|
|
list, err := kubeClient.CoreV1().Secrets(installedInNamespace).List(context.Background(), metav1.ListOptions{})
|
|
r.NoError(err)
|
|
r.Len(list.Items, 2)
|
|
r.ElementsMatch([]string{"malformed secret", "some other unrelated secret"}, []string{list.Items[0].Name, list.Items[1].Name})
|
|
})
|
|
})
|
|
|
|
when("the kube API delete call fails", func() {
|
|
it.Before(func() {
|
|
erroringSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "erroring secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(-time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(erroringSecret))
|
|
r.NoError(kubeClient.Tracker().Add(erroringSecret))
|
|
kubeClient.PrependReactor("delete", "secrets", func(action kubetesting.Action) (bool, runtime.Object, error) {
|
|
if action.(kubetesting.DeleteActionImpl).Name == "erroring secret" {
|
|
return true, nil, errors.New("delete failed: some delete error")
|
|
}
|
|
return false, nil, nil
|
|
})
|
|
expiredSecret := &corev1.Secret{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "expired secret",
|
|
Namespace: installedInNamespace,
|
|
Annotations: map[string]string{
|
|
"storage.pinniped.dev/garbage-collect-after": frozenNow.Add(-time.Second).Format(time.RFC3339),
|
|
},
|
|
},
|
|
}
|
|
r.NoError(kubeInformerClient.Tracker().Add(expiredSecret))
|
|
r.NoError(kubeClient.Tracker().Add(expiredSecret))
|
|
})
|
|
|
|
it("ignores the error and continues on to delete the next expired Secret", func() {
|
|
startInformersAndController()
|
|
r.NoError(controllerlib.TestSync(t, subject, *syncContext))
|
|
|
|
r.ElementsMatch(
|
|
[]kubetesting.Action{
|
|
kubetesting.NewDeleteAction(secretsGVR, installedInNamespace, "erroring secret"),
|
|
kubetesting.NewDeleteAction(secretsGVR, installedInNamespace, "expired secret"),
|
|
},
|
|
kubeClient.Actions(),
|
|
)
|
|
list, err := kubeClient.CoreV1().Secrets(installedInNamespace).List(context.Background(), metav1.ListOptions{})
|
|
r.NoError(err)
|
|
r.Len(list.Items, 2)
|
|
r.ElementsMatch([]string{"erroring secret", "some other unrelated secret"}, []string{list.Items[0].Name, list.Items[1].Name})
|
|
})
|
|
})
|
|
}, spec.Parallel(), spec.Report(report.Terminal{}))
|
|
}
|