WIP: start to create kube-cert-agent pods in namespace

Signed-off-by: Andrew Keesler <akeesler@vmware.com>
This commit is contained in:
Andrew Keesler 2020-09-21 16:27:00 -04:00
parent e18b6fdddc
commit 50258fc569
No known key found for this signature in database
GPG Key ID: 27CE0444346F9413
5 changed files with 134 additions and 103 deletions

View File

@ -17,9 +17,10 @@ import (
) )
type createrController struct { type createrController struct {
agentInfo *Info agentInfo *Info
k8sClient kubernetes.Interface k8sClient kubernetes.Interface
podInformer corev1informers.PodInformer kubeSystemPodInformer corev1informers.PodInformer
agentPodInformer corev1informers.PodInformer
} }
// NewCreaterController returns a controller that creates new kube-cert-agent pods for every known // NewCreaterController returns a controller that creates new kube-cert-agent pods for every known
@ -29,7 +30,8 @@ type createrController struct {
func NewCreaterController( func NewCreaterController(
agentInfo *Info, agentInfo *Info,
k8sClient kubernetes.Interface, k8sClient kubernetes.Interface,
podInformer corev1informers.PodInformer, kubeSystemPodInformer corev1informers.PodInformer,
agentPodInformer corev1informers.PodInformer,
withInformer pinnipedcontroller.WithInformerOptionFunc, withInformer pinnipedcontroller.WithInformerOptionFunc,
) controllerlib.Controller { ) controllerlib.Controller {
return controllerlib.New( return controllerlib.New(
@ -37,15 +39,23 @@ func NewCreaterController(
//nolint: misspell //nolint: misspell
Name: "kube-cert-agent-creater-controller", Name: "kube-cert-agent-creater-controller",
Syncer: &createrController{ Syncer: &createrController{
agentInfo: agentInfo, agentInfo: agentInfo,
k8sClient: k8sClient, k8sClient: k8sClient,
podInformer: podInformer, kubeSystemPodInformer: kubeSystemPodInformer,
agentPodInformer: agentPodInformer,
}, },
}, },
withInformer( withInformer(
podInformer, kubeSystemPodInformer,
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels) return isControllerManagerPod(obj)
}),
controllerlib.InformerOption{},
),
withInformer(
agentPodInformer,
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
return isAgentPod(obj, agentInfo.Template.Labels)
}), }),
controllerlib.InformerOption{}, controllerlib.InformerOption{},
), ),
@ -59,13 +69,18 @@ func (c *createrController) Sync(ctx controllerlib.Context) error {
return fmt.Errorf("cannot create controller manager selector: %w", err) return fmt.Errorf("cannot create controller manager selector: %w", err)
} }
controllerManagerPods, err := c.podInformer.Lister().List(controllerManagerSelector) controllerManagerPods, err := c.kubeSystemPodInformer.Lister().List(controllerManagerSelector)
if err != nil { if err != nil {
return fmt.Errorf("informer cannot list controller manager pods: %w", err) return fmt.Errorf("informer cannot list controller manager pods: %w", err)
} }
for _, controllerManagerPod := range controllerManagerPods { for _, controllerManagerPod := range controllerManagerPods {
agentPod, err := findAgentPod(controllerManagerPod, c.podInformer, c.agentInfo.Template.Labels) agentPod, err := findAgentPod(
controllerManagerPod,
c.kubeSystemPodInformer,
c.agentPodInformer,
c.agentInfo.Template.Labels,
)
if err != nil { if err != nil {
return err return err
} }
@ -80,7 +95,7 @@ func (c *createrController) Sync(ctx controllerlib.Context) error {
klog.KObj(controllerManagerPod), klog.KObj(controllerManagerPod),
) )
_, err := c.k8sClient.CoreV1(). _, err := c.k8sClient.CoreV1().
Pods(ControllerManagerNamespace). Pods(c.agentInfo.Template.Namespace).
Create(ctx.Context, agentPod, metav1.CreateOptions{}) Create(ctx.Context, agentPod, metav1.CreateOptions{})
if err != nil { if err != nil {
return fmt.Errorf("cannot create agent pod: %w", err) return fmt.Errorf("cannot create agent pod: %w", err)

View File

@ -30,7 +30,8 @@ func TestCreaterControllerFilter(t *testing.T) {
"CreaterControllerFilter", "CreaterControllerFilter",
func( func(
agentPodTemplate *corev1.Pod, agentPodTemplate *corev1.Pod,
podsInformer corev1informers.PodInformer, kubeSystemPodInformer corev1informers.PodInformer,
//agentPodInformer corev1informers.PodInformer,
observableWithInformerOption *testutil.ObservableWithInformerOption, observableWithInformerOption *testutil.ObservableWithInformerOption,
) { ) {
_ = NewCreaterController( _ = NewCreaterController(
@ -38,7 +39,8 @@ func TestCreaterControllerFilter(t *testing.T) {
Template: agentPodTemplate, Template: agentPodTemplate,
}, },
nil, // k8sClient, shouldn't matter nil, // k8sClient, shouldn't matter
podsInformer, kubeSystemPodInformer,
nil, //agentPodInformer,
observableWithInformerOption.WithInformer, observableWithInformerOption.WithInformer,
) )
}, },
@ -53,15 +55,18 @@ func TestCreaterControllerSync(t *testing.T) {
var subject controllerlib.Controller var subject controllerlib.Controller
var kubeAPIClient *kubernetesfake.Clientset var kubeAPIClient *kubernetesfake.Clientset
var kubeInformerClient *kubernetesfake.Clientset var kubeSystemInformerClient *kubernetesfake.Clientset
var kubeInformers kubeinformers.SharedInformerFactory var kubeSystemInformers kubeinformers.SharedInformerFactory
var agentInformerClient *kubernetesfake.Clientset
var agentInformers kubeinformers.SharedInformerFactory
var timeoutContext context.Context var timeoutContext context.Context
var timeoutContextCancel context.CancelFunc var timeoutContextCancel context.CancelFunc
var syncContext *controllerlib.Context var syncContext *controllerlib.Context
agentPodTemplate := &corev1.Pod{ agentPodTemplate := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{ ObjectMeta: metav1.ObjectMeta{
Name: "some-agent-name-", Namespace: "some-agent-namespace",
Name: "some-agent-name-",
Labels: map[string]string{ Labels: map[string]string{
"some-label-key": "some-label-value", "some-label-key": "some-label-value",
}, },
@ -117,17 +122,10 @@ func TestCreaterControllerSync(t *testing.T) {
// fnv 32a hash of controller-manager uid // fnv 32a hash of controller-manager uid
controllerManagerPodHash := "fbb0addd" controllerManagerPodHash := "fbb0addd"
agentPod := agentPodTemplate.DeepCopy() agentPod := agentPodTemplate.DeepCopy()
agentPod.Namespace = kubeSystemNamespace
agentPod.Name += controllerManagerPodHash agentPod.Name += controllerManagerPodHash
agentPod.OwnerReferences = []metav1.OwnerReference{ agentPod.Annotations = map[string]string{
{ "kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name,
APIVersion: "v1", "kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID),
Kind: "Pod",
Name: controllerManagerPod.Name,
UID: controllerManagerPod.UID,
Controller: boolPtr(true),
BlockOwnerDeletion: boolPtr(true),
},
} }
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever
@ -151,7 +149,8 @@ func TestCreaterControllerSync(t *testing.T) {
Template: agentPodTemplate, Template: agentPodTemplate,
}, },
kubeAPIClient, kubeAPIClient,
kubeInformers.Core().V1().Pods(), kubeSystemInformers.Core().V1().Pods(),
agentInformers.Core().V1().Pods(),
controllerlib.WithInformer, controllerlib.WithInformer,
) )
@ -166,32 +165,39 @@ func TestCreaterControllerSync(t *testing.T) {
} }
// Must start informers before calling TestRunSynchronously() // Must start informers before calling TestRunSynchronously()
kubeInformers.Start(timeoutContext.Done()) kubeSystemInformers.Start(timeoutContext.Done())
agentInformers.Start(timeoutContext.Done())
controllerlib.TestRunSynchronously(t, subject) controllerlib.TestRunSynchronously(t, subject)
} }
it.Before(func() { it.Before(func() {
r = require.New(t) r = require.New(t)
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
kubeInformerClient = kubernetesfake.NewSimpleClientset()
kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
kubeAPIClient = kubernetesfake.NewSimpleClientset() kubeAPIClient = kubernetesfake.NewSimpleClientset()
kubeSystemInformerClient = kubernetesfake.NewSimpleClientset()
kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0)
agentInformerClient = kubernetesfake.NewSimpleClientset()
agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0)
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
// Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any // Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any
// logic on this thing. // logic on this thing.
ignorablePod := corev1.Pod{} ignorablePod := corev1.Pod{}
ignorablePod.Name = "some-ignorable-pod" ignorablePod.Name = "some-ignorable-pod"
r.NoError(kubeInformerClient.Tracker().Add(&ignorablePod)) r.NoError(kubeSystemInformerClient.Tracker().Add(&ignorablePod))
r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod))
// Add another valid agent pod to make sure our logic works for just the pod we care about. // Add another valid agent pod to make sure our logic works for just the pod we care about.
otherAgentPod := agentPod.DeepCopy() otherAgentPod := agentPod.DeepCopy()
otherAgentPod.Name = "some-other-agent" otherAgentPod.Name = "some-other-agent"
otherAgentPod.OwnerReferences[0].Name = "some-other-controller-manager-name" otherAgentPod.Annotations = map[string]string{
otherAgentPod.OwnerReferences[0].UID = "some-other-controller-manager-uid" "kube-cert-agent.pinniped.dev/controller-manager-name": "some-other-controller-manager-name",
r.NoError(kubeInformerClient.Tracker().Add(otherAgentPod)) "kube-cert-agent.pinniped.dev/controller-manager-uid": "some-other-controller-manager-uid",
}
r.NoError(agentInformerClient.Tracker().Add(otherAgentPod))
r.NoError(kubeAPIClient.Tracker().Add(otherAgentPod)) r.NoError(kubeAPIClient.Tracker().Add(otherAgentPod))
}) })
@ -201,13 +207,13 @@ func TestCreaterControllerSync(t *testing.T) {
when("there is a controller manager pod", func() { when("there is a controller manager pod", func() {
it.Before(func() { it.Before(func() {
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
when("there is a matching agent pod", func() { when("there is a matching agent pod", func() {
it.Before(func() { it.Before(func() {
r.NoError(kubeInformerClient.Tracker().Add(agentPod)) r.NoError(agentInformerClient.Tracker().Add(agentPod))
r.NoError(kubeAPIClient.Tracker().Add(agentPod)) r.NoError(kubeAPIClient.Tracker().Add(agentPod))
}) })
@ -227,8 +233,8 @@ func TestCreaterControllerSync(t *testing.T) {
it.Before(func() { it.Before(func() {
nonMatchingAgentPod := agentPod.DeepCopy() nonMatchingAgentPod := agentPod.DeepCopy()
nonMatchingAgentPod.Name = "some-agent-name-85da432e" nonMatchingAgentPod.Name = "some-agent-name-85da432e"
nonMatchingAgentPod.OwnerReferences[0].UID = "some-non-matching-uid" nonMatchingAgentPod.Annotations[controllerManagerUIDAnnotationKey] = "some-non-matching-uid"
r.NoError(kubeInformerClient.Tracker().Add(nonMatchingAgentPod)) r.NoError(agentInformerClient.Tracker().Add(nonMatchingAgentPod))
r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod)) r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod))
}) })
@ -241,7 +247,7 @@ func TestCreaterControllerSync(t *testing.T) {
[]coretesting.Action{ []coretesting.Action{
coretesting.NewCreateAction( coretesting.NewCreateAction(
podsGVR, podsGVR,
kubeSystemNamespace, agentPod.Namespace,
agentPod, agentPod,
), ),
}, },
@ -260,7 +266,7 @@ func TestCreaterControllerSync(t *testing.T) {
[]coretesting.Action{ []coretesting.Action{
coretesting.NewCreateAction( coretesting.NewCreateAction(
podsGVR, podsGVR,
kubeSystemNamespace, agentPod.Namespace,
agentPod, agentPod,
), ),
}, },

View File

@ -17,9 +17,10 @@ import (
) )
type deleterController struct { type deleterController struct {
agentInfo *Info agentInfo *Info
k8sClient kubernetes.Interface k8sClient kubernetes.Interface
podInformer corev1informers.PodInformer kubeSystemPodInformer corev1informers.PodInformer
agentPodInformer corev1informers.PodInformer
} }
// NewDeleterController returns a controller that deletes any kube-cert-agent pods that are out of // NewDeleterController returns a controller that deletes any kube-cert-agent pods that are out of
@ -29,20 +30,22 @@ type deleterController struct {
func NewDeleterController( func NewDeleterController(
agentInfo *Info, agentInfo *Info,
k8sClient kubernetes.Interface, k8sClient kubernetes.Interface,
podInformer corev1informers.PodInformer, kubeSystemPodInformer corev1informers.PodInformer,
agentPodInformer corev1informers.PodInformer,
withInformer pinnipedcontroller.WithInformerOptionFunc, withInformer pinnipedcontroller.WithInformerOptionFunc,
) controllerlib.Controller { ) controllerlib.Controller {
return controllerlib.New( return controllerlib.New(
controllerlib.Config{ controllerlib.Config{
Name: "kube-cert-agent-deleter-controller", Name: "kube-cert-agent-deleter-controller",
Syncer: &deleterController{ Syncer: &deleterController{
agentInfo: agentInfo, agentInfo: agentInfo,
k8sClient: k8sClient, k8sClient: k8sClient,
podInformer: podInformer, kubeSystemPodInformer: kubeSystemPodInformer,
agentPodInformer: agentPodInformer,
}, },
}, },
withInformer( withInformer(
podInformer, agentPodInformer,
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels) return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels)
}), }),
@ -54,7 +57,7 @@ func NewDeleterController(
// Sync implements controllerlib.Syncer. // Sync implements controllerlib.Syncer.
func (c *deleterController) Sync(ctx controllerlib.Context) error { func (c *deleterController) Sync(ctx controllerlib.Context) error {
agentSelector := labels.SelectorFromSet(c.agentInfo.Template.Labels) agentSelector := labels.SelectorFromSet(c.agentInfo.Template.Labels)
agentPods, err := c.podInformer. agentPods, err := c.agentPodInformer.
Lister(). Lister().
Pods(ControllerManagerNamespace). Pods(ControllerManagerNamespace).
List(agentSelector) List(agentSelector)
@ -63,7 +66,7 @@ func (c *deleterController) Sync(ctx controllerlib.Context) error {
} }
for _, agentPod := range agentPods { for _, agentPod := range agentPods {
controllerManagerPod, err := findControllerManagerPod(agentPod, c.podInformer) controllerManagerPod, err := findControllerManagerPod(agentPod, c.kubeSystemPodInformer)
if err != nil { if err != nil {
return err return err
} }

View File

@ -53,8 +53,10 @@ func TestDeleterControllerSync(t *testing.T) {
var subject controllerlib.Controller var subject controllerlib.Controller
var kubeAPIClient *kubernetesfake.Clientset var kubeAPIClient *kubernetesfake.Clientset
var kubeInformerClient *kubernetesfake.Clientset var kubeSystemInformerClient *kubernetesfake.Clientset
var kubeInformers kubeinformers.SharedInformerFactory var kubeSystemInformers kubeinformers.SharedInformerFactory
var agentInformerClient *kubernetesfake.Clientset
var agentInformers kubeinformers.SharedInformerFactory
var timeoutContext context.Context var timeoutContext context.Context
var timeoutContextCancel context.CancelFunc var timeoutContextCancel context.CancelFunc
var syncContext *controllerlib.Context var syncContext *controllerlib.Context
@ -125,15 +127,9 @@ func TestDeleterControllerSync(t *testing.T) {
agentPod := agentPodTemplate.DeepCopy() agentPod := agentPodTemplate.DeepCopy()
agentPod.Namespace = kubeSystemNamespace agentPod.Namespace = kubeSystemNamespace
agentPod.Name += controllerManagerPodHash agentPod.Name += controllerManagerPodHash
agentPod.OwnerReferences = []metav1.OwnerReference{ agentPod.Annotations = map[string]string{
{ "kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name,
APIVersion: "v1", "kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID),
Kind: "Pod",
Name: controllerManagerPod.Name,
UID: controllerManagerPod.UID,
Controller: boolPtr(true),
BlockOwnerDeletion: boolPtr(true),
},
} }
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever
@ -151,7 +147,8 @@ func TestDeleterControllerSync(t *testing.T) {
Template: agentPodTemplate, Template: agentPodTemplate,
}, },
kubeAPIClient, kubeAPIClient,
kubeInformers.Core().V1().Pods(), kubeSystemInformers.Core().V1().Pods(),
agentInformers.Core().V1().Pods(),
controllerlib.WithInformer, controllerlib.WithInformer,
) )
@ -166,7 +163,8 @@ func TestDeleterControllerSync(t *testing.T) {
} }
// Must start informers before calling TestRunSynchronously() // Must start informers before calling TestRunSynchronously()
kubeInformers.Start(timeoutContext.Done()) kubeSystemInformers.Start(timeoutContext.Done())
agentInformers.Start(timeoutContext.Done())
controllerlib.TestRunSynchronously(t, subject) controllerlib.TestRunSynchronously(t, subject)
} }
@ -175,15 +173,19 @@ func TestDeleterControllerSync(t *testing.T) {
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
kubeInformerClient = kubernetesfake.NewSimpleClientset()
kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
kubeAPIClient = kubernetesfake.NewSimpleClientset() kubeAPIClient = kubernetesfake.NewSimpleClientset()
kubeSystemInformerClient = kubernetesfake.NewSimpleClientset()
kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0)
agentInformerClient = kubernetesfake.NewSimpleClientset()
agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0)
// Add an pod into the test that doesn't matter to make sure we don't accidentally // Add an pod into the test that doesn't matter to make sure we don't accidentally
// trigger any logic on this thing. // trigger any logic on this thing.
ignorablePod := corev1.Pod{} ignorablePod := corev1.Pod{}
ignorablePod.Name = "some-ignorable-pod" ignorablePod.Name = "some-ignorable-pod"
r.NoError(kubeInformerClient.Tracker().Add(&ignorablePod)) r.NoError(agentInformerClient.Tracker().Add(&ignorablePod))
r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod))
}) })
@ -193,13 +195,13 @@ func TestDeleterControllerSync(t *testing.T) {
when("there is an agent pod", func() { when("there is an agent pod", func() {
it.Before(func() { it.Before(func() {
r.NoError(kubeInformerClient.Tracker().Add(agentPod)) r.NoError(agentInformerClient.Tracker().Add(agentPod))
r.NoError(kubeAPIClient.Tracker().Add(agentPod)) r.NoError(kubeAPIClient.Tracker().Add(agentPod))
}) })
when("there is a matching controller manager pod", func() { when("there is a matching controller manager pod", func() {
it.Before(func() { it.Before(func() {
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -218,7 +220,7 @@ func TestDeleterControllerSync(t *testing.T) {
when("there is a non-matching controller manager pod via uid", func() { when("there is a non-matching controller manager pod via uid", func() {
it.Before(func() { it.Before(func() {
controllerManagerPod.UID = "some-other-controller-manager-uid" controllerManagerPod.UID = "some-other-controller-manager-uid"
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -243,7 +245,7 @@ func TestDeleterControllerSync(t *testing.T) {
when("there is a non-matching controller manager pod via name", func() { when("there is a non-matching controller manager pod via name", func() {
it.Before(func() { it.Before(func() {
controllerManagerPod.Name = "some-other-controller-manager-name" controllerManagerPod.Name = "some-other-controller-manager-name"
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -272,7 +274,7 @@ func TestDeleterControllerSync(t *testing.T) {
Name: "some-other-volume-mount", Name: "some-other-volume-mount",
}, },
} }
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -301,7 +303,7 @@ func TestDeleterControllerSync(t *testing.T) {
Name: "some-other-volume", Name: "some-other-volume",
}, },
} }
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -328,7 +330,7 @@ func TestDeleterControllerSync(t *testing.T) {
controllerManagerPod.Spec.NodeSelector = map[string]string{ controllerManagerPod.Spec.NodeSelector = map[string]string{
"some-other-node-selector-key": "some-other-node-selector-value", "some-other-node-selector-key": "some-other-node-selector-value",
} }
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -353,7 +355,7 @@ func TestDeleterControllerSync(t *testing.T) {
when("the agent pod is out of sync with the controller manager via node name", func() { when("the agent pod is out of sync with the controller manager via node name", func() {
it.Before(func() { it.Before(func() {
controllerManagerPod.Spec.NodeName = "some-other-node-name" controllerManagerPod.Spec.NodeName = "some-other-node-name"
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -382,7 +384,7 @@ func TestDeleterControllerSync(t *testing.T) {
Key: "some-other-toleration-key", Key: "some-other-toleration-key",
}, },
} }
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
}) })
@ -408,7 +410,7 @@ func TestDeleterControllerSync(t *testing.T) {
it.Before(func() { it.Before(func() {
updatedAgentPod := agentPod.DeepCopy() updatedAgentPod := agentPod.DeepCopy()
updatedAgentPod.Spec.RestartPolicy = corev1.RestartPolicyAlways updatedAgentPod.Spec.RestartPolicy = corev1.RestartPolicyAlways
r.NoError(kubeInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace))
r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace))
}) })
@ -433,7 +435,7 @@ func TestDeleterControllerSync(t *testing.T) {
when("the agent pod is out of sync via automount service account tokem", func() { when("the agent pod is out of sync via automount service account tokem", func() {
it.Before(func() { it.Before(func() {
agentPod.Spec.AutomountServiceAccountToken = boolPtr(true) agentPod.Spec.AutomountServiceAccountToken = boolPtr(true)
r.NoError(kubeInformerClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace)) r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace))
r.NoError(kubeAPIClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace)) r.NoError(kubeAPIClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace))
}) })

View File

@ -19,7 +19,6 @@ import (
k8serrors "k8s.io/apimachinery/pkg/api/errors" k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
corev1informers "k8s.io/client-go/informers/core/v1" corev1informers "k8s.io/client-go/informers/core/v1"
"k8s.io/klog/v2" "k8s.io/klog/v2"
) )
@ -27,12 +26,16 @@ import (
const ( const (
// ControllerManagerNamespace is the assumed namespace of the kube-controller-manager pod(s). // ControllerManagerNamespace is the assumed namespace of the kube-controller-manager pod(s).
ControllerManagerNamespace = "kube-system" ControllerManagerNamespace = "kube-system"
controllerManagerNameAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-name"
controllerManagerUIDAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-uid"
) )
// Info holds necessary information about the agent pod. It was pulled out into a struct to have a // Info holds necessary information about the agent pod. It was pulled out into a struct to have a
// common parameter for each controller. // common parameter for each controller.
type Info struct { type Info struct {
// Template is an injection point for pod fields. The required pod fields are as follows. // Template is an injection point for pod fields. The required pod fields are as follows.
// .Namespace: serves as the namespace of the agent pods
// .Name: serves as the name prefix for each of the agent pods // .Name: serves as the name prefix for each of the agent pods
// .Labels: serves as a way to filter for agent pods // .Labels: serves as a way to filter for agent pods
// .Spec.Containers[0].Image: serves as the container image for the agent pods // .Spec.Containers[0].Image: serves as the container image for the agent pods
@ -40,11 +43,11 @@ type Info struct {
Template *corev1.Pod Template *corev1.Pod
// CertPathAnnotation is the name of the annotation key that will be used when setting the // CertPathAnnotation is the name of the annotation key that will be used when setting the
// best-guess path to the kube API's certificate. // best-guess path to the kube API's certificate in the agent pod.
CertPathAnnotation string CertPathAnnotation string
// KeyPathAnnotation is the name of the annotation key that will be used when setting the // KeyPathAnnotation is the name of the annotation key that will be used when setting the
// best-guess path to the kube API's private key. // best-guess path to the kube API's private key in the agent pod.
KeyPathAnnotation string KeyPathAnnotation string
} }
@ -91,18 +94,14 @@ func newAgentPod(
agentPod.Name = fmt.Sprintf("%s%s", agentPod.Name, hash(controllerManagerPod)) agentPod.Name = fmt.Sprintf("%s%s", agentPod.Name, hash(controllerManagerPod))
// controller manager namespace because it is our owner // It would be nice to use the OwnerReferences field here, but the agent pod is most likely in a
agentPod.Namespace = ControllerManagerNamespace // different namespace than the kube-controller-manager pod, and therefore that breaks the
agentPod.OwnerReferences = []metav1.OwnerReference{ // OwnerReferences contract (see metav1.OwnerReference doc).
*metav1.NewControllerRef( if agentPod.Annotations == nil {
controllerManagerPod, agentPod.Annotations = make(map[string]string)
schema.GroupVersionKind{
Group: corev1.SchemeGroupVersion.Group,
Version: corev1.SchemeGroupVersion.Version,
Kind: "Pod",
},
),
} }
agentPod.Annotations[controllerManagerNameAnnotationKey] = controllerManagerPod.Name
agentPod.Annotations[controllerManagerUIDAnnotationKey] = string(controllerManagerPod.UID)
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
agentPod.Spec.Volumes = controllerManagerPod.Spec.Volumes agentPod.Spec.Volumes = controllerManagerPod.Spec.Volumes
@ -148,13 +147,13 @@ func isAgentPodUpToDate(actualAgentPod, expectedAgentPod *corev1.Pod) bool {
func findAgentPod( func findAgentPod(
controllerManagerPod *corev1.Pod, controllerManagerPod *corev1.Pod,
informer corev1informers.PodInformer, kubeSystemPodInformer corev1informers.PodInformer,
agentPodInformer corev1informers.PodInformer,
agentLabels map[string]string, agentLabels map[string]string,
) (*corev1.Pod, error) { ) (*corev1.Pod, error) {
agentSelector := labels.SelectorFromSet(agentLabels) agentSelector := labels.SelectorFromSet(agentLabels)
agentPods, err := informer. agentPods, err := agentPodInformer.
Lister(). Lister().
Pods(ControllerManagerNamespace).
List(agentSelector) List(agentSelector)
if err != nil { if err != nil {
return nil, fmt.Errorf("informer cannot list agent pods: %w", err) return nil, fmt.Errorf("informer cannot list agent pods: %w", err)
@ -163,7 +162,7 @@ func findAgentPod(
for _, maybeAgentPod := range agentPods { for _, maybeAgentPod := range agentPods {
maybeControllerManagerPod, err := findControllerManagerPod( maybeControllerManagerPod, err := findControllerManagerPod(
maybeAgentPod, maybeAgentPod,
informer, kubeSystemPodInformer,
) )
if err != nil { if err != nil {
return nil, err return nil, err
@ -181,22 +180,28 @@ func findControllerManagerPod(
agentPod *corev1.Pod, agentPod *corev1.Pod,
informer corev1informers.PodInformer, informer corev1informers.PodInformer,
) (*corev1.Pod, error) { ) (*corev1.Pod, error) {
controller := metav1.GetControllerOf(agentPod) name, ok := agentPod.Annotations[controllerManagerNameAnnotationKey]
if controller == nil { if !ok {
klog.InfoS("found orphan agent pod", "pod", klog.KObj(agentPod)) klog.InfoS("agent pod missing parent name annotation", "pod", agentPod)
return nil, nil
}
uid, ok := agentPod.Annotations[controllerManagerUIDAnnotationKey]
if !ok {
klog.InfoS("agent pod missing parent uid annotation", "pod", agentPod)
return nil, nil return nil, nil
} }
maybeControllerManagerPod, err := informer. maybeControllerManagerPod, err := informer.
Lister(). Lister().
Pods(ControllerManagerNamespace). Pods(ControllerManagerNamespace).
Get(controller.Name) Get(name)
notFound := k8serrors.IsNotFound(err) notFound := k8serrors.IsNotFound(err)
if err != nil && !notFound { if err != nil && !notFound {
return nil, fmt.Errorf("cannot get controller pod: %w", err) return nil, fmt.Errorf("cannot get controller pod: %w", err)
} else if notFound || } else if notFound ||
maybeControllerManagerPod == nil || maybeControllerManagerPod == nil ||
maybeControllerManagerPod.UID != controller.UID { string(maybeControllerManagerPod.UID) != uid {
return nil, nil return nil, nil
} }