From 50258fc56958e40b9d8d2698272bd4a1334aa6f0 Mon Sep 17 00:00:00 2001 From: Andrew Keesler Date: Mon, 21 Sep 2020 16:27:00 -0400 Subject: [PATCH] WIP: start to create kube-cert-agent pods in namespace Signed-off-by: Andrew Keesler --- internal/controller/kubecertagent/creater.go | 39 +++++++---- .../controller/kubecertagent/creater_test.go | 68 ++++++++++--------- internal/controller/kubecertagent/deleter.go | 23 ++++--- .../controller/kubecertagent/deleter_test.go | 56 +++++++-------- .../controller/kubecertagent/kubecertagent.go | 51 +++++++------- 5 files changed, 134 insertions(+), 103 deletions(-) diff --git a/internal/controller/kubecertagent/creater.go b/internal/controller/kubecertagent/creater.go index 52d21e3c..d61d0104 100644 --- a/internal/controller/kubecertagent/creater.go +++ b/internal/controller/kubecertagent/creater.go @@ -17,9 +17,10 @@ import ( ) type createrController struct { - agentInfo *Info - k8sClient kubernetes.Interface - podInformer corev1informers.PodInformer + agentInfo *Info + k8sClient kubernetes.Interface + kubeSystemPodInformer corev1informers.PodInformer + agentPodInformer corev1informers.PodInformer } // NewCreaterController returns a controller that creates new kube-cert-agent pods for every known @@ -29,7 +30,8 @@ type createrController struct { func NewCreaterController( agentInfo *Info, k8sClient kubernetes.Interface, - podInformer corev1informers.PodInformer, + kubeSystemPodInformer corev1informers.PodInformer, + agentPodInformer corev1informers.PodInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, ) controllerlib.Controller { return controllerlib.New( @@ -37,15 +39,23 @@ func NewCreaterController( //nolint: misspell Name: "kube-cert-agent-creater-controller", Syncer: &createrController{ - agentInfo: agentInfo, - k8sClient: k8sClient, - podInformer: podInformer, + agentInfo: agentInfo, + k8sClient: k8sClient, + kubeSystemPodInformer: kubeSystemPodInformer, + agentPodInformer: agentPodInformer, }, }, withInformer( - podInformer, + kubeSystemPodInformer, pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { - return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels) + return isControllerManagerPod(obj) + }), + controllerlib.InformerOption{}, + ), + withInformer( + agentPodInformer, + pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { + return isAgentPod(obj, agentInfo.Template.Labels) }), controllerlib.InformerOption{}, ), @@ -59,13 +69,18 @@ func (c *createrController) Sync(ctx controllerlib.Context) error { return fmt.Errorf("cannot create controller manager selector: %w", err) } - controllerManagerPods, err := c.podInformer.Lister().List(controllerManagerSelector) + controllerManagerPods, err := c.kubeSystemPodInformer.Lister().List(controllerManagerSelector) if err != nil { return fmt.Errorf("informer cannot list controller manager pods: %w", err) } for _, controllerManagerPod := range controllerManagerPods { - agentPod, err := findAgentPod(controllerManagerPod, c.podInformer, c.agentInfo.Template.Labels) + agentPod, err := findAgentPod( + controllerManagerPod, + c.kubeSystemPodInformer, + c.agentPodInformer, + c.agentInfo.Template.Labels, + ) if err != nil { return err } @@ -80,7 +95,7 @@ func (c *createrController) Sync(ctx controllerlib.Context) error { klog.KObj(controllerManagerPod), ) _, err := c.k8sClient.CoreV1(). - Pods(ControllerManagerNamespace). + Pods(c.agentInfo.Template.Namespace). Create(ctx.Context, agentPod, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("cannot create agent pod: %w", err) diff --git a/internal/controller/kubecertagent/creater_test.go b/internal/controller/kubecertagent/creater_test.go index 2eec27bf..216c8ea2 100644 --- a/internal/controller/kubecertagent/creater_test.go +++ b/internal/controller/kubecertagent/creater_test.go @@ -30,7 +30,8 @@ func TestCreaterControllerFilter(t *testing.T) { "CreaterControllerFilter", func( agentPodTemplate *corev1.Pod, - podsInformer corev1informers.PodInformer, + kubeSystemPodInformer corev1informers.PodInformer, + //agentPodInformer corev1informers.PodInformer, observableWithInformerOption *testutil.ObservableWithInformerOption, ) { _ = NewCreaterController( @@ -38,7 +39,8 @@ func TestCreaterControllerFilter(t *testing.T) { Template: agentPodTemplate, }, nil, // k8sClient, shouldn't matter - podsInformer, + kubeSystemPodInformer, + nil, //agentPodInformer, observableWithInformerOption.WithInformer, ) }, @@ -53,15 +55,18 @@ func TestCreaterControllerSync(t *testing.T) { var subject controllerlib.Controller var kubeAPIClient *kubernetesfake.Clientset - var kubeInformerClient *kubernetesfake.Clientset - var kubeInformers kubeinformers.SharedInformerFactory + var kubeSystemInformerClient *kubernetesfake.Clientset + var kubeSystemInformers kubeinformers.SharedInformerFactory + var agentInformerClient *kubernetesfake.Clientset + var agentInformers kubeinformers.SharedInformerFactory var timeoutContext context.Context var timeoutContextCancel context.CancelFunc var syncContext *controllerlib.Context agentPodTemplate := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: "some-agent-name-", + Namespace: "some-agent-namespace", + Name: "some-agent-name-", Labels: map[string]string{ "some-label-key": "some-label-value", }, @@ -117,17 +122,10 @@ func TestCreaterControllerSync(t *testing.T) { // fnv 32a hash of controller-manager uid controllerManagerPodHash := "fbb0addd" agentPod := agentPodTemplate.DeepCopy() - agentPod.Namespace = kubeSystemNamespace agentPod.Name += controllerManagerPodHash - agentPod.OwnerReferences = []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "Pod", - Name: controllerManagerPod.Name, - UID: controllerManagerPod.UID, - Controller: boolPtr(true), - BlockOwnerDeletion: boolPtr(true), - }, + agentPod.Annotations = map[string]string{ + "kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name, + "kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID), } agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever @@ -151,7 +149,8 @@ func TestCreaterControllerSync(t *testing.T) { Template: agentPodTemplate, }, kubeAPIClient, - kubeInformers.Core().V1().Pods(), + kubeSystemInformers.Core().V1().Pods(), + agentInformers.Core().V1().Pods(), controllerlib.WithInformer, ) @@ -166,32 +165,39 @@ func TestCreaterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(timeoutContext.Done()) + agentInformers.Start(timeoutContext.Done()) controllerlib.TestRunSynchronously(t, subject) } it.Before(func() { r = require.New(t) - timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) - - kubeInformerClient = kubernetesfake.NewSimpleClientset() - kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) kubeAPIClient = kubernetesfake.NewSimpleClientset() + kubeSystemInformerClient = kubernetesfake.NewSimpleClientset() + kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0) + + agentInformerClient = kubernetesfake.NewSimpleClientset() + agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0) + + timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) + // Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any // logic on this thing. ignorablePod := corev1.Pod{} ignorablePod.Name = "some-ignorable-pod" - r.NoError(kubeInformerClient.Tracker().Add(&ignorablePod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(&ignorablePod)) r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) // Add another valid agent pod to make sure our logic works for just the pod we care about. otherAgentPod := agentPod.DeepCopy() otherAgentPod.Name = "some-other-agent" - otherAgentPod.OwnerReferences[0].Name = "some-other-controller-manager-name" - otherAgentPod.OwnerReferences[0].UID = "some-other-controller-manager-uid" - r.NoError(kubeInformerClient.Tracker().Add(otherAgentPod)) + otherAgentPod.Annotations = map[string]string{ + "kube-cert-agent.pinniped.dev/controller-manager-name": "some-other-controller-manager-name", + "kube-cert-agent.pinniped.dev/controller-manager-uid": "some-other-controller-manager-uid", + } + r.NoError(agentInformerClient.Tracker().Add(otherAgentPod)) r.NoError(kubeAPIClient.Tracker().Add(otherAgentPod)) }) @@ -201,13 +207,13 @@ func TestCreaterControllerSync(t *testing.T) { when("there is a controller manager pod", func() { it.Before(func() { - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) when("there is a matching agent pod", func() { it.Before(func() { - r.NoError(kubeInformerClient.Tracker().Add(agentPod)) + r.NoError(agentInformerClient.Tracker().Add(agentPod)) r.NoError(kubeAPIClient.Tracker().Add(agentPod)) }) @@ -227,8 +233,8 @@ func TestCreaterControllerSync(t *testing.T) { it.Before(func() { nonMatchingAgentPod := agentPod.DeepCopy() nonMatchingAgentPod.Name = "some-agent-name-85da432e" - nonMatchingAgentPod.OwnerReferences[0].UID = "some-non-matching-uid" - r.NoError(kubeInformerClient.Tracker().Add(nonMatchingAgentPod)) + nonMatchingAgentPod.Annotations[controllerManagerUIDAnnotationKey] = "some-non-matching-uid" + r.NoError(agentInformerClient.Tracker().Add(nonMatchingAgentPod)) r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod)) }) @@ -241,7 +247,7 @@ func TestCreaterControllerSync(t *testing.T) { []coretesting.Action{ coretesting.NewCreateAction( podsGVR, - kubeSystemNamespace, + agentPod.Namespace, agentPod, ), }, @@ -260,7 +266,7 @@ func TestCreaterControllerSync(t *testing.T) { []coretesting.Action{ coretesting.NewCreateAction( podsGVR, - kubeSystemNamespace, + agentPod.Namespace, agentPod, ), }, diff --git a/internal/controller/kubecertagent/deleter.go b/internal/controller/kubecertagent/deleter.go index 240cb534..d3fb6a43 100644 --- a/internal/controller/kubecertagent/deleter.go +++ b/internal/controller/kubecertagent/deleter.go @@ -17,9 +17,10 @@ import ( ) type deleterController struct { - agentInfo *Info - k8sClient kubernetes.Interface - podInformer corev1informers.PodInformer + agentInfo *Info + k8sClient kubernetes.Interface + kubeSystemPodInformer corev1informers.PodInformer + agentPodInformer corev1informers.PodInformer } // NewDeleterController returns a controller that deletes any kube-cert-agent pods that are out of @@ -29,20 +30,22 @@ type deleterController struct { func NewDeleterController( agentInfo *Info, k8sClient kubernetes.Interface, - podInformer corev1informers.PodInformer, + kubeSystemPodInformer corev1informers.PodInformer, + agentPodInformer corev1informers.PodInformer, withInformer pinnipedcontroller.WithInformerOptionFunc, ) controllerlib.Controller { return controllerlib.New( controllerlib.Config{ Name: "kube-cert-agent-deleter-controller", Syncer: &deleterController{ - agentInfo: agentInfo, - k8sClient: k8sClient, - podInformer: podInformer, + agentInfo: agentInfo, + k8sClient: k8sClient, + kubeSystemPodInformer: kubeSystemPodInformer, + agentPodInformer: agentPodInformer, }, }, withInformer( - podInformer, + agentPodInformer, pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool { return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels) }), @@ -54,7 +57,7 @@ func NewDeleterController( // Sync implements controllerlib.Syncer. func (c *deleterController) Sync(ctx controllerlib.Context) error { agentSelector := labels.SelectorFromSet(c.agentInfo.Template.Labels) - agentPods, err := c.podInformer. + agentPods, err := c.agentPodInformer. Lister(). Pods(ControllerManagerNamespace). List(agentSelector) @@ -63,7 +66,7 @@ func (c *deleterController) Sync(ctx controllerlib.Context) error { } for _, agentPod := range agentPods { - controllerManagerPod, err := findControllerManagerPod(agentPod, c.podInformer) + controllerManagerPod, err := findControllerManagerPod(agentPod, c.kubeSystemPodInformer) if err != nil { return err } diff --git a/internal/controller/kubecertagent/deleter_test.go b/internal/controller/kubecertagent/deleter_test.go index 3e1d2c89..00a86f5c 100644 --- a/internal/controller/kubecertagent/deleter_test.go +++ b/internal/controller/kubecertagent/deleter_test.go @@ -53,8 +53,10 @@ func TestDeleterControllerSync(t *testing.T) { var subject controllerlib.Controller var kubeAPIClient *kubernetesfake.Clientset - var kubeInformerClient *kubernetesfake.Clientset - var kubeInformers kubeinformers.SharedInformerFactory + var kubeSystemInformerClient *kubernetesfake.Clientset + var kubeSystemInformers kubeinformers.SharedInformerFactory + var agentInformerClient *kubernetesfake.Clientset + var agentInformers kubeinformers.SharedInformerFactory var timeoutContext context.Context var timeoutContextCancel context.CancelFunc var syncContext *controllerlib.Context @@ -125,15 +127,9 @@ func TestDeleterControllerSync(t *testing.T) { agentPod := agentPodTemplate.DeepCopy() agentPod.Namespace = kubeSystemNamespace agentPod.Name += controllerManagerPodHash - agentPod.OwnerReferences = []metav1.OwnerReference{ - { - APIVersion: "v1", - Kind: "Pod", - Name: controllerManagerPod.Name, - UID: controllerManagerPod.UID, - Controller: boolPtr(true), - BlockOwnerDeletion: boolPtr(true), - }, + agentPod.Annotations = map[string]string{ + "kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name, + "kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID), } agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever @@ -151,7 +147,8 @@ func TestDeleterControllerSync(t *testing.T) { Template: agentPodTemplate, }, kubeAPIClient, - kubeInformers.Core().V1().Pods(), + kubeSystemInformers.Core().V1().Pods(), + agentInformers.Core().V1().Pods(), controllerlib.WithInformer, ) @@ -166,7 +163,8 @@ func TestDeleterControllerSync(t *testing.T) { } // Must start informers before calling TestRunSynchronously() - kubeInformers.Start(timeoutContext.Done()) + kubeSystemInformers.Start(timeoutContext.Done()) + agentInformers.Start(timeoutContext.Done()) controllerlib.TestRunSynchronously(t, subject) } @@ -175,15 +173,19 @@ func TestDeleterControllerSync(t *testing.T) { timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3) - kubeInformerClient = kubernetesfake.NewSimpleClientset() - kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0) kubeAPIClient = kubernetesfake.NewSimpleClientset() + kubeSystemInformerClient = kubernetesfake.NewSimpleClientset() + kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0) + + agentInformerClient = kubernetesfake.NewSimpleClientset() + agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0) + // Add an pod into the test that doesn't matter to make sure we don't accidentally // trigger any logic on this thing. ignorablePod := corev1.Pod{} ignorablePod.Name = "some-ignorable-pod" - r.NoError(kubeInformerClient.Tracker().Add(&ignorablePod)) + r.NoError(agentInformerClient.Tracker().Add(&ignorablePod)) r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod)) }) @@ -193,13 +195,13 @@ func TestDeleterControllerSync(t *testing.T) { when("there is an agent pod", func() { it.Before(func() { - r.NoError(kubeInformerClient.Tracker().Add(agentPod)) + r.NoError(agentInformerClient.Tracker().Add(agentPod)) r.NoError(kubeAPIClient.Tracker().Add(agentPod)) }) when("there is a matching controller manager pod", func() { it.Before(func() { - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -218,7 +220,7 @@ func TestDeleterControllerSync(t *testing.T) { when("there is a non-matching controller manager pod via uid", func() { it.Before(func() { controllerManagerPod.UID = "some-other-controller-manager-uid" - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -243,7 +245,7 @@ func TestDeleterControllerSync(t *testing.T) { when("there is a non-matching controller manager pod via name", func() { it.Before(func() { controllerManagerPod.Name = "some-other-controller-manager-name" - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -272,7 +274,7 @@ func TestDeleterControllerSync(t *testing.T) { Name: "some-other-volume-mount", }, } - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -301,7 +303,7 @@ func TestDeleterControllerSync(t *testing.T) { Name: "some-other-volume", }, } - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -328,7 +330,7 @@ func TestDeleterControllerSync(t *testing.T) { controllerManagerPod.Spec.NodeSelector = map[string]string{ "some-other-node-selector-key": "some-other-node-selector-value", } - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -353,7 +355,7 @@ func TestDeleterControllerSync(t *testing.T) { when("the agent pod is out of sync with the controller manager via node name", func() { it.Before(func() { controllerManagerPod.Spec.NodeName = "some-other-node-name" - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -382,7 +384,7 @@ func TestDeleterControllerSync(t *testing.T) { Key: "some-other-toleration-key", }, } - r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod)) + r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod)) r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod)) }) @@ -408,7 +410,7 @@ func TestDeleterControllerSync(t *testing.T) { it.Before(func() { updatedAgentPod := agentPod.DeepCopy() updatedAgentPod.Spec.RestartPolicy = corev1.RestartPolicyAlways - r.NoError(kubeInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) + r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace)) }) @@ -433,7 +435,7 @@ func TestDeleterControllerSync(t *testing.T) { when("the agent pod is out of sync via automount service account tokem", func() { it.Before(func() { agentPod.Spec.AutomountServiceAccountToken = boolPtr(true) - r.NoError(kubeInformerClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace)) + r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace)) r.NoError(kubeAPIClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace)) }) diff --git a/internal/controller/kubecertagent/kubecertagent.go b/internal/controller/kubecertagent/kubecertagent.go index 82eb3e8f..38cbe485 100644 --- a/internal/controller/kubecertagent/kubecertagent.go +++ b/internal/controller/kubecertagent/kubecertagent.go @@ -19,7 +19,6 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" corev1informers "k8s.io/client-go/informers/core/v1" "k8s.io/klog/v2" ) @@ -27,12 +26,16 @@ import ( const ( // ControllerManagerNamespace is the assumed namespace of the kube-controller-manager pod(s). ControllerManagerNamespace = "kube-system" + + controllerManagerNameAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-name" + controllerManagerUIDAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-uid" ) // Info holds necessary information about the agent pod. It was pulled out into a struct to have a // common parameter for each controller. type Info struct { // Template is an injection point for pod fields. The required pod fields are as follows. + // .Namespace: serves as the namespace of the agent pods // .Name: serves as the name prefix for each of the agent pods // .Labels: serves as a way to filter for agent pods // .Spec.Containers[0].Image: serves as the container image for the agent pods @@ -40,11 +43,11 @@ type Info struct { Template *corev1.Pod // CertPathAnnotation is the name of the annotation key that will be used when setting the - // best-guess path to the kube API's certificate. + // best-guess path to the kube API's certificate in the agent pod. CertPathAnnotation string // KeyPathAnnotation is the name of the annotation key that will be used when setting the - // best-guess path to the kube API's private key. + // best-guess path to the kube API's private key in the agent pod. KeyPathAnnotation string } @@ -91,18 +94,14 @@ func newAgentPod( agentPod.Name = fmt.Sprintf("%s%s", agentPod.Name, hash(controllerManagerPod)) - // controller manager namespace because it is our owner - agentPod.Namespace = ControllerManagerNamespace - agentPod.OwnerReferences = []metav1.OwnerReference{ - *metav1.NewControllerRef( - controllerManagerPod, - schema.GroupVersionKind{ - Group: corev1.SchemeGroupVersion.Group, - Version: corev1.SchemeGroupVersion.Version, - Kind: "Pod", - }, - ), + // It would be nice to use the OwnerReferences field here, but the agent pod is most likely in a + // different namespace than the kube-controller-manager pod, and therefore that breaks the + // OwnerReferences contract (see metav1.OwnerReference doc). + if agentPod.Annotations == nil { + agentPod.Annotations = make(map[string]string) } + agentPod.Annotations[controllerManagerNameAnnotationKey] = controllerManagerPod.Name + agentPod.Annotations[controllerManagerUIDAnnotationKey] = string(controllerManagerPod.UID) agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts agentPod.Spec.Volumes = controllerManagerPod.Spec.Volumes @@ -148,13 +147,13 @@ func isAgentPodUpToDate(actualAgentPod, expectedAgentPod *corev1.Pod) bool { func findAgentPod( controllerManagerPod *corev1.Pod, - informer corev1informers.PodInformer, + kubeSystemPodInformer corev1informers.PodInformer, + agentPodInformer corev1informers.PodInformer, agentLabels map[string]string, ) (*corev1.Pod, error) { agentSelector := labels.SelectorFromSet(agentLabels) - agentPods, err := informer. + agentPods, err := agentPodInformer. Lister(). - Pods(ControllerManagerNamespace). List(agentSelector) if err != nil { return nil, fmt.Errorf("informer cannot list agent pods: %w", err) @@ -163,7 +162,7 @@ func findAgentPod( for _, maybeAgentPod := range agentPods { maybeControllerManagerPod, err := findControllerManagerPod( maybeAgentPod, - informer, + kubeSystemPodInformer, ) if err != nil { return nil, err @@ -181,22 +180,28 @@ func findControllerManagerPod( agentPod *corev1.Pod, informer corev1informers.PodInformer, ) (*corev1.Pod, error) { - controller := metav1.GetControllerOf(agentPod) - if controller == nil { - klog.InfoS("found orphan agent pod", "pod", klog.KObj(agentPod)) + name, ok := agentPod.Annotations[controllerManagerNameAnnotationKey] + if !ok { + klog.InfoS("agent pod missing parent name annotation", "pod", agentPod) + return nil, nil + } + + uid, ok := agentPod.Annotations[controllerManagerUIDAnnotationKey] + if !ok { + klog.InfoS("agent pod missing parent uid annotation", "pod", agentPod) return nil, nil } maybeControllerManagerPod, err := informer. Lister(). Pods(ControllerManagerNamespace). - Get(controller.Name) + Get(name) notFound := k8serrors.IsNotFound(err) if err != nil && !notFound { return nil, fmt.Errorf("cannot get controller pod: %w", err) } else if notFound || maybeControllerManagerPod == nil || - maybeControllerManagerPod.UID != controller.UID { + string(maybeControllerManagerPod.UID) != uid { return nil, nil }