WIP: start to create kube-cert-agent pods in namespace
Signed-off-by: Andrew Keesler <akeesler@vmware.com>
This commit is contained in:
parent
e18b6fdddc
commit
50258fc569
@ -17,9 +17,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type createrController struct {
|
type createrController struct {
|
||||||
agentInfo *Info
|
agentInfo *Info
|
||||||
k8sClient kubernetes.Interface
|
k8sClient kubernetes.Interface
|
||||||
podInformer corev1informers.PodInformer
|
kubeSystemPodInformer corev1informers.PodInformer
|
||||||
|
agentPodInformer corev1informers.PodInformer
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewCreaterController returns a controller that creates new kube-cert-agent pods for every known
|
// NewCreaterController returns a controller that creates new kube-cert-agent pods for every known
|
||||||
@ -29,7 +30,8 @@ type createrController struct {
|
|||||||
func NewCreaterController(
|
func NewCreaterController(
|
||||||
agentInfo *Info,
|
agentInfo *Info,
|
||||||
k8sClient kubernetes.Interface,
|
k8sClient kubernetes.Interface,
|
||||||
podInformer corev1informers.PodInformer,
|
kubeSystemPodInformer corev1informers.PodInformer,
|
||||||
|
agentPodInformer corev1informers.PodInformer,
|
||||||
withInformer pinnipedcontroller.WithInformerOptionFunc,
|
withInformer pinnipedcontroller.WithInformerOptionFunc,
|
||||||
) controllerlib.Controller {
|
) controllerlib.Controller {
|
||||||
return controllerlib.New(
|
return controllerlib.New(
|
||||||
@ -37,15 +39,23 @@ func NewCreaterController(
|
|||||||
//nolint: misspell
|
//nolint: misspell
|
||||||
Name: "kube-cert-agent-creater-controller",
|
Name: "kube-cert-agent-creater-controller",
|
||||||
Syncer: &createrController{
|
Syncer: &createrController{
|
||||||
agentInfo: agentInfo,
|
agentInfo: agentInfo,
|
||||||
k8sClient: k8sClient,
|
k8sClient: k8sClient,
|
||||||
podInformer: podInformer,
|
kubeSystemPodInformer: kubeSystemPodInformer,
|
||||||
|
agentPodInformer: agentPodInformer,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
withInformer(
|
withInformer(
|
||||||
podInformer,
|
kubeSystemPodInformer,
|
||||||
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
|
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
|
||||||
return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels)
|
return isControllerManagerPod(obj)
|
||||||
|
}),
|
||||||
|
controllerlib.InformerOption{},
|
||||||
|
),
|
||||||
|
withInformer(
|
||||||
|
agentPodInformer,
|
||||||
|
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
|
||||||
|
return isAgentPod(obj, agentInfo.Template.Labels)
|
||||||
}),
|
}),
|
||||||
controllerlib.InformerOption{},
|
controllerlib.InformerOption{},
|
||||||
),
|
),
|
||||||
@ -59,13 +69,18 @@ func (c *createrController) Sync(ctx controllerlib.Context) error {
|
|||||||
return fmt.Errorf("cannot create controller manager selector: %w", err)
|
return fmt.Errorf("cannot create controller manager selector: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
controllerManagerPods, err := c.podInformer.Lister().List(controllerManagerSelector)
|
controllerManagerPods, err := c.kubeSystemPodInformer.Lister().List(controllerManagerSelector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("informer cannot list controller manager pods: %w", err)
|
return fmt.Errorf("informer cannot list controller manager pods: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, controllerManagerPod := range controllerManagerPods {
|
for _, controllerManagerPod := range controllerManagerPods {
|
||||||
agentPod, err := findAgentPod(controllerManagerPod, c.podInformer, c.agentInfo.Template.Labels)
|
agentPod, err := findAgentPod(
|
||||||
|
controllerManagerPod,
|
||||||
|
c.kubeSystemPodInformer,
|
||||||
|
c.agentPodInformer,
|
||||||
|
c.agentInfo.Template.Labels,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -80,7 +95,7 @@ func (c *createrController) Sync(ctx controllerlib.Context) error {
|
|||||||
klog.KObj(controllerManagerPod),
|
klog.KObj(controllerManagerPod),
|
||||||
)
|
)
|
||||||
_, err := c.k8sClient.CoreV1().
|
_, err := c.k8sClient.CoreV1().
|
||||||
Pods(ControllerManagerNamespace).
|
Pods(c.agentInfo.Template.Namespace).
|
||||||
Create(ctx.Context, agentPod, metav1.CreateOptions{})
|
Create(ctx.Context, agentPod, metav1.CreateOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("cannot create agent pod: %w", err)
|
return fmt.Errorf("cannot create agent pod: %w", err)
|
||||||
|
@ -30,7 +30,8 @@ func TestCreaterControllerFilter(t *testing.T) {
|
|||||||
"CreaterControllerFilter",
|
"CreaterControllerFilter",
|
||||||
func(
|
func(
|
||||||
agentPodTemplate *corev1.Pod,
|
agentPodTemplate *corev1.Pod,
|
||||||
podsInformer corev1informers.PodInformer,
|
kubeSystemPodInformer corev1informers.PodInformer,
|
||||||
|
//agentPodInformer corev1informers.PodInformer,
|
||||||
observableWithInformerOption *testutil.ObservableWithInformerOption,
|
observableWithInformerOption *testutil.ObservableWithInformerOption,
|
||||||
) {
|
) {
|
||||||
_ = NewCreaterController(
|
_ = NewCreaterController(
|
||||||
@ -38,7 +39,8 @@ func TestCreaterControllerFilter(t *testing.T) {
|
|||||||
Template: agentPodTemplate,
|
Template: agentPodTemplate,
|
||||||
},
|
},
|
||||||
nil, // k8sClient, shouldn't matter
|
nil, // k8sClient, shouldn't matter
|
||||||
podsInformer,
|
kubeSystemPodInformer,
|
||||||
|
nil, //agentPodInformer,
|
||||||
observableWithInformerOption.WithInformer,
|
observableWithInformerOption.WithInformer,
|
||||||
)
|
)
|
||||||
},
|
},
|
||||||
@ -53,15 +55,18 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
|
|
||||||
var subject controllerlib.Controller
|
var subject controllerlib.Controller
|
||||||
var kubeAPIClient *kubernetesfake.Clientset
|
var kubeAPIClient *kubernetesfake.Clientset
|
||||||
var kubeInformerClient *kubernetesfake.Clientset
|
var kubeSystemInformerClient *kubernetesfake.Clientset
|
||||||
var kubeInformers kubeinformers.SharedInformerFactory
|
var kubeSystemInformers kubeinformers.SharedInformerFactory
|
||||||
|
var agentInformerClient *kubernetesfake.Clientset
|
||||||
|
var agentInformers kubeinformers.SharedInformerFactory
|
||||||
var timeoutContext context.Context
|
var timeoutContext context.Context
|
||||||
var timeoutContextCancel context.CancelFunc
|
var timeoutContextCancel context.CancelFunc
|
||||||
var syncContext *controllerlib.Context
|
var syncContext *controllerlib.Context
|
||||||
|
|
||||||
agentPodTemplate := &corev1.Pod{
|
agentPodTemplate := &corev1.Pod{
|
||||||
ObjectMeta: metav1.ObjectMeta{
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
Name: "some-agent-name-",
|
Namespace: "some-agent-namespace",
|
||||||
|
Name: "some-agent-name-",
|
||||||
Labels: map[string]string{
|
Labels: map[string]string{
|
||||||
"some-label-key": "some-label-value",
|
"some-label-key": "some-label-value",
|
||||||
},
|
},
|
||||||
@ -117,17 +122,10 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
// fnv 32a hash of controller-manager uid
|
// fnv 32a hash of controller-manager uid
|
||||||
controllerManagerPodHash := "fbb0addd"
|
controllerManagerPodHash := "fbb0addd"
|
||||||
agentPod := agentPodTemplate.DeepCopy()
|
agentPod := agentPodTemplate.DeepCopy()
|
||||||
agentPod.Namespace = kubeSystemNamespace
|
|
||||||
agentPod.Name += controllerManagerPodHash
|
agentPod.Name += controllerManagerPodHash
|
||||||
agentPod.OwnerReferences = []metav1.OwnerReference{
|
agentPod.Annotations = map[string]string{
|
||||||
{
|
"kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name,
|
||||||
APIVersion: "v1",
|
"kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID),
|
||||||
Kind: "Pod",
|
|
||||||
Name: controllerManagerPod.Name,
|
|
||||||
UID: controllerManagerPod.UID,
|
|
||||||
Controller: boolPtr(true),
|
|
||||||
BlockOwnerDeletion: boolPtr(true),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
|
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
|
||||||
agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever
|
agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever
|
||||||
@ -151,7 +149,8 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
Template: agentPodTemplate,
|
Template: agentPodTemplate,
|
||||||
},
|
},
|
||||||
kubeAPIClient,
|
kubeAPIClient,
|
||||||
kubeInformers.Core().V1().Pods(),
|
kubeSystemInformers.Core().V1().Pods(),
|
||||||
|
agentInformers.Core().V1().Pods(),
|
||||||
controllerlib.WithInformer,
|
controllerlib.WithInformer,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -166,32 +165,39 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Must start informers before calling TestRunSynchronously()
|
// Must start informers before calling TestRunSynchronously()
|
||||||
kubeInformers.Start(timeoutContext.Done())
|
kubeSystemInformers.Start(timeoutContext.Done())
|
||||||
|
agentInformers.Start(timeoutContext.Done())
|
||||||
controllerlib.TestRunSynchronously(t, subject)
|
controllerlib.TestRunSynchronously(t, subject)
|
||||||
}
|
}
|
||||||
|
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
r = require.New(t)
|
r = require.New(t)
|
||||||
|
|
||||||
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
|
|
||||||
|
|
||||||
kubeInformerClient = kubernetesfake.NewSimpleClientset()
|
|
||||||
kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
|
|
||||||
kubeAPIClient = kubernetesfake.NewSimpleClientset()
|
kubeAPIClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
|
||||||
|
kubeSystemInformerClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0)
|
||||||
|
|
||||||
|
agentInformerClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0)
|
||||||
|
|
||||||
|
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
|
||||||
|
|
||||||
// Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any
|
// Add a pod into the test that doesn't matter to make sure we don't accidentally trigger any
|
||||||
// logic on this thing.
|
// logic on this thing.
|
||||||
ignorablePod := corev1.Pod{}
|
ignorablePod := corev1.Pod{}
|
||||||
ignorablePod.Name = "some-ignorable-pod"
|
ignorablePod.Name = "some-ignorable-pod"
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(&ignorablePod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(&ignorablePod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod))
|
r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod))
|
||||||
|
|
||||||
// Add another valid agent pod to make sure our logic works for just the pod we care about.
|
// Add another valid agent pod to make sure our logic works for just the pod we care about.
|
||||||
otherAgentPod := agentPod.DeepCopy()
|
otherAgentPod := agentPod.DeepCopy()
|
||||||
otherAgentPod.Name = "some-other-agent"
|
otherAgentPod.Name = "some-other-agent"
|
||||||
otherAgentPod.OwnerReferences[0].Name = "some-other-controller-manager-name"
|
otherAgentPod.Annotations = map[string]string{
|
||||||
otherAgentPod.OwnerReferences[0].UID = "some-other-controller-manager-uid"
|
"kube-cert-agent.pinniped.dev/controller-manager-name": "some-other-controller-manager-name",
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(otherAgentPod))
|
"kube-cert-agent.pinniped.dev/controller-manager-uid": "some-other-controller-manager-uid",
|
||||||
|
}
|
||||||
|
r.NoError(agentInformerClient.Tracker().Add(otherAgentPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(otherAgentPod))
|
r.NoError(kubeAPIClient.Tracker().Add(otherAgentPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -201,13 +207,13 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
|
|
||||||
when("there is a controller manager pod", func() {
|
when("there is a controller manager pod", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
when("there is a matching agent pod", func() {
|
when("there is a matching agent pod", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(agentPod))
|
r.NoError(agentInformerClient.Tracker().Add(agentPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(agentPod))
|
r.NoError(kubeAPIClient.Tracker().Add(agentPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -227,8 +233,8 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
nonMatchingAgentPod := agentPod.DeepCopy()
|
nonMatchingAgentPod := agentPod.DeepCopy()
|
||||||
nonMatchingAgentPod.Name = "some-agent-name-85da432e"
|
nonMatchingAgentPod.Name = "some-agent-name-85da432e"
|
||||||
nonMatchingAgentPod.OwnerReferences[0].UID = "some-non-matching-uid"
|
nonMatchingAgentPod.Annotations[controllerManagerUIDAnnotationKey] = "some-non-matching-uid"
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(nonMatchingAgentPod))
|
r.NoError(agentInformerClient.Tracker().Add(nonMatchingAgentPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod))
|
r.NoError(kubeAPIClient.Tracker().Add(nonMatchingAgentPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -241,7 +247,7 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
[]coretesting.Action{
|
[]coretesting.Action{
|
||||||
coretesting.NewCreateAction(
|
coretesting.NewCreateAction(
|
||||||
podsGVR,
|
podsGVR,
|
||||||
kubeSystemNamespace,
|
agentPod.Namespace,
|
||||||
agentPod,
|
agentPod,
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
@ -260,7 +266,7 @@ func TestCreaterControllerSync(t *testing.T) {
|
|||||||
[]coretesting.Action{
|
[]coretesting.Action{
|
||||||
coretesting.NewCreateAction(
|
coretesting.NewCreateAction(
|
||||||
podsGVR,
|
podsGVR,
|
||||||
kubeSystemNamespace,
|
agentPod.Namespace,
|
||||||
agentPod,
|
agentPod,
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
|
@ -17,9 +17,10 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type deleterController struct {
|
type deleterController struct {
|
||||||
agentInfo *Info
|
agentInfo *Info
|
||||||
k8sClient kubernetes.Interface
|
k8sClient kubernetes.Interface
|
||||||
podInformer corev1informers.PodInformer
|
kubeSystemPodInformer corev1informers.PodInformer
|
||||||
|
agentPodInformer corev1informers.PodInformer
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewDeleterController returns a controller that deletes any kube-cert-agent pods that are out of
|
// NewDeleterController returns a controller that deletes any kube-cert-agent pods that are out of
|
||||||
@ -29,20 +30,22 @@ type deleterController struct {
|
|||||||
func NewDeleterController(
|
func NewDeleterController(
|
||||||
agentInfo *Info,
|
agentInfo *Info,
|
||||||
k8sClient kubernetes.Interface,
|
k8sClient kubernetes.Interface,
|
||||||
podInformer corev1informers.PodInformer,
|
kubeSystemPodInformer corev1informers.PodInformer,
|
||||||
|
agentPodInformer corev1informers.PodInformer,
|
||||||
withInformer pinnipedcontroller.WithInformerOptionFunc,
|
withInformer pinnipedcontroller.WithInformerOptionFunc,
|
||||||
) controllerlib.Controller {
|
) controllerlib.Controller {
|
||||||
return controllerlib.New(
|
return controllerlib.New(
|
||||||
controllerlib.Config{
|
controllerlib.Config{
|
||||||
Name: "kube-cert-agent-deleter-controller",
|
Name: "kube-cert-agent-deleter-controller",
|
||||||
Syncer: &deleterController{
|
Syncer: &deleterController{
|
||||||
agentInfo: agentInfo,
|
agentInfo: agentInfo,
|
||||||
k8sClient: k8sClient,
|
k8sClient: k8sClient,
|
||||||
podInformer: podInformer,
|
kubeSystemPodInformer: kubeSystemPodInformer,
|
||||||
|
agentPodInformer: agentPodInformer,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
withInformer(
|
withInformer(
|
||||||
podInformer,
|
agentPodInformer,
|
||||||
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
|
pinnipedcontroller.SimpleFilter(func(obj metav1.Object) bool {
|
||||||
return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels)
|
return isControllerManagerPod(obj) || isAgentPod(obj, agentInfo.Template.Labels)
|
||||||
}),
|
}),
|
||||||
@ -54,7 +57,7 @@ func NewDeleterController(
|
|||||||
// Sync implements controllerlib.Syncer.
|
// Sync implements controllerlib.Syncer.
|
||||||
func (c *deleterController) Sync(ctx controllerlib.Context) error {
|
func (c *deleterController) Sync(ctx controllerlib.Context) error {
|
||||||
agentSelector := labels.SelectorFromSet(c.agentInfo.Template.Labels)
|
agentSelector := labels.SelectorFromSet(c.agentInfo.Template.Labels)
|
||||||
agentPods, err := c.podInformer.
|
agentPods, err := c.agentPodInformer.
|
||||||
Lister().
|
Lister().
|
||||||
Pods(ControllerManagerNamespace).
|
Pods(ControllerManagerNamespace).
|
||||||
List(agentSelector)
|
List(agentSelector)
|
||||||
@ -63,7 +66,7 @@ func (c *deleterController) Sync(ctx controllerlib.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
for _, agentPod := range agentPods {
|
for _, agentPod := range agentPods {
|
||||||
controllerManagerPod, err := findControllerManagerPod(agentPod, c.podInformer)
|
controllerManagerPod, err := findControllerManagerPod(agentPod, c.kubeSystemPodInformer)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -53,8 +53,10 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
|
|
||||||
var subject controllerlib.Controller
|
var subject controllerlib.Controller
|
||||||
var kubeAPIClient *kubernetesfake.Clientset
|
var kubeAPIClient *kubernetesfake.Clientset
|
||||||
var kubeInformerClient *kubernetesfake.Clientset
|
var kubeSystemInformerClient *kubernetesfake.Clientset
|
||||||
var kubeInformers kubeinformers.SharedInformerFactory
|
var kubeSystemInformers kubeinformers.SharedInformerFactory
|
||||||
|
var agentInformerClient *kubernetesfake.Clientset
|
||||||
|
var agentInformers kubeinformers.SharedInformerFactory
|
||||||
var timeoutContext context.Context
|
var timeoutContext context.Context
|
||||||
var timeoutContextCancel context.CancelFunc
|
var timeoutContextCancel context.CancelFunc
|
||||||
var syncContext *controllerlib.Context
|
var syncContext *controllerlib.Context
|
||||||
@ -125,15 +127,9 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
agentPod := agentPodTemplate.DeepCopy()
|
agentPod := agentPodTemplate.DeepCopy()
|
||||||
agentPod.Namespace = kubeSystemNamespace
|
agentPod.Namespace = kubeSystemNamespace
|
||||||
agentPod.Name += controllerManagerPodHash
|
agentPod.Name += controllerManagerPodHash
|
||||||
agentPod.OwnerReferences = []metav1.OwnerReference{
|
agentPod.Annotations = map[string]string{
|
||||||
{
|
"kube-cert-agent.pinniped.dev/controller-manager-name": controllerManagerPod.Name,
|
||||||
APIVersion: "v1",
|
"kube-cert-agent.pinniped.dev/controller-manager-uid": string(controllerManagerPod.UID),
|
||||||
Kind: "Pod",
|
|
||||||
Name: controllerManagerPod.Name,
|
|
||||||
UID: controllerManagerPod.UID,
|
|
||||||
Controller: boolPtr(true),
|
|
||||||
BlockOwnerDeletion: boolPtr(true),
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
|
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
|
||||||
agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever
|
agentPod.Spec.RestartPolicy = corev1.RestartPolicyNever
|
||||||
@ -151,7 +147,8 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
Template: agentPodTemplate,
|
Template: agentPodTemplate,
|
||||||
},
|
},
|
||||||
kubeAPIClient,
|
kubeAPIClient,
|
||||||
kubeInformers.Core().V1().Pods(),
|
kubeSystemInformers.Core().V1().Pods(),
|
||||||
|
agentInformers.Core().V1().Pods(),
|
||||||
controllerlib.WithInformer,
|
controllerlib.WithInformer,
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -166,7 +163,8 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Must start informers before calling TestRunSynchronously()
|
// Must start informers before calling TestRunSynchronously()
|
||||||
kubeInformers.Start(timeoutContext.Done())
|
kubeSystemInformers.Start(timeoutContext.Done())
|
||||||
|
agentInformers.Start(timeoutContext.Done())
|
||||||
controllerlib.TestRunSynchronously(t, subject)
|
controllerlib.TestRunSynchronously(t, subject)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -175,15 +173,19 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
|
|
||||||
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
|
timeoutContext, timeoutContextCancel = context.WithTimeout(context.Background(), time.Second*3)
|
||||||
|
|
||||||
kubeInformerClient = kubernetesfake.NewSimpleClientset()
|
|
||||||
kubeInformers = kubeinformers.NewSharedInformerFactory(kubeInformerClient, 0)
|
|
||||||
kubeAPIClient = kubernetesfake.NewSimpleClientset()
|
kubeAPIClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
|
||||||
|
kubeSystemInformerClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
kubeSystemInformers = kubeinformers.NewSharedInformerFactory(kubeSystemInformerClient, 0)
|
||||||
|
|
||||||
|
agentInformerClient = kubernetesfake.NewSimpleClientset()
|
||||||
|
agentInformers = kubeinformers.NewSharedInformerFactory(agentInformerClient, 0)
|
||||||
|
|
||||||
// Add an pod into the test that doesn't matter to make sure we don't accidentally
|
// Add an pod into the test that doesn't matter to make sure we don't accidentally
|
||||||
// trigger any logic on this thing.
|
// trigger any logic on this thing.
|
||||||
ignorablePod := corev1.Pod{}
|
ignorablePod := corev1.Pod{}
|
||||||
ignorablePod.Name = "some-ignorable-pod"
|
ignorablePod.Name = "some-ignorable-pod"
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(&ignorablePod))
|
r.NoError(agentInformerClient.Tracker().Add(&ignorablePod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod))
|
r.NoError(kubeAPIClient.Tracker().Add(&ignorablePod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -193,13 +195,13 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
|
|
||||||
when("there is an agent pod", func() {
|
when("there is an agent pod", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(agentPod))
|
r.NoError(agentInformerClient.Tracker().Add(agentPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(agentPod))
|
r.NoError(kubeAPIClient.Tracker().Add(agentPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
when("there is a matching controller manager pod", func() {
|
when("there is a matching controller manager pod", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -218,7 +220,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
when("there is a non-matching controller manager pod via uid", func() {
|
when("there is a non-matching controller manager pod via uid", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
controllerManagerPod.UID = "some-other-controller-manager-uid"
|
controllerManagerPod.UID = "some-other-controller-manager-uid"
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -243,7 +245,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
when("there is a non-matching controller manager pod via name", func() {
|
when("there is a non-matching controller manager pod via name", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
controllerManagerPod.Name = "some-other-controller-manager-name"
|
controllerManagerPod.Name = "some-other-controller-manager-name"
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -272,7 +274,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
Name: "some-other-volume-mount",
|
Name: "some-other-volume-mount",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -301,7 +303,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
Name: "some-other-volume",
|
Name: "some-other-volume",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -328,7 +330,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
controllerManagerPod.Spec.NodeSelector = map[string]string{
|
controllerManagerPod.Spec.NodeSelector = map[string]string{
|
||||||
"some-other-node-selector-key": "some-other-node-selector-value",
|
"some-other-node-selector-key": "some-other-node-selector-value",
|
||||||
}
|
}
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -353,7 +355,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
when("the agent pod is out of sync with the controller manager via node name", func() {
|
when("the agent pod is out of sync with the controller manager via node name", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
controllerManagerPod.Spec.NodeName = "some-other-node-name"
|
controllerManagerPod.Spec.NodeName = "some-other-node-name"
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -382,7 +384,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
Key: "some-other-toleration-key",
|
Key: "some-other-toleration-key",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
r.NoError(kubeInformerClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeSystemInformerClient.Tracker().Add(controllerManagerPod))
|
||||||
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
r.NoError(kubeAPIClient.Tracker().Add(controllerManagerPod))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -408,7 +410,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
updatedAgentPod := agentPod.DeepCopy()
|
updatedAgentPod := agentPod.DeepCopy()
|
||||||
updatedAgentPod.Spec.RestartPolicy = corev1.RestartPolicyAlways
|
updatedAgentPod.Spec.RestartPolicy = corev1.RestartPolicyAlways
|
||||||
r.NoError(kubeInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace))
|
r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace))
|
||||||
r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace))
|
r.NoError(kubeAPIClient.Tracker().Update(podsGVR, updatedAgentPod, updatedAgentPod.Namespace))
|
||||||
})
|
})
|
||||||
|
|
||||||
@ -433,7 +435,7 @@ func TestDeleterControllerSync(t *testing.T) {
|
|||||||
when("the agent pod is out of sync via automount service account tokem", func() {
|
when("the agent pod is out of sync via automount service account tokem", func() {
|
||||||
it.Before(func() {
|
it.Before(func() {
|
||||||
agentPod.Spec.AutomountServiceAccountToken = boolPtr(true)
|
agentPod.Spec.AutomountServiceAccountToken = boolPtr(true)
|
||||||
r.NoError(kubeInformerClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace))
|
r.NoError(kubeSystemInformerClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace))
|
||||||
r.NoError(kubeAPIClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace))
|
r.NoError(kubeAPIClient.Tracker().Update(podsGVR, agentPod, agentPod.Namespace))
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -19,7 +19,6 @@ import (
|
|||||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
"k8s.io/apimachinery/pkg/labels"
|
"k8s.io/apimachinery/pkg/labels"
|
||||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
||||||
corev1informers "k8s.io/client-go/informers/core/v1"
|
corev1informers "k8s.io/client-go/informers/core/v1"
|
||||||
"k8s.io/klog/v2"
|
"k8s.io/klog/v2"
|
||||||
)
|
)
|
||||||
@ -27,12 +26,16 @@ import (
|
|||||||
const (
|
const (
|
||||||
// ControllerManagerNamespace is the assumed namespace of the kube-controller-manager pod(s).
|
// ControllerManagerNamespace is the assumed namespace of the kube-controller-manager pod(s).
|
||||||
ControllerManagerNamespace = "kube-system"
|
ControllerManagerNamespace = "kube-system"
|
||||||
|
|
||||||
|
controllerManagerNameAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-name"
|
||||||
|
controllerManagerUIDAnnotationKey = "kube-cert-agent.pinniped.dev/controller-manager-uid"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Info holds necessary information about the agent pod. It was pulled out into a struct to have a
|
// Info holds necessary information about the agent pod. It was pulled out into a struct to have a
|
||||||
// common parameter for each controller.
|
// common parameter for each controller.
|
||||||
type Info struct {
|
type Info struct {
|
||||||
// Template is an injection point for pod fields. The required pod fields are as follows.
|
// Template is an injection point for pod fields. The required pod fields are as follows.
|
||||||
|
// .Namespace: serves as the namespace of the agent pods
|
||||||
// .Name: serves as the name prefix for each of the agent pods
|
// .Name: serves as the name prefix for each of the agent pods
|
||||||
// .Labels: serves as a way to filter for agent pods
|
// .Labels: serves as a way to filter for agent pods
|
||||||
// .Spec.Containers[0].Image: serves as the container image for the agent pods
|
// .Spec.Containers[0].Image: serves as the container image for the agent pods
|
||||||
@ -40,11 +43,11 @@ type Info struct {
|
|||||||
Template *corev1.Pod
|
Template *corev1.Pod
|
||||||
|
|
||||||
// CertPathAnnotation is the name of the annotation key that will be used when setting the
|
// CertPathAnnotation is the name of the annotation key that will be used when setting the
|
||||||
// best-guess path to the kube API's certificate.
|
// best-guess path to the kube API's certificate in the agent pod.
|
||||||
CertPathAnnotation string
|
CertPathAnnotation string
|
||||||
|
|
||||||
// KeyPathAnnotation is the name of the annotation key that will be used when setting the
|
// KeyPathAnnotation is the name of the annotation key that will be used when setting the
|
||||||
// best-guess path to the kube API's private key.
|
// best-guess path to the kube API's private key in the agent pod.
|
||||||
KeyPathAnnotation string
|
KeyPathAnnotation string
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -91,18 +94,14 @@ func newAgentPod(
|
|||||||
|
|
||||||
agentPod.Name = fmt.Sprintf("%s%s", agentPod.Name, hash(controllerManagerPod))
|
agentPod.Name = fmt.Sprintf("%s%s", agentPod.Name, hash(controllerManagerPod))
|
||||||
|
|
||||||
// controller manager namespace because it is our owner
|
// It would be nice to use the OwnerReferences field here, but the agent pod is most likely in a
|
||||||
agentPod.Namespace = ControllerManagerNamespace
|
// different namespace than the kube-controller-manager pod, and therefore that breaks the
|
||||||
agentPod.OwnerReferences = []metav1.OwnerReference{
|
// OwnerReferences contract (see metav1.OwnerReference doc).
|
||||||
*metav1.NewControllerRef(
|
if agentPod.Annotations == nil {
|
||||||
controllerManagerPod,
|
agentPod.Annotations = make(map[string]string)
|
||||||
schema.GroupVersionKind{
|
|
||||||
Group: corev1.SchemeGroupVersion.Group,
|
|
||||||
Version: corev1.SchemeGroupVersion.Version,
|
|
||||||
Kind: "Pod",
|
|
||||||
},
|
|
||||||
),
|
|
||||||
}
|
}
|
||||||
|
agentPod.Annotations[controllerManagerNameAnnotationKey] = controllerManagerPod.Name
|
||||||
|
agentPod.Annotations[controllerManagerUIDAnnotationKey] = string(controllerManagerPod.UID)
|
||||||
|
|
||||||
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
|
agentPod.Spec.Containers[0].VolumeMounts = controllerManagerPod.Spec.Containers[0].VolumeMounts
|
||||||
agentPod.Spec.Volumes = controllerManagerPod.Spec.Volumes
|
agentPod.Spec.Volumes = controllerManagerPod.Spec.Volumes
|
||||||
@ -148,13 +147,13 @@ func isAgentPodUpToDate(actualAgentPod, expectedAgentPod *corev1.Pod) bool {
|
|||||||
|
|
||||||
func findAgentPod(
|
func findAgentPod(
|
||||||
controllerManagerPod *corev1.Pod,
|
controllerManagerPod *corev1.Pod,
|
||||||
informer corev1informers.PodInformer,
|
kubeSystemPodInformer corev1informers.PodInformer,
|
||||||
|
agentPodInformer corev1informers.PodInformer,
|
||||||
agentLabels map[string]string,
|
agentLabels map[string]string,
|
||||||
) (*corev1.Pod, error) {
|
) (*corev1.Pod, error) {
|
||||||
agentSelector := labels.SelectorFromSet(agentLabels)
|
agentSelector := labels.SelectorFromSet(agentLabels)
|
||||||
agentPods, err := informer.
|
agentPods, err := agentPodInformer.
|
||||||
Lister().
|
Lister().
|
||||||
Pods(ControllerManagerNamespace).
|
|
||||||
List(agentSelector)
|
List(agentSelector)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("informer cannot list agent pods: %w", err)
|
return nil, fmt.Errorf("informer cannot list agent pods: %w", err)
|
||||||
@ -163,7 +162,7 @@ func findAgentPod(
|
|||||||
for _, maybeAgentPod := range agentPods {
|
for _, maybeAgentPod := range agentPods {
|
||||||
maybeControllerManagerPod, err := findControllerManagerPod(
|
maybeControllerManagerPod, err := findControllerManagerPod(
|
||||||
maybeAgentPod,
|
maybeAgentPod,
|
||||||
informer,
|
kubeSystemPodInformer,
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -181,22 +180,28 @@ func findControllerManagerPod(
|
|||||||
agentPod *corev1.Pod,
|
agentPod *corev1.Pod,
|
||||||
informer corev1informers.PodInformer,
|
informer corev1informers.PodInformer,
|
||||||
) (*corev1.Pod, error) {
|
) (*corev1.Pod, error) {
|
||||||
controller := metav1.GetControllerOf(agentPod)
|
name, ok := agentPod.Annotations[controllerManagerNameAnnotationKey]
|
||||||
if controller == nil {
|
if !ok {
|
||||||
klog.InfoS("found orphan agent pod", "pod", klog.KObj(agentPod))
|
klog.InfoS("agent pod missing parent name annotation", "pod", agentPod)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
uid, ok := agentPod.Annotations[controllerManagerUIDAnnotationKey]
|
||||||
|
if !ok {
|
||||||
|
klog.InfoS("agent pod missing parent uid annotation", "pod", agentPod)
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
maybeControllerManagerPod, err := informer.
|
maybeControllerManagerPod, err := informer.
|
||||||
Lister().
|
Lister().
|
||||||
Pods(ControllerManagerNamespace).
|
Pods(ControllerManagerNamespace).
|
||||||
Get(controller.Name)
|
Get(name)
|
||||||
notFound := k8serrors.IsNotFound(err)
|
notFound := k8serrors.IsNotFound(err)
|
||||||
if err != nil && !notFound {
|
if err != nil && !notFound {
|
||||||
return nil, fmt.Errorf("cannot get controller pod: %w", err)
|
return nil, fmt.Errorf("cannot get controller pod: %w", err)
|
||||||
} else if notFound ||
|
} else if notFound ||
|
||||||
maybeControllerManagerPod == nil ||
|
maybeControllerManagerPod == nil ||
|
||||||
maybeControllerManagerPod.UID != controller.UID {
|
string(maybeControllerManagerPod.UID) != uid {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user