diff --git a/internal/concierge/impersonator/config.go b/internal/concierge/impersonator/config.go index 7ced9b66..20d97647 100644 --- a/internal/concierge/impersonator/config.go +++ b/internal/concierge/impersonator/config.go @@ -45,9 +45,13 @@ type Config struct { // Enable or disable the impersonation proxy. Optional. Defaults to ModeAuto. Mode Mode `json:"mode,omitempty"` - // The HTTPS URL of the impersonation proxy for clients to use from outside the cluster. Used when creating TLS - // certificates and for clients to discover the endpoint. Optional. When not specified, if the impersonation proxy - // is started, then it will automatically create a LoadBalancer Service and use its ingress as the endpoint. + // Used when creating TLS certificates and for clients to discover the endpoint. Optional. When not specified, if the + // impersonation proxy is started, then it will automatically create a LoadBalancer Service and use its ingress as the + // endpoint. + // + // When specified, it may be a hostname or IP address, optionally with a port number, of the impersonation proxy + // for clients to use from outside the cluster. E.g. myhost.mycompany.com:8443. Clients should assume that they should + // connect via HTTPS to this service. Endpoint string `json:"endpoint,omitempty"` // The TLS configuration of the impersonation proxy's endpoints. Optional. When not specified, a CA and TLS diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go index 508d17a4..98dfd046 100644 --- a/internal/controller/impersonatorconfig/impersonator_config.go +++ b/internal/controller/impersonatorconfig/impersonator_config.go @@ -6,7 +6,9 @@ package impersonatorconfig import ( "context" "crypto/tls" + "crypto/x509" "crypto/x509/pkix" + "encoding/pem" "errors" "fmt" "net" @@ -170,11 +172,13 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { } if c.shouldHaveTLSSecret(config) { - if err = c.ensureTLSSecretIsCreated(ctx.Context, config); err != nil { - // return err // TODO + err = c.ensureTLSSecret(ctx, config) + if err != nil { + return err } } else { - if err = c.ensureTLSSecretIsRemoved(ctx.Context); err != nil { + err = c.ensureTLSSecretIsRemoved(ctx.Context) + if err != nil { return err } } @@ -184,8 +188,24 @@ func (c *impersonatorConfigController) Sync(ctx controllerlib.Context) error { return nil } -func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { - return c.shouldHaveImpersonator(config) +func (c *impersonatorConfigController) ensureTLSSecret(ctx controllerlib.Context, config *impersonator.Config) error { + secret, err := c.secretsInformer.Lister().Secrets(c.namespace).Get(c.tlsSecretName) + notFound := k8serrors.IsNotFound(err) + if notFound { + secret = nil + } + if !notFound && err != nil { + return err + } + //nolint:staticcheck // TODO remove this nolint when we fix the TODO below + if secret, err = c.deleteTLSCertificateWithWrongName(ctx.Context, config, secret); err != nil { + // TODO + // return err + } + if err = c.ensureTLSSecretIsCreatedAndLoaded(ctx.Context, config, secret); err != nil { + return err + } + return nil } func (c *impersonatorConfigController) shouldHaveImpersonator(config *impersonator.Config) bool { @@ -196,6 +216,10 @@ func (c *impersonatorConfigController) shouldHaveLoadBalancer(config *impersonat return c.shouldHaveImpersonator(config) && config.Endpoint == "" } +func (c *impersonatorConfigController) shouldHaveTLSSecret(config *impersonator.Config) bool { + return c.shouldHaveImpersonator(config) // TODO is this the logic that we want here? +} + func (c *impersonatorConfigController) ensureImpersonatorIsStopped() error { if c.server != nil { plog.Info("Stopping impersonation proxy", "port", impersonationProxyPort) @@ -266,26 +290,6 @@ func (c *impersonatorConfigController) tlsSecretExists() (bool, *v1.Secret, erro return true, secret, nil } -func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { - running, err := c.isLoadBalancerRunning() - if err != nil { - return err - } - if !running { - return nil - } - - plog.Info("Deleting load balancer for impersonation proxy", - "service", c.generatedLoadBalancerServiceName, - "namespace", c.namespace) - err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) - if err != nil { - return err - } - - return nil -} - func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.Context) error { running, err := c.isLoadBalancerRunning() if err != nil { @@ -323,56 +327,147 @@ func (c *impersonatorConfigController) ensureLoadBalancerIsStarted(ctx context.C return nil } -func (c *impersonatorConfigController) ensureTLSSecretIsCreated(ctx context.Context, config *impersonator.Config) error { - tlsSecretExists, tlsSecret, err := c.tlsSecretExists() +func (c *impersonatorConfigController) ensureLoadBalancerIsStopped(ctx context.Context) error { + running, err := c.isLoadBalancerRunning() if err != nil { return err } - if tlsSecretExists { - certPEM := tlsSecret.Data[v1.TLSCertKey] - keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] - tlsCert, _ := tls.X509KeyPair(certPEM, keyPEM) - // TODO handle err, like when the Secret did not contain the fields that we expected - c.setTLSCert(&tlsCert) + if !running { return nil } - impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) // TODO change the length of this + + plog.Info("Deleting load balancer for impersonation proxy", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + err = c.k8sClient.CoreV1().Services(c.namespace).Delete(ctx, c.generatedLoadBalancerServiceName, metav1.DeleteOptions{}) if err != nil { - return fmt.Errorf("could not create impersonation CA: %w", err) + return err } - var ips []net.IP - if config.Endpoint == "" { // TODO are there other cases where we need to do this? - lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) - notFound := k8serrors.IsNotFound(err) - if notFound { - return nil + + return nil +} + +func (c *impersonatorConfigController) deleteTLSCertificateWithWrongName(ctx context.Context, config *impersonator.Config, secret *v1.Secret) (*v1.Secret, error) { + if secret == nil { + // There is no Secret, so there is nothing to delete. + return secret, nil + } + + certPEM := secret.Data[v1.TLSCertKey] + block, _ := pem.Decode(certPEM) + if block == nil { + // The certPEM is not valid. + return secret, nil // TODO what should we do? + } + parsed, _ := x509.ParseCertificate(block.Bytes) + // TODO handle err + + desiredIPs, nameIsReady, err := c.findTLSCertificateName(config) + //nolint:staticcheck // TODO remove this nolint when we fix the TODO below + if err != nil { + // TODO return err + } + if !nameIsReady { + // We currently have a secret but we are waiting for a load balancer to be assigned an ingress, so + // our current secret must be old/unwanted. + err = c.ensureTLSSecretIsRemoved(ctx) + if err != nil { + return secret, err } + return nil, nil + } + + actualIPs := parsed.IPAddresses + // TODO handle multiple IPs, and handle when there is no IP + if desiredIPs[0].Equal(actualIPs[0]) { + // The cert matches the desired state, so we do not need to delete it. + return secret, nil + } + + err = c.ensureTLSSecretIsRemoved(ctx) + if err != nil { + return secret, err + } + return nil, nil +} + +func (c *impersonatorConfigController) ensureTLSSecretIsCreatedAndLoaded(ctx context.Context, config *impersonator.Config, secret *v1.Secret) error { + if secret != nil { + err := c.loadTLSCertFromSecret(secret) if err != nil { return err } + return nil + } + + // TODO create/save/watch the CA separately so we can reuse it to mint tls certs as the settings are dynamically changed, + // so that clients don't need to be updated to use a different CA just because the server-side settings were changed. + impersonationCA, err := certauthority.New(pkix.Name{CommonName: "test CA"}, 24*time.Hour) // TODO change the expiration of this to 100 years + if err != nil { + return fmt.Errorf("could not create impersonation CA: %w", err) + } + + ips, nameIsReady, err := c.findTLSCertificateName(config) + if err != nil { + return err + } + if !nameIsReady { + // Sync will get called again when the load balancer is updated with its ingress info, so this is not an error. + return nil + } + + newTLSSecret, err := c.createNewTLSSecret(ctx, impersonationCA, ips) + if err != nil { + return err + } + + err = c.loadTLSCertFromSecret(newTLSSecret) + if err != nil { + return err + } + + return nil +} + +func (c *impersonatorConfigController) findTLSCertificateName(config *impersonator.Config) ([]net.IP, bool, error) { + var ips []net.IP + if config.Endpoint != "" { + // TODO Endpoint could be a hostname + // TODO Endpoint could have a port number in it, which we should parse out and ignore for this purpose + ips = []net.IP{net.ParseIP(config.Endpoint)} + } else { + lb, err := c.servicesInformer.Lister().Services(c.namespace).Get(c.generatedLoadBalancerServiceName) + notFound := k8serrors.IsNotFound(err) + if notFound { + // TODO is this an error? we should have already created the load balancer, so why would it not exist here? + return nil, false, nil + } + if err != nil { + return nil, false, err + } ingresses := lb.Status.LoadBalancer.Ingress if len(ingresses) == 0 { - return nil + plog.Info("load balancer for impersonation proxy does not have an ingress yet, so skipping tls cert generation while we wait", + "service", c.generatedLoadBalancerServiceName, + "namespace", c.namespace) + return nil, false, nil } - ip := ingresses[0].IP // TODO multiple ips + ip := ingresses[0].IP // TODO handle multiple ips? ips = []net.IP{net.ParseIP(ip)} - // check with informer to get the ip address of the load balancer if its available - // if not, return - } else { - ips = []net.IP{net.ParseIP(config.Endpoint)} } - impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, ips, 24*time.Hour) // TODO change the length of this + return ips, true, nil +} + +func (c *impersonatorConfigController) createNewTLSSecret(ctx context.Context, ca *certauthority.CA, ips []net.IP) (*v1.Secret, error) { + impersonationCert, err := ca.Issue(pkix.Name{}, nil, ips, 24*time.Hour) // TODO change the length of this too 100 years for now? if err != nil { - return fmt.Errorf("could not create impersonation cert: %w", err) + return nil, fmt.Errorf("could not create impersonation cert: %w", err) } - c.setTLSCert(impersonationCert) + certPEM, keyPEM, _ := certauthority.ToPEM(impersonationCert) + // TODO handle err - certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) - // TODO error handling? - - // TODO handle error on create - c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, &v1.Secret{ + newTLSSecret := &v1.Secret{ Type: v1.SecretTypeTLS, ObjectMeta: metav1.ObjectMeta{ Name: c.tlsSecretName, @@ -380,11 +475,30 @@ func (c *impersonatorConfigController) ensureTLSSecretIsCreated(ctx context.Cont Labels: c.labels, }, Data: map[string][]byte{ - "ca.crt": impersonationCA.Bundle(), + "ca.crt": ca.Bundle(), v1.TLSPrivateKeyKey: keyPEM, v1.TLSCertKey: certPEM, }, - }, metav1.CreateOptions{}) + } + _, _ = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx, newTLSSecret, metav1.CreateOptions{}) + // TODO handle error on create + return newTLSSecret, nil +} + +func (c *impersonatorConfigController) loadTLSCertFromSecret(tlsSecret *v1.Secret) error { + certPEM := tlsSecret.Data[v1.TLSCertKey] + keyPEM := tlsSecret.Data[v1.TLSPrivateKeyKey] + tlsCert, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + plog.Error("Could not parse TLS cert PEM data from Secret", + err, + "secret", c.tlsSecretName, + "namespace", c.namespace, + ) + // TODO clear the secret if it was already set previously... c.setTLSCert(nil) + return fmt.Errorf("could not parse TLS cert PEM data from Secret: %w", err) + } + c.setTLSCert(&tlsCert) return nil } @@ -404,6 +518,8 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont return err } + c.setTLSCert(nil) + return nil } diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go index b6fee18e..c63a9c54 100644 --- a/internal/controller/impersonatorconfig/impersonator_config_test.go +++ b/internal/controller/impersonatorconfig/impersonator_config_test.go @@ -289,7 +289,6 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { return nil, startTLSListenerFuncError } var err error - //nolint: gosec // Intentionally binding to all network interfaces. startedTLSListener, err = tls.Listen(network, "127.0.0.1:0", config) // automatically choose the port for unit tests r.NoError(err) return &tlsListenerWrapper{listener: startedTLSListener, closeError: startTLSListenerUponCloseError}, nil @@ -306,13 +305,26 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { } } - var requireTLSServerIsRunning = func(caCrt []byte) { + var requireTLSServerIsRunning = func(caCrt []byte, addr string, dnsOverrides map[string]string) { r.Greater(startTLSListenerFuncWasCalled, 0) + realDialer := &net.Dialer{} + overrideDialContext := func(ctx context.Context, network, addr string) (net.Conn, error) { + replacementAddr, hasKey := dnsOverrides[addr] + if hasKey { + t.Logf("DialContext replacing addr %s with %s", addr, replacementAddr) + addr = replacementAddr + } else if dnsOverrides != nil { + t.Fatal("dnsOverrides was provided but not used, which was probably a mistake") + } + return realDialer.DialContext(ctx, network, addr) + } + var tr *http.Transport if caCrt == nil { tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + DialContext: overrideDialContext, } } else { rootCAs := x509.NewCertPool() @@ -320,10 +332,11 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { tr = &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: rootCAs}, + DialContext: overrideDialContext, } } client := &http.Client{Transport: tr} - url := "https://" + startedTLSListener.Addr().String() + url := "https://" + addr req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) resp, err := client.Do(req) @@ -347,7 +360,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { url := "https://" + startedTLSListener.Addr().String() req, err := http.NewRequestWithContext(context.Background(), "GET", url, nil) r.NoError(err) - _, err = client.Do(req) + _, err = client.Do(req) //nolint:bodyclose r.Error(err) r.Regexp("Get .*: remote error: tls: unrecognized name", err.Error()) } @@ -357,7 +370,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { _, err := tls.Dial( startedTLSListener.Addr().Network(), startedTLSListener.Addr().String(), - &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // TODO once we're using certs, do not skip verify + &tls.Config{InsecureSkipVerify: true}, //nolint:gosec ) r.Error(err) r.Regexp(`dial tcp .*: connect: connection refused`, err.Error()) @@ -380,6 +393,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }, 10*time.Second, time.Millisecond) } + var waitForTLSCertSecretToBeDeleted = func(informer corev1informers.SecretInformer, name string) { + r.Eventually(func() bool { + _, err := informer.Lister().Secrets(installedInNamespace).Get(name) + return k8serrors.IsNotFound(err) + }, 10*time.Second, time.Millisecond) + } + // Defer starting the informers until the last possible moment so that the // nested Before's can keep adding things to the informer caches. var startInformersAndController = func() { @@ -477,6 +497,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { impersonationCert, err := impersonationCA.Issue(pkix.Name{}, nil, []net.IP{net.ParseIP("127.0.0.1")}, 24*time.Hour) r.NoError(err) certPEM, keyPEM, err := certauthority.ToPEM(impersonationCert) + r.NoError(err) return &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -512,6 +533,31 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(client.Tracker().Add(loadBalancerService)) } + var updateLoadBalancerServiceInTracker = func(resourceName, lbIngressIP, newResourceVersion string) { + loadBalancerService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: resourceName, + Namespace: installedInNamespace, + ResourceVersion: newResourceVersion, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: lbIngressIP}, + }, + }, + }, + } + r.NoError(kubeInformerClient.Tracker().Update( + schema.GroupVersionResource{Version: "v1", Resource: "services"}, + loadBalancerService, + installedInNamespace, + )) + } + var addLoadBalancerServiceWithIPToTracker = func(resourceName string, client *kubernetesfake.Clientset) { loadBalancerService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -544,6 +590,20 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { )) } + var deleteTLSCertSecretFromTracker = func(resourceName string, client *kubernetesfake.Clientset) { + r.NoError(client.Tracker().Delete( + schema.GroupVersionResource{Version: "v1", Resource: "secrets"}, + installedInNamespace, + resourceName, + )) + } + + var addSecretFromCreateActionToTracker = func(action coretesting.Action, client *kubernetesfake.Clientset, resourceVersion string) { + createdSecret := action.(coretesting.CreateAction).GetObject().(*corev1.Secret) + createdSecret.ResourceVersion = resourceVersion + r.NoError(client.Tracker().Add(createdSecret)) + } + var addNodeWithRoleToTracker = func(role string) { r.NoError(kubeAPIClient.Tracker().Add( &corev1.Node{ @@ -740,18 +800,16 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a second time r.Len(kubeAPIClient.Actions(), 3) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[2]) - requireTLSServerIsRunning(ca) // running with certs now + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // running with certs now // update manually because the kubeAPIClient isn't connected to the informer in the tests - createdSecret := kubeAPIClient.Actions()[2].(coretesting.CreateAction).GetObject().(*corev1.Secret) - createdSecret.ResourceVersion = "1" - r.NoError(kubeInformerClient.Tracker().Add(createdSecret)) + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[2], kubeInformerClient, "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time - r.Len(kubeAPIClient.Actions(), 3) // no more actions - requireTLSServerIsRunning(ca) // still running + r.Equal(1, startTLSListenerFuncWasCalled) // wasn't started a third time + r.Len(kubeAPIClient.Actions(), 3) // no more actions + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) // still running }) }) @@ -794,8 +852,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: auto endpoint: 127.0.0.1 - `), // TODO what to do about ports - // TODO IP address and hostname should work + `), ) }) @@ -824,7 +881,7 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) - requireTLSServerIsRunning(ca) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) }) }) }) @@ -900,12 +957,13 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { }) }) - when("a load balancer and a secret already exist", func() { - var tlsSecret *corev1.Secret + when("a load balancer and a secret already exists", func() { + var ca []byte it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") addNodeWithRoleToTracker("worker") - tlsSecret = createActualTLSSecret(tlsSecretName) + tlsSecret := createActualTLSSecret(tlsSecretName) + ca = tlsSecret.Data["ca.crt"] r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) @@ -917,7 +975,29 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { r.NoError(controllerlib.TestSync(t, subject, *syncContext)) r.Len(kubeAPIClient.Actions(), 1) requireNodesListed(kubeAPIClient.Actions()[0]) - requireTLSServerIsRunning(tlsSecret.Data["ca.crt"]) + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + }) + }) + + when("a load balancer and a secret already exists but the tls secret is not valid", func() { + var tlsSecret *corev1.Secret + it.Before(func() { + addImpersonatorConfigMapToTracker(configMapResourceName, "mode: enabled") + addNodeWithRoleToTracker("worker") + tlsSecret = createStubTLSSecret(tlsSecretName) // secret exists but lacks certs + r.NoError(kubeAPIClient.Tracker().Add(tlsSecret)) + r.NoError(kubeInformerClient.Tracker().Add(tlsSecret)) + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeInformerClient) + addLoadBalancerServiceWithIPToTracker(generatedLoadBalancerServiceName, kubeAPIClient) + }) + + it("returns an error and leaves the impersonator running without tls certs", func() { + startInformersAndController() + r.EqualError(controllerlib.TestSync(t, subject, *syncContext), + "could not parse TLS cert PEM data from Secret: tls: failed to find any PEM data in certificate input") + r.Len(kubeAPIClient.Actions(), 1) + requireNodesListed(kubeAPIClient.Actions()[0]) + requireTLSServerIsRunningWithoutCerts() }) }) }) @@ -984,39 +1064,67 @@ func TestImpersonatorConfigControllerSync(t *testing.T) { it.Before(func() { addImpersonatorConfigMapToTracker(configMapResourceName, here.Doc(` mode: enabled - endpoint: https://proxy.example.com:8443/ + endpoint: 127.0.0.1 `)) addNodeWithRoleToTracker("worker") }) - it("doesn't start, then creates, then deletes the load balancer", func() { + it("doesn't create, then creates, then deletes the load balancer", func() { startInformersAndController() + // Should have started in "enabled" mode with an "endpoint", so no load balancer is needed. r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - - r.Len(kubeAPIClient.Actions(), 1) + r.Len(kubeAPIClient.Actions(), 2) requireNodesListed(kubeAPIClient.Actions()[0]) + ca := requireTLSSecretWasCreated(kubeAPIClient.Actions()[1]) // created immediately because "endpoint" was specified + requireTLSServerIsRunning(ca, startedTLSListener.Addr().String(), nil) + // update manually because the kubeAPIClient isn't connected to the informer in the tests + addSecretFromCreateActionToTracker(kubeAPIClient.Actions()[1], kubeInformerClient, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Secrets().Informer(), "1") + + // Switch to "enabled" mode without an "endpoint", so a load balancer is needed now. updateImpersonatorConfigMapInTracker(configMapResourceName, "mode: enabled", "1") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "1") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 2) - requireLoadBalancerWasCreated(kubeAPIClient.Actions()[1]) + r.Len(kubeAPIClient.Actions(), 4) + requireLoadBalancerWasCreated(kubeAPIClient.Actions()[2]) + requireTLSSecretDeleted(kubeAPIClient.Actions()[3]) // the Secret was deleted because it contained a cert with the wrong IP + requireTLSServerIsRunningWithoutCerts() // update manually because the kubeAPIClient isn't connected to the informer in the tests addLoadBalancerServiceToTracker(generatedLoadBalancerServiceName, kubeInformerClient) waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "0") + deleteTLSCertSecretFromTracker(tlsSecretName, kubeInformerClient) + waitForTLSCertSecretToBeDeleted(kubeInformers.Core().V1().Secrets(), tlsSecretName) + // The controller should be waiting for the load balancer's ingress to become available. + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 4) // no new actions while it is waiting for the load balancer's ingress + requireTLSServerIsRunningWithoutCerts() + + // Update the ingress of the LB in the informer's client and run Sync again. + fakeIP := "127.0.0.123" + updateLoadBalancerServiceInTracker(generatedLoadBalancerServiceName, fakeIP, "1") + waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().Services().Informer(), "1") + r.NoError(controllerlib.TestSync(t, subject, *syncContext)) + r.Len(kubeAPIClient.Actions(), 5) + ca = requireTLSSecretWasCreated(kubeAPIClient.Actions()[4]) // created because the LB ingress became available + // Check that the server is running and that TLS certs that are being served are are for fakeIP. + requireTLSServerIsRunning(ca, fakeIP, map[string]string{fakeIP + ":443": startedTLSListener.Addr().String()}) + + // Now switch back to having the "endpoint" specified, so the load balancer is not needed anymore. updateImpersonatorConfigMapInTracker(configMapResourceName, here.Doc(` mode: enabled - endpoint: https://proxy.example.com:8443/ + endpoint: 127.0.0.1 `), "2") waitForInformerCacheToSeeResourceVersion(kubeInformers.Core().V1().ConfigMaps().Informer(), "2") r.NoError(controllerlib.TestSync(t, subject, *syncContext)) - r.Len(kubeAPIClient.Actions(), 3) - requireLoadBalancerDeleted(kubeAPIClient.Actions()[2]) + r.Len(kubeAPIClient.Actions(), 7) + requireLoadBalancerDeleted(kubeAPIClient.Actions()[5]) + requireTLSSecretWasCreated(kubeAPIClient.Actions()[6]) // recreated because the endpoint was updated }) }) })