diff --git a/internal/controller/impersonatorconfig/impersonator_config.go b/internal/controller/impersonatorconfig/impersonator_config.go
index ae99de98..c94aae15 100644
--- a/internal/controller/impersonatorconfig/impersonator_config.go
+++ b/internal/controller/impersonatorconfig/impersonator_config.go
@@ -781,6 +781,12 @@ func (c *impersonatorConfigController) ensureTLSSecretIsRemoved(ctx context.Cont
 		"secret", c.tlsSecretName,
 		"namespace", c.namespace)
 	err = c.k8sClient.CoreV1().Secrets(c.namespace).Delete(ctx, c.tlsSecretName, metav1.DeleteOptions{})
+	notFound := k8serrors.IsNotFound(err)
+	if notFound {
+		// its okay if we tried to delete and we got a not found error. This probably means
+		// another instance of the concierge got here first so there's nothing to delete.
+		return nil
+	}
 	if err != nil {
 		return err
 	}
diff --git a/internal/controller/impersonatorconfig/impersonator_config_test.go b/internal/controller/impersonatorconfig/impersonator_config_test.go
index 8b9b107b..c7f7767a 100644
--- a/internal/controller/impersonatorconfig/impersonator_config_test.go
+++ b/internal/controller/impersonatorconfig/impersonator_config_test.go
@@ -2203,6 +2203,25 @@ func TestImpersonatorConfigControllerSync(t *testing.T) {
 			})
 		})
 
+		when("deleting the tls secret when informer and api are out of sync", func() {
+			it.Before(func() {
+				addNodeWithRoleToTracker("control-plane", kubeAPIClient)
+				addSecretToTrackers(newEmptySecret(tlsSecretName), kubeInformerClient)
+				addImpersonatorConfigMapToTracker(configMapResourceName, "{mode: disabled}", kubeInformerClient)
+			})
+
+			it("does not pass the not found error through", func() {
+				startInformersAndController()
+				r.NoError(runControllerSync())
+				requireTLSServerWasNeverStarted()
+				r.Len(kubeAPIClient.Actions(), 2)
+				requireNodesListed(kubeAPIClient.Actions()[0])
+				requireTLSSecretWasDeleted(kubeAPIClient.Actions()[1])
+				requireCredentialIssuer(newManuallyDisabledStrategy())
+				requireSigningCertProviderIsEmpty()
+			})
+		})
+
 		when("the PEM formatted data in the TLS Secret is not a valid cert", func() {
 			it.Before(func() {
 				addSecretToTrackers(signingCASecret, kubeInformerClient)