First draft of moving API server TLS cert generation to controllers
- Refactors the existing cert generation code into controllers which read and write a Secret containing the certs - Does not add any new functionality yet, e.g. no new handling for cert expiration, and no leader election to allow for multiple servers running simultaneously - This commit also doesn't add new tests for the cert generation code, but it should be more unit testable now as controllers
This commit is contained in:
Ryan Richard
parent
b00cec954e
commit
86c3f89b2e
|
|
@ -1,3 +1,6 @@
|
|||
# Copyright 2020 VMware, Inc.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
FROM golang:1.14-alpine as build-env
|
||||
|
||||
# It is important that these ARG's are defined after the FROM statement
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
# placeholder-name
|
||||
# placeholder-name
|
||||
|
||||
Copyright 2020 VMware, Inc.
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#@ load("@ytt:data", "data")
|
||||
|
||||
#! Give permission to various cluster-scoped objects
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
|
|
@ -28,6 +29,8 @@ roleRef:
|
|||
kind: ClusterRole
|
||||
name: #@ data.values.app_name + "-aggregated-api-server-cluster-role"
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
#! Give permission to various objects within the app's own namespace
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: Role
|
||||
|
|
@ -38,6 +41,9 @@ rules:
|
|||
- apiGroups: [""]
|
||||
resources: [services]
|
||||
verbs: [create, get, list, patch, update, watch]
|
||||
- apiGroups: [""]
|
||||
resources: [secrets]
|
||||
verbs: [create, get, list, patch, update, watch]
|
||||
- apiGroups: [crds.placeholder.suzerain-io.github.io]
|
||||
resources: [logindiscoveryconfigs]
|
||||
verbs: [create, get, list, update, watch]
|
||||
|
|
@ -55,6 +61,8 @@ roleRef:
|
|||
kind: Role
|
||||
name: #@ data.values.app_name + "-aggregated-api-server-role"
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
#! Allow both authenticated and unauthenticated LoginRequests (i.e. allow all requests)
|
||||
---
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
|
|
@ -70,7 +78,6 @@ apiVersion: rbac.authorization.k8s.io/v1
|
|||
metadata:
|
||||
name: #@ data.values.app_name + "-loginrequests-cluster-role-binding"
|
||||
subjects:
|
||||
#! both authenticated and unauthenticated requests (i.e. all requests) should be allowed
|
||||
- kind: Group
|
||||
name: system:authenticated
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
|
@ -81,6 +88,8 @@ roleRef:
|
|||
kind: ClusterRole
|
||||
name: #@ data.values.app_name + "-loginrequests-cluster-role"
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
#! Give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers
|
||||
---
|
||||
kind: ClusterRoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
|
@ -93,9 +102,10 @@ subjects:
|
|||
namespace: #@ data.values.namespace
|
||||
roleRef:
|
||||
kind: ClusterRole
|
||||
#! give permissions for subjectaccessreviews, tokenreview that is needed by aggregated api servers
|
||||
name: system:auth-delegator
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
#! Give permissions for a special configmap of CA bundles that is needed by aggregated api servers
|
||||
---
|
||||
kind: RoleBinding
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
|
@ -108,9 +118,10 @@ subjects:
|
|||
namespace: #@ data.values.namespace
|
||||
roleRef:
|
||||
kind: Role
|
||||
#! give permissions for a special configmap of CA bundles that is needed by aggregated api servers
|
||||
name: extension-apiserver-authentication-reader
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
|
||||
#! Give permission to list and watch ConfigMaps in kube-public
|
||||
---
|
||||
kind: Role
|
||||
apiVersion: rbac.authorization.k8s.io/v1
|
||||
|
|
|
|||
|
|
@ -12,14 +12,14 @@ import (
|
|||
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/client-go/util/retry"
|
||||
aggregatationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
|
||||
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
|
||||
|
||||
placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
|
||||
)
|
||||
|
||||
// UpdateAPIService updates the APIService's CA bundle.
|
||||
func UpdateAPIService(ctx context.Context, aggregationV1 aggregatationv1client.Interface, aggregatedAPIServerCA []byte) error {
|
||||
apiServices := aggregationV1.ApiregistrationV1().APIServices()
|
||||
func UpdateAPIService(ctx context.Context, aggregatorClient aggregatorclient.Interface, aggregatedAPIServerCA []byte) error {
|
||||
apiServices := aggregatorClient.ApiregistrationV1().APIServices()
|
||||
apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
|
||||
|
||||
if err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,164 @@
|
|||
/*
|
||||
Copyright 2020 VMware, Inc.
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package apicerts
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
corev1informers "k8s.io/client-go/informers/core/v1"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
"k8s.io/klog/v2"
|
||||
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
|
||||
|
||||
"github.com/suzerain-io/controller-go"
|
||||
"github.com/suzerain-io/placeholder-name/internal/autoregistration"
|
||||
"github.com/suzerain-io/placeholder-name/internal/certauthority"
|
||||
placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
|
||||
placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
|
||||
)
|
||||
|
||||
const (
|
||||
//nolint: gosec
|
||||
certsSecretName = "api-serving-cert"
|
||||
caCertificateSecretKey = "caCertificate"
|
||||
tlsPrivateKeySecretKey = "tlsPrivateKey"
|
||||
tlsCertificateChainSecretKey = "tlsCertificateChain"
|
||||
)
|
||||
|
||||
type certsManagerController struct {
|
||||
namespace string
|
||||
apiServiceName string
|
||||
k8sClient kubernetes.Interface
|
||||
aggregatorClient *aggregatorclient.Clientset
|
||||
secretInformer corev1informers.SecretInformer
|
||||
}
|
||||
|
||||
func NewCertsManagerController(
|
||||
namespace string,
|
||||
k8sClient kubernetes.Interface,
|
||||
aggregationClient *aggregatorclient.Clientset,
|
||||
secretInformer corev1informers.SecretInformer,
|
||||
withInformer placeholdernamecontroller.WithInformerOptionFunc,
|
||||
) controller.Controller {
|
||||
apiServiceName := placeholderv1alpha1.SchemeGroupVersion.Version + "." + placeholderv1alpha1.GroupName
|
||||
return controller.New(
|
||||
controller.Config{
|
||||
Name: "certs-manager-controller",
|
||||
Syncer: &certsManagerController{
|
||||
apiServiceName: apiServiceName,
|
||||
namespace: namespace,
|
||||
k8sClient: k8sClient,
|
||||
aggregatorClient: aggregationClient,
|
||||
secretInformer: secretInformer,
|
||||
},
|
||||
},
|
||||
withInformer(
|
||||
secretInformer,
|
||||
placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
|
||||
controller.InformerOption{},
|
||||
),
|
||||
// Be sure to run once even if the Secret that the informer is watching doesn't exist.
|
||||
controller.WithInitialEvent(controller.Key{
|
||||
Namespace: namespace,
|
||||
Name: certsSecretName,
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
func (c *certsManagerController) Sync(ctx controller.Context) error {
|
||||
// Try to get the secret from the informer cache.
|
||||
_, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
|
||||
notFound := k8serrors.IsNotFound(err)
|
||||
if err != nil && !notFound {
|
||||
return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
|
||||
}
|
||||
if !notFound {
|
||||
// The secret already exists, so nothing to do.
|
||||
return nil
|
||||
}
|
||||
|
||||
// Create a CA.
|
||||
aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not initialize CA: %w", err)
|
||||
}
|
||||
|
||||
// This string must match the name of the Service declared in the deployment yaml.
|
||||
const serviceName = "placeholder-name-api"
|
||||
|
||||
// Using the CA from above, create a TLS server cert for the aggregated API server to use.
|
||||
aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
|
||||
pkix.Name{CommonName: serviceName + "." + c.namespace + ".svc"},
|
||||
[]string{},
|
||||
24*365*time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not issue serving certificate: %w", err)
|
||||
}
|
||||
|
||||
// Write the CA's public key bundle and the serving certs to a secret.
|
||||
tlsPrivateKeyPEM, tlsCertChainPEM, err := pemEncode(aggregatedAPIServerTLSCert)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not PEM encode serving certificate: %w", err)
|
||||
}
|
||||
secret := corev1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: certsSecretName,
|
||||
Namespace: c.namespace,
|
||||
},
|
||||
StringData: map[string]string{
|
||||
caCertificateSecretKey: string(aggregatedAPIServerCA.Bundle()),
|
||||
tlsPrivateKeySecretKey: string(tlsPrivateKeyPEM),
|
||||
tlsCertificateChainSecretKey: string(tlsCertChainPEM),
|
||||
},
|
||||
}
|
||||
_, err = c.k8sClient.CoreV1().Secrets(c.namespace).Create(ctx.Context, &secret, metav1.CreateOptions{})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create secret: %w", err)
|
||||
}
|
||||
|
||||
// Update the APIService to give it the new CA bundle.
|
||||
if err := autoregistration.UpdateAPIService(ctx.Context, c.aggregatorClient, aggregatedAPIServerCA.Bundle()); err != nil {
|
||||
return fmt.Errorf("could not update the API service: %w", err)
|
||||
}
|
||||
|
||||
klog.Info("certsManagerController Sync successfully created secret and updated API service")
|
||||
return nil
|
||||
}
|
||||
|
||||
// Encode a tls.Certificate into a private key PEM and a cert chain PEM.
|
||||
func pemEncode(cert *tls.Certificate) ([]byte, []byte, error) {
|
||||
privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("error marshalling private key: %w", err)
|
||||
}
|
||||
privateKeyPEM := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "PRIVATE KEY",
|
||||
Headers: nil,
|
||||
Bytes: privateKeyDER,
|
||||
})
|
||||
|
||||
certChainPEM := make([]byte, 0)
|
||||
for _, certFromChain := range cert.Certificate {
|
||||
certPEMBytes := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Headers: nil,
|
||||
Bytes: certFromChain,
|
||||
})
|
||||
certChainPEM = append(certChainPEM, certPEMBytes...)
|
||||
}
|
||||
|
||||
return privateKeyPEM, certChainPEM, nil
|
||||
}
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
/*
|
||||
Copyright 2020 VMware, Inc.
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package apicerts
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
k8serrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
corev1informers "k8s.io/client-go/informers/core/v1"
|
||||
"k8s.io/klog/v2"
|
||||
|
||||
"github.com/suzerain-io/controller-go"
|
||||
placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
|
||||
"github.com/suzerain-io/placeholder-name/internal/provider"
|
||||
)
|
||||
|
||||
type certsObserverController struct {
|
||||
namespace string
|
||||
dynamicCertProvider *provider.DynamicTLSServingCertProvider
|
||||
secretInformer corev1informers.SecretInformer
|
||||
}
|
||||
|
||||
func NewCertsObserverController(
|
||||
namespace string,
|
||||
dynamicCertProvider *provider.DynamicTLSServingCertProvider,
|
||||
secretInformer corev1informers.SecretInformer,
|
||||
withInformer placeholdernamecontroller.WithInformerOptionFunc,
|
||||
) controller.Controller {
|
||||
return controller.New(
|
||||
controller.Config{
|
||||
Name: "certs-observer-controller",
|
||||
Syncer: &certsObserverController{
|
||||
namespace: namespace,
|
||||
dynamicCertProvider: dynamicCertProvider,
|
||||
secretInformer: secretInformer,
|
||||
},
|
||||
},
|
||||
withInformer(
|
||||
secretInformer,
|
||||
placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(certsSecretName, namespace),
|
||||
controller.InformerOption{},
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
func (c *certsObserverController) Sync(_ controller.Context) error {
|
||||
// Try to get the secret from the informer cache.
|
||||
certSecret, err := c.secretInformer.Lister().Secrets(c.namespace).Get(certsSecretName)
|
||||
notFound := k8serrors.IsNotFound(err)
|
||||
if err != nil && !notFound {
|
||||
return fmt.Errorf("failed to get %s/%s secret: %w", c.namespace, certsSecretName, err)
|
||||
}
|
||||
if notFound {
|
||||
klog.Info("certsObserverController Sync() found that the secret does not exist yet or was deleted")
|
||||
// The secret does not exist yet or was deleted.
|
||||
c.dynamicCertProvider.CertPEM = nil
|
||||
c.dynamicCertProvider.KeyPEM = nil
|
||||
return nil
|
||||
}
|
||||
|
||||
// Mutate the in-memory cert provider to update with the latest cert values.
|
||||
c.dynamicCertProvider.CertPEM = certSecret.Data[tlsCertificateChainSecretKey]
|
||||
c.dynamicCertProvider.KeyPEM = certSecret.Data[tlsPrivateKeySecretKey]
|
||||
klog.Info("certsObserverController Sync updated certs in the dynamic cert provider")
|
||||
return nil
|
||||
}
|
||||
|
|
@ -17,6 +17,7 @@ import (
|
|||
"k8s.io/klog/v2"
|
||||
|
||||
"github.com/suzerain-io/controller-go"
|
||||
placeholdernamecontroller "github.com/suzerain-io/placeholder-name/internal/controller"
|
||||
crdsplaceholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/crdsplaceholder/v1alpha1"
|
||||
placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
|
||||
crdsplaceholderv1alpha1informers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions/crdsplaceholder/v1alpha1"
|
||||
|
|
@ -31,25 +32,6 @@ const (
|
|||
configName = "placeholder-name-config"
|
||||
)
|
||||
|
||||
func nameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
|
||||
objMatchesFunc := func(obj metav1.Object) bool {
|
||||
return obj.GetName() == name && obj.GetNamespace() == namespace
|
||||
}
|
||||
return controller.FilterFuncs{
|
||||
AddFunc: objMatchesFunc,
|
||||
UpdateFunc: func(oldObj, newObj metav1.Object) bool {
|
||||
return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
|
||||
},
|
||||
DeleteFunc: objMatchesFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// Same signature as controller.WithInformer().
|
||||
type withInformerOptionFunc func(
|
||||
getter controller.InformerGetter,
|
||||
filter controller.Filter,
|
||||
opt controller.InformerOption) controller.Option
|
||||
|
||||
type publisherController struct {
|
||||
namespace string
|
||||
serverOverride *string
|
||||
|
|
@ -64,7 +46,7 @@ func NewPublisherController(
|
|||
placeholderClient placeholderclientset.Interface,
|
||||
configMapInformer corev1informers.ConfigMapInformer,
|
||||
loginDiscoveryConfigInformer crdsplaceholderv1alpha1informers.LoginDiscoveryConfigInformer,
|
||||
withInformer withInformerOptionFunc,
|
||||
withInformer placeholdernamecontroller.WithInformerOptionFunc,
|
||||
) controller.Controller {
|
||||
return controller.New(
|
||||
controller.Config{
|
||||
|
|
@ -79,12 +61,12 @@ func NewPublisherController(
|
|||
},
|
||||
withInformer(
|
||||
configMapInformer,
|
||||
nameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
|
||||
placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(clusterInfoName, ClusterInfoNamespace),
|
||||
controller.InformerOption{},
|
||||
),
|
||||
withInformer(
|
||||
loginDiscoveryConfigInformer,
|
||||
nameAndNamespaceExactMatchFilterFactory(configName, namespace),
|
||||
placeholdernamecontroller.NameAndNamespaceExactMatchFilterFactory(configName, namespace),
|
||||
controller.InformerOption{},
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,31 @@
|
|||
/*
|
||||
Copyright 2020 VMware, Inc.
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package controller
|
||||
|
||||
import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"github.com/suzerain-io/controller-go"
|
||||
)
|
||||
|
||||
func NameAndNamespaceExactMatchFilterFactory(name, namespace string) controller.FilterFuncs {
|
||||
objMatchesFunc := func(obj metav1.Object) bool {
|
||||
return obj.GetName() == name && obj.GetNamespace() == namespace
|
||||
}
|
||||
return controller.FilterFuncs{
|
||||
AddFunc: objMatchesFunc,
|
||||
UpdateFunc: func(oldObj, newObj metav1.Object) bool {
|
||||
return objMatchesFunc(oldObj) || objMatchesFunc(newObj)
|
||||
},
|
||||
DeleteFunc: objMatchesFunc,
|
||||
}
|
||||
}
|
||||
|
||||
// Same signature as controller.WithInformer().
|
||||
type WithInformerOptionFunc func(
|
||||
getter controller.InformerGetter,
|
||||
filter controller.Filter,
|
||||
opt controller.InformerOption) controller.Option
|
||||
|
|
@ -3,7 +3,7 @@ Copyright 2020 VMware, Inc.
|
|||
SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package controller
|
||||
package controllermanager
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
|
@ -14,11 +14,12 @@ import (
|
|||
k8sinformers "k8s.io/client-go/informers"
|
||||
"k8s.io/client-go/kubernetes"
|
||||
restclient "k8s.io/client-go/rest"
|
||||
aggregationv1client "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
|
||||
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
|
||||
|
||||
"github.com/suzerain-io/controller-go"
|
||||
"github.com/suzerain-io/placeholder-name/internal/autoregistration"
|
||||
"github.com/suzerain-io/placeholder-name/internal/controller/apicerts"
|
||||
"github.com/suzerain-io/placeholder-name/internal/controller/logindiscovery"
|
||||
"github.com/suzerain-io/placeholder-name/internal/provider"
|
||||
placeholderclientset "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/clientset/versioned"
|
||||
placeholderinformers "github.com/suzerain-io/placeholder-name/kubernetes/1.19/client-go/informers/externalversions"
|
||||
)
|
||||
|
|
@ -30,25 +31,19 @@ const (
|
|||
|
||||
// Prepare the controllers and their informers and return a function that will start them when called.
|
||||
func PrepareControllers(
|
||||
ctx context.Context,
|
||||
caBundle []byte,
|
||||
serverInstallationNamespace string,
|
||||
discoveryURLOverride *string,
|
||||
dynamicCertProvider *provider.DynamicTLSServingCertProvider,
|
||||
) (func(ctx context.Context), error) {
|
||||
// Create k8s clients.
|
||||
k8sClient, aggregationClient, placeholderClient, err := createClients()
|
||||
k8sClient, aggregatorClient, placeholderClient, err := createClients()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create clients for the controllers: %w", err)
|
||||
}
|
||||
|
||||
// TODO Putting this here temporarily on the way toward moving it elsewhere.
|
||||
// When it moves elsewhere then PrepareControllers() will not need to take ctx and caBundle parameters.
|
||||
if err := autoregistration.UpdateAPIService(ctx, aggregationClient, caBundle); err != nil {
|
||||
return nil, fmt.Errorf("could not update the API service: %w", err)
|
||||
}
|
||||
|
||||
// Create informers.
|
||||
k8sInformers, placeholderInformers := createInformers(serverInstallationNamespace, k8sClient, placeholderClient)
|
||||
// Create informers. Don't forget to make sure they get started in the function returned below.
|
||||
kubePublicNamespaceK8sInformers, installationNamespaceK8sInformers, installationNamespacePlaceholderInformers :=
|
||||
createInformers(serverInstallationNamespace, k8sClient, placeholderClient)
|
||||
|
||||
// Create controller manager.
|
||||
controllerManager := controller.
|
||||
|
|
@ -58,8 +53,27 @@ func PrepareControllers(
|
|||
serverInstallationNamespace,
|
||||
discoveryURLOverride,
|
||||
placeholderClient,
|
||||
k8sInformers.Core().V1().ConfigMaps(),
|
||||
placeholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
|
||||
kubePublicNamespaceK8sInformers.Core().V1().ConfigMaps(),
|
||||
installationNamespacePlaceholderInformers.Crds().V1alpha1().LoginDiscoveryConfigs(),
|
||||
controller.WithInformer,
|
||||
),
|
||||
singletonWorker,
|
||||
).
|
||||
WithController(
|
||||
apicerts.NewCertsManagerController(
|
||||
serverInstallationNamespace,
|
||||
k8sClient,
|
||||
aggregatorClient,
|
||||
installationNamespaceK8sInformers.Core().V1().Secrets(),
|
||||
controller.WithInformer,
|
||||
),
|
||||
singletonWorker,
|
||||
).
|
||||
WithController(
|
||||
apicerts.NewCertsObserverController(
|
||||
serverInstallationNamespace,
|
||||
dynamicCertProvider,
|
||||
installationNamespaceK8sInformers.Core().V1().Secrets(),
|
||||
controller.WithInformer,
|
||||
),
|
||||
singletonWorker,
|
||||
|
|
@ -67,14 +81,21 @@ func PrepareControllers(
|
|||
|
||||
// Return a function which starts the informers and controllers.
|
||||
return func(ctx context.Context) {
|
||||
k8sInformers.Start(ctx.Done())
|
||||
placeholderInformers.Start(ctx.Done())
|
||||
kubePublicNamespaceK8sInformers.Start(ctx.Done())
|
||||
installationNamespaceK8sInformers.Start(ctx.Done())
|
||||
installationNamespacePlaceholderInformers.Start(ctx.Done())
|
||||
|
||||
go controllerManager.Start(ctx)
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Create the k8s clients that will be used by the controllers.
|
||||
func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *placeholderclientset.Clientset, error) {
|
||||
func createClients() (
|
||||
k8sClient *kubernetes.Clientset,
|
||||
aggregatorClient *aggregatorclient.Clientset,
|
||||
placeholderClient *placeholderclientset.Clientset,
|
||||
err error,
|
||||
) {
|
||||
// Load the Kubernetes client configuration (kubeconfig),
|
||||
kubeConfig, err := restclient.InClusterConfig()
|
||||
if err != nil {
|
||||
|
|
@ -85,13 +106,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl
|
|||
protoKubeConfig := createProtoKubeConfig(kubeConfig)
|
||||
|
||||
// Connect to the core Kubernetes API.
|
||||
k8sClient, err := kubernetes.NewForConfig(protoKubeConfig)
|
||||
k8sClient, err = kubernetes.NewForConfig(protoKubeConfig)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
|
||||
}
|
||||
|
||||
// Connect to the Kubernetes aggregation API.
|
||||
aggregationClient, err := aggregationv1client.NewForConfig(protoKubeConfig)
|
||||
aggregatorClient, err = aggregatorclient.NewForConfig(protoKubeConfig)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("could not initialize Kubernetes client: %w", err)
|
||||
}
|
||||
|
|
@ -99,12 +120,13 @@ func createClients() (*kubernetes.Clientset, *aggregationv1client.Clientset, *pl
|
|||
// Connect to the placeholder API.
|
||||
// I think we can't use protobuf encoding here because we are using CRDs
|
||||
// (for which protobuf encoding is not supported).
|
||||
placeholderClient, err := placeholderclientset.NewForConfig(kubeConfig)
|
||||
placeholderClient, err = placeholderclientset.NewForConfig(kubeConfig)
|
||||
if err != nil {
|
||||
return nil, nil, nil, fmt.Errorf("could not initialize placeholder client: %w", err)
|
||||
}
|
||||
|
||||
return k8sClient, aggregationClient, placeholderClient, nil
|
||||
//nolint: nakedret
|
||||
return
|
||||
}
|
||||
|
||||
// Create the informers that will be used by the controllers.
|
||||
|
|
@ -112,20 +134,27 @@ func createInformers(
|
|||
serverInstallationNamespace string,
|
||||
k8sClient *kubernetes.Clientset,
|
||||
placeholderClient *placeholderclientset.Clientset,
|
||||
) (k8sinformers.SharedInformerFactory, placeholderinformers.SharedInformerFactory) {
|
||||
k8sInformers := k8sinformers.NewSharedInformerFactoryWithOptions(
|
||||
) (
|
||||
kubePublicNamespaceK8sInformers k8sinformers.SharedInformerFactory,
|
||||
installationNamespaceK8sInformers k8sinformers.SharedInformerFactory,
|
||||
installationNamespacePlaceholderInformers placeholderinformers.SharedInformerFactory,
|
||||
) {
|
||||
kubePublicNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions(
|
||||
k8sClient,
|
||||
defaultResyncInterval,
|
||||
k8sinformers.WithNamespace(
|
||||
logindiscovery.ClusterInfoNamespace,
|
||||
),
|
||||
k8sinformers.WithNamespace(logindiscovery.ClusterInfoNamespace),
|
||||
)
|
||||
placeholderInformers := placeholderinformers.NewSharedInformerFactoryWithOptions(
|
||||
installationNamespaceK8sInformers = k8sinformers.NewSharedInformerFactoryWithOptions(
|
||||
k8sClient,
|
||||
defaultResyncInterval,
|
||||
k8sinformers.WithNamespace(serverInstallationNamespace),
|
||||
)
|
||||
installationNamespacePlaceholderInformers = placeholderinformers.NewSharedInformerFactoryWithOptions(
|
||||
placeholderClient,
|
||||
defaultResyncInterval,
|
||||
placeholderinformers.WithNamespace(serverInstallationNamespace),
|
||||
)
|
||||
return k8sInformers, placeholderInformers
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a copy of the input config with the ContentConfig set to use protobuf.
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
/*
|
||||
Copyright 2020 VMware, Inc.
|
||||
SPDX-License-Identifier: Apache-2.0
|
||||
*/
|
||||
|
||||
package provider
|
||||
|
||||
type DynamicTLSServingCertProvider struct {
|
||||
CertPEM []byte
|
||||
KeyPEM []byte
|
||||
}
|
||||
|
||||
func (*DynamicTLSServingCertProvider) Name() string {
|
||||
return "DynamicTLSServingCertProvider"
|
||||
}
|
||||
|
||||
func (p *DynamicTLSServingCertProvider) CurrentCertKeyContent() (cert []byte, key []byte) {
|
||||
return p.CertPEM, p.KeyPEM
|
||||
}
|
||||
|
|
@ -8,24 +8,19 @@ package server
|
|||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
"k8s.io/apiserver/pkg/server/dynamiccertificates"
|
||||
genericoptions "k8s.io/apiserver/pkg/server/options"
|
||||
"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
|
||||
|
||||
"github.com/suzerain-io/placeholder-name/internal/apiserver"
|
||||
"github.com/suzerain-io/placeholder-name/internal/certauthority"
|
||||
"github.com/suzerain-io/placeholder-name/internal/controller"
|
||||
"github.com/suzerain-io/placeholder-name/internal/controllermanager"
|
||||
"github.com/suzerain-io/placeholder-name/internal/downward"
|
||||
"github.com/suzerain-io/placeholder-name/internal/provider"
|
||||
placeholderv1alpha1 "github.com/suzerain-io/placeholder-name/kubernetes/1.19/api/apis/placeholder/v1alpha1"
|
||||
"github.com/suzerain-io/placeholder-name/pkg/config"
|
||||
)
|
||||
|
|
@ -135,31 +130,19 @@ func (app *App) runServer(ctx context.Context) error {
|
|||
}
|
||||
serverInstallationNamespace := podInfo.Namespace
|
||||
|
||||
// Create a CA.
|
||||
aggregatedAPIServerCA, err := certauthority.New(pkix.Name{CommonName: "Placeholder CA"})
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not initialize CA: %w", err)
|
||||
}
|
||||
|
||||
// This string must match the name of the Service declared in the deployment yaml.
|
||||
const serviceName = "placeholder-name-api"
|
||||
// Using the CA from above, create a TLS server cert for the aggregated API server to use.
|
||||
aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
|
||||
pkix.Name{CommonName: serviceName + "." + serverInstallationNamespace + ".svc"},
|
||||
[]string{},
|
||||
24*365*time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not issue serving certificate: %w", err)
|
||||
}
|
||||
// This cert provider will provide certs to the API server and will
|
||||
// be mutated by a controller to keep the certs up to date with what
|
||||
// is stored in a k8s Secret. Therefore it also effectively acting as
|
||||
// an in-memory cache of what is stored in the k8s Secret, helping to
|
||||
// keep incoming requests fast.
|
||||
dynamicCertProvider := &provider.DynamicTLSServingCertProvider{}
|
||||
|
||||
// Prepare to start the controllers, but defer actually starting them until the
|
||||
// post start hook of the aggregated API server.
|
||||
startControllersFunc, err := controller.PrepareControllers(
|
||||
ctx,
|
||||
aggregatedAPIServerCA.Bundle(),
|
||||
startControllersFunc, err := controllermanager.PrepareControllers(
|
||||
serverInstallationNamespace,
|
||||
cfg.DiscoveryConfig.URL,
|
||||
dynamicCertProvider,
|
||||
)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not prepare controllers: %w", err)
|
||||
|
|
@ -167,7 +150,7 @@ func (app *App) runServer(ctx context.Context) error {
|
|||
|
||||
// Get the aggregated API server config.
|
||||
aggregatedAPIServerConfig, err := getAggregatedAPIServerConfig(
|
||||
aggregatedAPIServerTLSCert,
|
||||
dynamicCertProvider,
|
||||
webhookTokenAuthenticator,
|
||||
k8sClusterCA,
|
||||
startControllersFunc,
|
||||
|
|
@ -188,25 +171,29 @@ func (app *App) runServer(ctx context.Context) error {
|
|||
|
||||
// Create a configuration for the aggregated API server.
|
||||
func getAggregatedAPIServerConfig(
|
||||
cert *tls.Certificate,
|
||||
dynamicCertProvider *provider.DynamicTLSServingCertProvider,
|
||||
webhookTokenAuthenticator *webhook.WebhookTokenAuthenticator,
|
||||
ca *certauthority.CA,
|
||||
startControllersPostStartHook func(context.Context),
|
||||
) (*apiserver.Config, error) {
|
||||
provider, err := createStaticCertKeyProvider(cert)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("could not create static cert key provider: %w", err)
|
||||
}
|
||||
|
||||
recommendedOptions := genericoptions.NewRecommendedOptions(
|
||||
defaultEtcdPathPrefix,
|
||||
apiserver.Codecs.LegacyCodec(placeholderv1alpha1.SchemeGroupVersion),
|
||||
// TODO we should check to see if all the other default settings are acceptable for us
|
||||
)
|
||||
recommendedOptions.Etcd = nil // turn off etcd storage because we don't need it yet
|
||||
recommendedOptions.SecureServing.ServerCert.GeneratedCert = provider
|
||||
recommendedOptions.SecureServing.ServerCert.GeneratedCert = dynamicCertProvider
|
||||
|
||||
serverConfig := genericapiserver.NewRecommendedConfig(apiserver.Codecs)
|
||||
// Note that among other things, this ApplyTo() function copies
|
||||
// `recommendedOptions.SecureServing.ServerCert.GeneratedCert` into
|
||||
// `serverConfig.SecureServing.Cert` thus making `dynamicCertProvider`
|
||||
// the cert provider for the running server. The provider will be called
|
||||
// by the API machinery periodically. When the provider returns nil certs,
|
||||
// the API server will return "the server is currently unable to
|
||||
// handle the request" error responses for all incoming requests.
|
||||
// If the provider later starts returning certs, then the API server
|
||||
// will use them to handle the incoming requests successfully.
|
||||
if err := recommendedOptions.ApplyTo(serverConfig); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -221,27 +208,3 @@ func getAggregatedAPIServerConfig(
|
|||
}
|
||||
return apiServerConfig, nil
|
||||
}
|
||||
|
||||
func createStaticCertKeyProvider(cert *tls.Certificate) (dynamiccertificates.CertKeyContentProvider, error) {
|
||||
privateKeyDER, err := x509.MarshalPKCS8PrivateKey(cert.PrivateKey)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error marshalling private key: %w", err)
|
||||
}
|
||||
privateKeyPEM := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "PRIVATE KEY",
|
||||
Headers: nil,
|
||||
Bytes: privateKeyDER,
|
||||
})
|
||||
|
||||
certChainPEM := make([]byte, 0)
|
||||
for _, certFromChain := range cert.Certificate {
|
||||
certPEMBytes := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Headers: nil,
|
||||
Bytes: certFromChain,
|
||||
})
|
||||
certChainPEM = append(certChainPEM, certPEMBytes...)
|
||||
}
|
||||
|
||||
return dynamiccertificates.NewStaticCertKeyContent("some-name???", certChainPEM, privateKeyPEM)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue