Use duration and renewBefore to control API cert rotation

These configuration knobs are much more human-understandable than the
previous percentage-based threshold flag.

We now allow users to set the lifetime of the serving cert via a ConfigMap.
Previously this was hardcoded to 1 year.

Signed-off-by: Andrew Keesler <akeesler@vmware.com>
This commit is contained in:
Andrew Keesler 2020-08-20 15:17:18 -04:00
parent 3929fa672e
commit 39c299a32d
No known key found for this signature in database
GPG Key ID: 27CE0444346F9413
14 changed files with 190 additions and 136 deletions

View File

@ -28,24 +28,20 @@ type certsExpirerController struct {
k8sClient kubernetes.Interface
secretInformer corev1informers.SecretInformer
// ageThreshold is a percentage (i.e., a real number between 0 and 1,
// inclusive) indicating the point in a certificate's lifetime where this
// controller will start to try to rotate it.
//
// Said another way, once ageThreshold % of a certificate's lifetime has
// passed, this controller will try to delete it to force a new certificate
// to be created.
ageThreshold float32
// renewBefore is the amount of time after the cert's issuance where
// this controller will start to try to rotate it.
renewBefore time.Duration
}
// NewCertsExpirerController returns a controller.Controller that will delete a
// CA once it gets within some threshold of its expiration time.
// certificate secret once it gets within some threshold of its expiration time. The
// deletion forces rotation of the secret with the help of other controllers.
func NewCertsExpirerController(
namespace string,
k8sClient kubernetes.Interface,
secretInformer corev1informers.SecretInformer,
withInformer pinnipedcontroller.WithInformerOptionFunc,
ageThreshold float32,
renewBefore time.Duration,
) controller.Controller {
return controller.New(
controller.Config{
@ -54,7 +50,7 @@ func NewCertsExpirerController(
namespace: namespace,
k8sClient: k8sClient,
secretInformer: secretInformer,
ageThreshold: ageThreshold,
renewBefore: renewBefore,
},
},
withInformer(
@ -77,20 +73,19 @@ func (c *certsExpirerController) Sync(ctx controller.Context) error {
return nil
}
notBefore, notAfter, err := getCABounds(secret)
notBefore, notAfter, err := getCertBounds(secret)
if err != nil {
// If we can't get the CA, then really all we can do is log something, since
// if we returned an error then the controller lib would just call us again
// and again, which would probably yield the same results.
// If we can't read the cert, then really all we can do is log something,
// since if we returned an error then the controller lib would just call us
// again and again, which would probably yield the same results.
klog.Warningf("certsExpirerController Sync() found that the secret is malformed: %s", err.Error())
return nil
}
caLifetime := notAfter.Sub(notBefore)
caAge := time.Since(notBefore)
thresholdDelta := (float32(caAge) / float32(caLifetime)) - c.ageThreshold
klog.Infof("certsExpirerController Sync() found a CA age threshold delta of %.9g", thresholdDelta)
if thresholdDelta > 0 {
certAge := time.Since(notBefore)
renewDelta := certAge - c.renewBefore
klog.Infof("certsExpirerController Sync() found a renew delta of %s", renewDelta)
if renewDelta >= 0 || time.Now().After(notAfter) {
err := c.k8sClient.
CoreV1().
Secrets(c.namespace).
@ -105,24 +100,25 @@ func (c *certsExpirerController) Sync(ctx controller.Context) error {
return nil
}
// getCABounds returns the NotBefore and NotAfter fields of the CA certificate
// in the provided secret, or an error. Not that it expects the provided secret
// to contain the well-known data keys from this package (see certs_manager.go).
func getCABounds(secret *corev1.Secret) (time.Time, time.Time, error) {
caPEM := secret.Data[caCertificateSecretKey]
if caPEM == nil {
return time.Time{}, time.Time{}, constable.Error("failed to find CA")
// getCertBounds returns the NotBefore and NotAfter fields of the TLS
// certificate in the provided secret, or an error. Not that it expects the
// provided secret to contain the well-known data keys from this package (see
// certs_manager.go).
func getCertBounds(secret *corev1.Secret) (time.Time, time.Time, error) {
certPEM := secret.Data[tlsCertificateChainSecretKey]
if certPEM == nil {
return time.Time{}, time.Time{}, constable.Error("failed to find certificate")
}
caBlock, _ := pem.Decode(caPEM)
if caBlock == nil {
return time.Time{}, time.Time{}, constable.Error("failed to decode CA PEM")
certBlock, _ := pem.Decode(certPEM)
if certBlock == nil {
return time.Time{}, time.Time{}, constable.Error("failed to decode certificate PEM")
}
caCrt, err := x509.ParseCertificate(caBlock.Bytes)
cert, err := x509.ParseCertificate(certBlock.Bytes)
if err != nil {
return time.Time{}, time.Time{}, fmt.Errorf("failed to parse CA: %w", err)
return time.Time{}, time.Time{}, fmt.Errorf("failed to parse certificate: %w", err)
}
return caCrt.NotBefore, caCrt.NotAfter, nil
return cert.NotBefore, cert.NotAfter, nil
}

View File

@ -96,7 +96,7 @@ func TestExpirerControllerFilters(t *testing.T) {
nil, // k8sClient, not needed
secretsInformer,
withInformer.WithInformer,
0, // ageThreshold, not needed
0, // renewBefore, not needed
)
unrelated := corev1.Secret{}
@ -114,7 +114,7 @@ func TestExpirerControllerSync(t *testing.T) {
tests := []struct {
name string
ageThreshold float32
renewBefore time.Duration
fillSecretData func(*testing.T, map[string][]byte)
configKubeAPIClient func(*kubernetesfake.Clientset)
wantDelete bool
@ -130,47 +130,62 @@ func TestExpirerControllerSync(t *testing.T) {
wantDelete: false,
},
{
name: "lifetime below threshold",
ageThreshold: 0.7,
name: "lifetime below threshold",
renewBefore: 7 * time.Hour,
fillSecretData: func(t *testing.T, m map[string][]byte) {
caPEM, err := testutil.CreateCertificate(
certPEM, err := testutil.CreateCertificate(
time.Now().Add(-5*time.Hour),
time.Now().Add(5*time.Hour),
)
require.NoError(t, err)
// See cert_manager.go for this constant.
m["caCertificate"] = caPEM
// See certs_manager.go for this constant.
m["tlsCertificateChain"] = certPEM
},
wantDelete: false,
},
{
name: "lifetime above threshold",
ageThreshold: 0.3,
name: "lifetime above threshold",
renewBefore: 3 * time.Hour,
fillSecretData: func(t *testing.T, m map[string][]byte) {
caPEM, err := testutil.CreateCertificate(
certPEM, err := testutil.CreateCertificate(
time.Now().Add(-5*time.Hour),
time.Now().Add(5*time.Hour),
)
require.NoError(t, err)
// See cert_manager.go for this constant.
m["caCertificate"] = caPEM
// See certs_manager.go for this constant.
m["tlsCertificateChain"] = certPEM
},
wantDelete: true,
},
{
name: "delete failure",
ageThreshold: 0.3,
name: "cert expired",
renewBefore: 3 * time.Hour,
fillSecretData: func(t *testing.T, m map[string][]byte) {
caPEM, err := testutil.CreateCertificate(
certPEM, err := testutil.CreateCertificate(
time.Now().Add(-2*time.Hour),
time.Now().Add(-1*time.Hour),
)
require.NoError(t, err)
// See certs_manager.go for this constant.
m["tlsCertificateChain"] = certPEM
},
wantDelete: true,
},
{
name: "delete failure",
renewBefore: 3 * time.Hour,
fillSecretData: func(t *testing.T, m map[string][]byte) {
certPEM, err := testutil.CreateCertificate(
time.Now().Add(-5*time.Hour),
time.Now().Add(5*time.Hour),
)
require.NoError(t, err)
// See cert_manager.go for this constant.
m["caCertificate"] = caPEM
// See certs_manager.go for this constant.
m["tlsCertificateChain"] = certPEM
},
configKubeAPIClient: func(c *kubernetesfake.Clientset) {
c.PrependReactor("delete", "secrets", func(_ kubetesting.Action) (bool, runtime.Object, error) {
@ -185,8 +200,8 @@ func TestExpirerControllerSync(t *testing.T) {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
// See cert_manager.go for this constant.
m["caCertificate"] = x509.MarshalPKCS1PrivateKey(privateKey)
// See certs_manager.go for this constant.
m["tlsCertificateChain"] = x509.MarshalPKCS1PrivateKey(privateKey)
},
wantDelete: false,
},
@ -205,7 +220,7 @@ func TestExpirerControllerSync(t *testing.T) {
}
kubeInformerClient := kubernetesfake.NewSimpleClientset()
name := "api-serving-cert" // See cert_manager.go.
name := "api-serving-cert" // See certs_manager.go.
namespace := "some-namespace"
if test.fillSecretData != nil {
secret := &corev1.Secret{
@ -231,7 +246,7 @@ func TestExpirerControllerSync(t *testing.T) {
kubeAPIClient,
kubeInformers.Core().V1().Secrets(),
controller.WithInformer,
test.ageThreshold,
test.renewBefore,
)
// Must start informers before calling TestRunSynchronously().

View File

@ -36,6 +36,10 @@ type certsManagerController struct {
k8sClient kubernetes.Interface
aggregatorClient aggregatorclient.Interface
secretInformer corev1informers.SecretInformer
// certDuration is the lifetime of the serving certificate that this
// controller will use when issuing the serving certificate.
certDuration time.Duration
}
func NewCertsManagerController(
@ -45,6 +49,7 @@ func NewCertsManagerController(
secretInformer corev1informers.SecretInformer,
withInformer pinnipedcontroller.WithInformerOptionFunc,
withInitialEvent pinnipedcontroller.WithInitialEventOptionFunc,
certDuration time.Duration,
) controller.Controller {
return controller.New(
controller.Config{
@ -54,6 +59,7 @@ func NewCertsManagerController(
k8sClient: k8sClient,
aggregatorClient: aggregatorClient,
secretInformer: secretInformer,
certDuration: certDuration,
},
},
withInformer(
@ -95,7 +101,7 @@ func (c *certsManagerController) Sync(ctx controller.Context) error {
aggregatedAPIServerTLSCert, err := aggregatedAPIServerCA.Issue(
pkix.Name{CommonName: serviceEndpoint},
[]string{serviceEndpoint},
24*365*time.Hour,
c.certDuration,
)
if err != nil {
return fmt.Errorf("could not issue serving certificate: %w", err)

View File

@ -50,6 +50,7 @@ func TestManagerControllerOptions(t *testing.T) {
secretsInformer,
observableWithInformerOption.WithInformer, // make it possible to observe the behavior of the Filters
observableWithInitialEventOption.WithInitialEvent, // make it possible to observe the behavior of the initial event
0, // certDuration, not needed for this test
)
secretsInformerFilter = observableWithInformerOption.GetFilterForInformer(secretsInformer)
})
@ -116,6 +117,7 @@ func TestManagerControllerOptions(t *testing.T) {
func TestManagerControllerSync(t *testing.T) {
spec.Run(t, "Sync", func(t *testing.T, when spec.G, it spec.S) {
const installedInNamespace = "some-namespace"
const certDuration = 12345678 * time.Second
var r *require.Assertions
@ -139,6 +141,7 @@ func TestManagerControllerSync(t *testing.T) {
kubeInformers.Core().V1().Secrets(),
controller.WithInformer,
controller.WithInitialEvent,
certDuration,
)
// Set this at the last second to support calling subject.Name().
@ -221,7 +224,7 @@ func TestManagerControllerSync(t *testing.T) {
// Validate the created cert using the CA, and also validate the cert's hostname
validCert := testutil.ValidateCertificate(t, actualCACert, actualCertChain)
validCert.RequireDNSName("pinniped-api." + installedInNamespace + ".svc")
validCert.RequireLifetime(time.Now(), time.Now().Add(24*365*time.Hour), 2*time.Minute)
validCert.RequireLifetime(time.Now(), time.Now().Add(certDuration), 2*time.Minute)
validCert.RequireMatchesPrivateKey(actualPrivateKey)
// Make sure we updated the APIService caBundle and left it otherwise unchanged

View File

@ -34,7 +34,8 @@ func PrepareControllers(
serverInstallationNamespace string,
discoveryURLOverride *string,
dynamicCertProvider provider.DynamicTLSServingCertProvider,
servingCertRotationThreshold float32,
servingCertDuration time.Duration,
servingCertRenewBefore time.Duration,
) (func(ctx context.Context), error) {
// Create k8s clients.
k8sClient, aggregatorClient, pinnipedClient, err := createClients()
@ -68,6 +69,7 @@ func PrepareControllers(
installationNamespaceK8sInformers.Core().V1().Secrets(),
controller.WithInformer,
controller.WithInitialEvent,
servingCertDuration,
),
singletonWorker,
).
@ -86,7 +88,7 @@ func PrepareControllers(
k8sClient,
installationNamespaceK8sInformers.Core().V1().Secrets(),
controller.WithInformer,
servingCertRotationThreshold,
servingCertRenewBefore,
),
singletonWorker,
)

View File

@ -10,11 +10,9 @@ import (
"context"
"fmt"
"io"
"strconv"
"time"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
"k8s.io/apiserver/plugin/pkg/authenticator/token/webhook"
@ -23,7 +21,6 @@ import (
"github.com/suzerain-io/pinniped/internal/apiserver"
"github.com/suzerain-io/pinniped/internal/certauthority/kubecertauthority"
"github.com/suzerain-io/pinniped/internal/constable"
"github.com/suzerain-io/pinniped/internal/controllermanager"
"github.com/suzerain-io/pinniped/internal/downward"
"github.com/suzerain-io/pinniped/internal/provider"
@ -32,39 +29,13 @@ import (
"github.com/suzerain-io/pinniped/pkg/config"
)
type percentageValue struct {
percentage float32
}
var _ pflag.Value = &percentageValue{}
func (p *percentageValue) String() string {
return fmt.Sprintf("%.2f%%", p.percentage*100)
}
func (p *percentageValue) Set(s string) error {
f, err := strconv.ParseFloat(s, 32)
if err != nil || f < 0 || f > 1 {
return constable.Error("must pass real number between 0 and 1")
}
p.percentage = float32(f)
return nil
}
func (p *percentageValue) Type() string {
return "percentage"
}
// App is an object that represents the pinniped-server application.
type App struct {
cmd *cobra.Command
// CLI flags
configPath string
downwardAPIPath string
servingCertRotationThreshold percentageValue
configPath string
downwardAPIPath string
}
// This is ignored for now because we turn off etcd storage below, but this is
@ -118,13 +89,6 @@ func addCommandlineFlagsToCommand(cmd *cobra.Command, app *App) {
"/etc/podinfo",
"path to Downward API volume mount",
)
app.servingCertRotationThreshold.percentage = .70 // default
cmd.Flags().Var(
&app.servingCertRotationThreshold,
"serving-cert-rotation-threshold",
"real number between 0 and 1 indicating percentage of lifetime before rotation of serving cert",
)
}
// Boot the aggregated API server, which will in turn boot the controllers.
@ -168,7 +132,8 @@ func (a *App) runServer(ctx context.Context) error {
serverInstallationNamespace,
cfg.DiscoveryInfo.URL,
dynamicCertProvider,
a.servingCertRotationThreshold.percentage,
time.Duration(*cfg.APIConfig.ServingCertificateConfig.DurationSeconds)*time.Second,
time.Duration(*cfg.APIConfig.ServingCertificateConfig.RenewBeforeSeconds)*time.Second,
)
if err != nil {
return fmt.Errorf("could not prepare controllers: %w", err)

View File

@ -25,11 +25,10 @@ Usage:
pinniped-server [flags]
Flags:
-c, --config string path to configuration file (default "pinniped.yaml")
--downward-api-path string path to Downward API volume mount (default "/etc/podinfo")
-h, --help help for pinniped-server
--log-flush-frequency duration Maximum number of seconds between log flushes (default 5s)
--serving-cert-rotation-threshold percentage real number between 0 and 1 indicating percentage of lifetime before rotation of serving cert (default 70.00%)
-c, --config string path to configuration file (default "pinniped.yaml")
--downward-api-path string path to Downward API volume mount (default "/etc/podinfo")
-h, --help help for pinniped-server
--log-flush-frequency duration Maximum number of seconds between log flushes (default 5s)
`
func TestCommand(t *testing.T) {
@ -69,30 +68,6 @@ func TestCommand(t *testing.T) {
},
wantErr: `unknown command "tuna" for "pinniped-server"`,
},
{
name: "PercentageIsNotRealNumber",
args: []string{
"--config", "some/path/to/config.yaml",
"--serving-cert-rotation-threshold", "tuna",
},
wantErr: `invalid argument "tuna" for "--serving-cert-rotation-threshold" flag: must pass real number between 0 and 1`,
},
{
name: "PercentageIsTooSmall",
args: []string{
"--config", "some/path/to/config.yaml",
"--serving-cert-rotation-threshold", "-1",
},
wantErr: `invalid argument "-1" for "--serving-cert-rotation-threshold" flag: must pass real number between 0 and 1`,
},
{
name: "PercentageIsTooLarge",
args: []string{
"--config", "some/path/to/config.yaml",
"--serving-cert-rotation-threshold", "75",
},
wantErr: `invalid argument "75" for "--serving-cert-rotation-threshold" flag: must pass real number between 0 and 1`,
},
}
for _, test := range tests {
test := test

View File

@ -9,9 +9,10 @@ package api
type Config struct {
WebhookConfig WebhookConfigSpec `json:"webhook"`
DiscoveryInfo DiscoveryInfoSpec `json:"discovery"`
APIConfig APIConfigSpec `json:"api"`
}
// WebhookConfig contains configuration knobs specific to Pinniped's use
// WebhookConfig contains configuration knobs specific to pinniped's use
// of a webhook for token validation.
type WebhookConfigSpec struct {
// URL contains the URL of the webhook that pinniped will use
@ -31,3 +32,26 @@ type DiscoveryInfoSpec struct {
// URL contains the URL at which pinniped can be contacted.
URL *string `json:"url,omitempty"`
}
// APIConfigSpec contains configuration knobs for the pinniped API.
//nolint: golint
type APIConfigSpec struct {
ServingCertificateConfig ServingCertificateConfigSpec `json:"servingCertificate"`
}
// ServingCertificateConfigSpec contains the configuration knobs for the API's
// serving certificate, i.e., the x509 certificate that it uses for the server
// certificate in inbound TLS connections.
type ServingCertificateConfigSpec struct {
// DurationSeconds is the validity period, in seconds, of the API serving
// certificate. By default, the serving certificate is issued for 31536000
// seconds (1 year).
DurationSeconds *int64 `json:"durationSeconds,omitempty"`
// RenewBeforeSeconds is the period of time, in seconds, that pinniped will
// wait before rotating the serving certificate. This period of time starts
// upon issuance of the serving certificate. This must be less than
// DurationSeconds. By default, pinniped begins rotation after 23328000
// seconds (about 9 months).
RenewBeforeSeconds *int64 `json:"renewBeforeSeconds,omitempty"`
}

View File

@ -13,10 +13,18 @@ import (
"sigs.k8s.io/yaml"
"github.com/suzerain-io/pinniped/internal/constable"
"github.com/suzerain-io/pinniped/pkg/config/api"
)
// FromPath loads an api.Config from a provided local file path.
const (
aboutAYear = 60 * 60 * 24 * 365
about9Months = 60 * 60 * 24 * 30 * 9
)
// FromPath loads an api.Config from a provided local file path, inserts any
// defaults (from the api.Config documentation), and verifies that the config is
// valid (per the api.Config documentation).
//
// Note! The api.Config file should contain base64-encoded WebhookCABundle data.
// This function will decode that base64-encoded data to PEM bytes to be stored
@ -32,5 +40,33 @@ func FromPath(path string) (*api.Config, error) {
return nil, fmt.Errorf("decode yaml: %w", err)
}
maybeSetAPIDefaults(&config.APIConfig)
if err := validateAPI(&config.APIConfig); err != nil {
return nil, fmt.Errorf("validate api: %w", err)
}
return &config, nil
}
func maybeSetAPIDefaults(apiConfig *api.APIConfigSpec) {
if apiConfig.ServingCertificateConfig.DurationSeconds == nil {
apiConfig.ServingCertificateConfig.DurationSeconds = int64Ptr(aboutAYear)
}
if apiConfig.ServingCertificateConfig.RenewBeforeSeconds == nil {
apiConfig.ServingCertificateConfig.RenewBeforeSeconds = int64Ptr(about9Months)
}
}
func validateAPI(apiConfig *api.APIConfigSpec) error {
if *apiConfig.ServingCertificateConfig.DurationSeconds < *apiConfig.ServingCertificateConfig.RenewBeforeSeconds {
return constable.Error("durationSeconds cannot be smaller than renewBeforeSeconds")
}
return nil
}
func int64Ptr(i int64) *int64 {
return &i
}

View File

@ -18,6 +18,7 @@ func TestFromPath(t *testing.T) {
name string
path string
wantConfig *api.Config
wantError string
}{
{
name: "Happy",
@ -30,11 +31,17 @@ func TestFromPath(t *testing.T) {
URL: "https://tuna.com/fish?marlin",
CABundle: []byte("-----BEGIN CERTIFICATE-----..."),
},
APIConfig: api.APIConfigSpec{
ServingCertificateConfig: api.ServingCertificateConfigSpec{
DurationSeconds: int64Ptr(3600),
RenewBeforeSeconds: int64Ptr(2400),
},
},
},
},
{
name: "NoDiscovery",
path: "testdata/no-discovery.yaml",
name: "Default",
path: "testdata/default.yaml",
wantConfig: &api.Config{
DiscoveryInfo: api.DiscoveryInfoSpec{
URL: nil,
@ -43,21 +50,34 @@ func TestFromPath(t *testing.T) {
URL: "https://tuna.com/fish?marlin",
CABundle: []byte("-----BEGIN CERTIFICATE-----..."),
},
APIConfig: api.APIConfigSpec{
ServingCertificateConfig: api.ServingCertificateConfigSpec{
DurationSeconds: int64Ptr(60 * 60 * 24 * 365), // about a year
RenewBeforeSeconds: int64Ptr(60 * 60 * 24 * 30 * 9), // about 9 months
},
},
},
},
{
name: "InvalidDurationRenewBefore",
path: "testdata/invalid-duration-renew-before.yaml",
wantError: "validate api: durationSeconds cannot be smaller than renewBeforeSeconds",
},
}
for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) {
config, err := FromPath(test.path)
require.NoError(t, err)
require.Equal(t, test.wantConfig, config)
if test.wantError != "" {
require.EqualError(t, err, test.wantError)
} else {
require.NoError(t, err)
require.Equal(t, test.wantConfig, config)
}
})
}
}
func stringPtr(s string) *string {
sPtr := new(string)
*sPtr = s
return sPtr
return &s
}

View File

@ -4,3 +4,7 @@ discovery:
webhook:
url: https://tuna.com/fish?marlin
caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tLi4u
api:
servingCertificate:
durationSeconds: 3600
renewBeforeSeconds: 2400

View File

@ -0,0 +1,8 @@
---
webhook:
url: https://tuna.com/fish?marlin
caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tLi4u
api:
servingCertificate:
durationSeconds: 2400
renewBeforeSeconds: 3600

View File

@ -58,7 +58,7 @@ func TestAPIServingCertificateAutoCreationAndRotation(t *testing.T) {
return err
}
secret.Data["caCertificate"], err = createExpiredCertificate()
secret.Data["tlsCertificateChain"], err = createExpiredCertificate()
if err != nil {
return err
}