Newer
Older
AnthosCertManager / pkg / controller / certificates / keymanager / keymanager_controller.go
package keymanager

import (
	"context"
	"crypto"
	"fmt"
	"time"

	apiutil "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/api/util"
	"gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util/pki"

	acmapi "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/apis/anthoscertmanager/v1"
	acmmeta "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/apis/meta/v1"
	acmclient "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/client/clientset/versioned"
	acminformers "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/client/informers/externalversions"
	acmlisters "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/client/listers/anthoscertmanager/v1"
	controllerpkg "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller"
	"gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/controller/certificates"
	"gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/util/predicate"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/selection"

	logf "gitbucket.jerxie.com/yangyangxie/AnthosCertManager/pkg/logs"
	"github.com/go-logr/logr"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/workqueue"
)

const (
	ControllerName            = "certificates-key-manager"
	reasonDecodeFailed        = "DecodeFailed"
	reasonCannotRegenerateKey = "CannotRegenerateKey"
	reasonDeleted             = "Deleted"
)

var (
	certificateGvk = acmapi.SchemeGroupVersion.WithKind("Certificate")
)

type controller struct {
	certificateLister acmlisters.CertificateLister
	secretLister      corelisters.SecretLister
	client            acmclient.Interface
	coreClient        kubernetes.Interface
	recorder          record.EventRecorder

	// fieldManager is the string which will be used as the Field Manager on
	// fields created or edited by the cert-manager Kubernetes client during
	// Apply API calls.
	fieldManager string
}

func NewController(
	log logr.Logger,
	client acmclient.Interface,
	coreClient kubernetes.Interface,
	factory informers.SharedInformerFactory,
	cmFactory acminformers.SharedInformerFactory,
	recorder record.EventRecorder,
	fieldManager string,
) (*controller, workqueue.RateLimitingInterface, []cache.InformerSynced) {
	// create a queue used to queue up items to be processed
	queue := workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(time.Second*1, time.Second*30), ControllerName)

	// obtain references to all the informers used by this controller
	certificateInformer := cmFactory.AnthosCertmanager().V1().Certificates()
	secretsInformer := factory.Core().V1().Secrets()

	certificateInformer.Informer().AddEventHandler(&controllerpkg.QueuingEventHandler{Queue: queue})

	secretsInformer.Informer().AddEventHandler(&controllerpkg.BlockingEventHandler{
		// Trigger reconciles on changes to any 'owned' secret resources
		WorkFunc: certificates.EnqueueCertificatesForResourceUsingPredicates(log, queue, certificateInformer.Lister(), labels.Everything(),
			predicate.ResourceOwnerOf,
		),
	})
	secretsInformer.Informer().AddEventHandler(&controllerpkg.BlockingEventHandler{
		// Trigger reconciles on changes to certificates named as spec.secretName
		WorkFunc: certificates.EnqueueCertificatesForResourceUsingPredicates(log, queue, certificateInformer.Lister(), labels.Everything(),
			predicate.ExtractResourceName(predicate.CertificateSecretName),
		),
	})

	// build a list of InformerSynced functions that will be returned by the Register method.
	// the controller will only begin processing items once all of these informers have synced.
	mustSync := []cache.InformerSynced{
		secretsInformer.Informer().HasSynced,
		certificateInformer.Informer().HasSynced,
	}

	return &controller{
		certificateLister: certificateInformer.Lister(),
		secretLister:      secretsInformer.Lister(),
		client:            client,
		coreClient:        coreClient,
		recorder:          recorder,
		fieldManager:      fieldManager,
	}, queue, mustSync
}

// isNextPrivateKeyLabelSelector is a label selector used to match Secret
// resources with the `cert-manager.io/next-private-key: "true"` label.
var isNextPrivateKeyLabelSelector labels.Selector

func init() {
	r, err := labels.NewRequirement("cert-manager.io/next-private-key", selection.Equals, []string{"true"})
	if err != nil {
		panic(err)
	}
	isNextPrivateKeyLabelSelector = labels.NewSelector().Add(*r)
}

func (c *controller) ProcessItem(ctx context.Context, key string) error {
	log := logf.FromContext(ctx).WithValues("key", key)
	ctx = logf.NewContext(ctx, log)

	namespace, name, err := cache.SplitMetaNamespaceKey(key)
	if err != nil {
		log.Error(err, "invalid resource key passed to ProcessItem")
		return nil
	}

	crt, err := c.certificateLister.Certificates(namespace).Get(name)
	if apierrors.IsNotFound(err) {
		log.V(logf.DebugLevel).Info("certificate not found for key", "error", err.Error())
		return nil
	}
	if err != nil {
		return err
	}

	// Discover all 'owned' secrets that have the `next-private-key` label
	secrets, err := certificates.ListSecretsMatchingPredicates(c.secretLister.Secrets(crt.Namespace), isNextPrivateKeyLabelSelector, predicate.ResourceOwnedBy(crt))
	if err != nil {
		return err
	}

	if !apiutil.CertificateHasCondition(crt, acmapi.CertificateCondition{
		Type:   acmapi.CertificateConditionIssuing,
		Status: acmmeta.ConditionTrue,
	}) {
		log.V(logf.DebugLevel).Info("Cleaning up Secret resources and unsetting nextPrivateKeySecretName as issuance is no longer in progress")
		if err := c.deleteSecretResources(ctx, secrets); err != nil {
			return err
		}
		return c.setNextPrivateKeySecretName(ctx, crt, nil)
	}

	// if there is no existing Secret resource, create a new one
	if len(secrets) == 0 {
		rotationPolicy := acmapi.RotationPolicyNever
		if crt.Spec.PrivateKey != nil && crt.Spec.PrivateKey.RotationPolicy != "" {
			rotationPolicy = crt.Spec.PrivateKey.RotationPolicy
		}
		switch rotationPolicy {
		case acmapi.RotationPolicyNever:
			return c.createNextPrivateKeyRotationPolicyNever(ctx, crt)
		case acmapi.RotationPolicyAlways:
			log.V(logf.DebugLevel).Info("Creating new nextPrivateKeySecretName Secret because no existing Secret found")
			return c.createAndSetNextPrivateKey(ctx, crt)
		default:
			log.V(logf.WarnLevel).Info("Certificate with unknown certificate.spec.privateKey.rotationPolicy value", "rotation_policy", rotationPolicy)
			return nil
		}
	}

	// always clean up if multiple are found
	if len(secrets) > 1 {
		// TODO: if nextPrivateKeySecretName is set, we should skip deleting that one Secret resource
		log.V(logf.DebugLevel).Info("Cleaning up Secret resources as multiple nextPrivateKeySecretName candidates found")
		return c.deleteSecretResources(ctx, secrets)
	}

	secret := secrets[0]
	log = logf.WithRelatedResource(log, secret)
	ctx = logf.NewContext(ctx, log)

	if crt.Status.NextPrivateKeySecretName == nil {
		log.V(logf.DebugLevel).Info("Adopting existing private key Secret")
		return c.setNextPrivateKeySecretName(ctx, crt, &secret.Name)
	}
	if *crt.Status.NextPrivateKeySecretName != secrets[0].Name {
		log.V(logf.DebugLevel).Info("Deleting existing private key secret as name does not match status.nextPrivateKeySecretName")
		return c.deleteSecretResources(ctx, secrets)
	}

	if secret.Data == nil || len(secret.Data[corev1.TLSPrivateKeyKey]) == 0 {
		log.V(logf.DebugLevel).Info("Deleting Secret resource as it contains no data")
		return c.deleteSecretResources(ctx, secrets)
	}
	pkData := secret.Data[corev1.TLSPrivateKeyKey]
	pk, err := pki.DecodePrivateKeyBytes(pkData)
	if err != nil {
		log.Error(err, "Deleting existing private key secret due to error decoding data")
		return c.deleteSecretResources(ctx, secrets)
	}

	violations, err := certificates.PrivateKeyMatchesSpec(pk, crt.Spec)
	if err != nil {
		log.Error(err, "Internal error verifying if private key matches spec - please open an issue.")
		return nil
	}
	if len(violations) > 0 {
		log.V(logf.DebugLevel).Info("Regenerating private key due to change in fields", "violations", violations)
		c.recorder.Eventf(crt, corev1.EventTypeNormal, reasonDeleted, "Regenerating private key due to change in fields: %v", violations)
		return c.deleteSecretResources(ctx, secrets)
	}

	return nil
}

func (c *controller) createNextPrivateKeyRotationPolicyNever(ctx context.Context, crt *acmapi.Certificate) error {
	log := logf.FromContext(ctx)
	s, err := c.secretLister.Secrets(crt.Namespace).Get(crt.Spec.SecretName)
	if apierrors.IsNotFound(err) {
		log.V(logf.DebugLevel).Info("Creating new nextPrivateKeySecretName Secret because no existing Secret found and rotation policy is Never")
		return c.createAndSetNextPrivateKey(ctx, crt)
	}
	if err != nil {
		return err
	}
	if s.Data == nil || len(s.Data[corev1.TLSPrivateKeyKey]) == 0 {
		log.V(logf.DebugLevel).Info("Creating new nextPrivateKeySecretName Secret because existing Secret contains empty data and rotation policy is Never")
		return c.createAndSetNextPrivateKey(ctx, crt)
	}
	existingPKData := s.Data[corev1.TLSPrivateKeyKey]
	pk, err := pki.DecodePrivateKeyBytes(existingPKData)
	if err != nil {
		c.recorder.Eventf(crt, corev1.EventTypeWarning, reasonDecodeFailed, "Failed to decode private key stored in Secret %q - generating new key", crt.Spec.SecretName)
		return c.createAndSetNextPrivateKey(ctx, crt)
	}
	violations, err := certificates.PrivateKeyMatchesSpec(pk, crt.Spec)
	if err != nil {
		c.recorder.Eventf(crt, corev1.EventTypeWarning, reasonDecodeFailed, "Failed to check if private key stored in Secret %q is up to date - generating new key", crt.Spec.SecretName)
		return c.createAndSetNextPrivateKey(ctx, crt)
	}
	if len(violations) > 0 {
		c.recorder.Eventf(crt, corev1.EventTypeWarning, reasonCannotRegenerateKey, "User intervention required: existing private key in Secret %q does not match requirements on Certificate resource, mismatching fields: %v, but cert-manager cannot create new private key as the Certificate's .spec.privateKey.rotationPolicy is unset or set to Never. To allow cert-manager to create a new private key you can set .spec.privateKey.rotationPolicy to 'Always' (this will result in the private key being regenerated every time a cert is renewed) ", crt.Spec.SecretName, violations)
		return nil
	}

	nextPkSecret, err := c.createNewPrivateKeySecret(ctx, crt, pk)
	if err != nil {
		return err
	}

	c.recorder.Event(crt, corev1.EventTypeNormal, "Reused", fmt.Sprintf("Reusing private key stored in existing Secret resource %q", s.Name))

	return c.setNextPrivateKeySecretName(ctx, crt, &nextPkSecret.Name)
}

func (c *controller) createAndSetNextPrivateKey(ctx context.Context, crt *acmapi.Certificate) error {
	pk, err := pki.GeneratePrivateKeyForCertificate(crt)
	if err != nil {
		return err
	}

	s, err := c.createNewPrivateKeySecret(ctx, crt, pk)
	if err != nil {
		return err
	}

	c.recorder.Event(crt, corev1.EventTypeNormal, "Generated", fmt.Sprintf("Stored new private key in temporary Secret resource %q", s.Name))

	return c.setNextPrivateKeySecretName(ctx, crt, &s.Name)
}

// deleteSecretResources will delete the given secret resources
func (c *controller) deleteSecretResources(ctx context.Context, secrets []*corev1.Secret) error {
	log := logf.FromContext(ctx)
	for _, s := range secrets {
		if err := c.coreClient.CoreV1().Secrets(s.Namespace).Delete(ctx, s.Name, metav1.DeleteOptions{}); err != nil {
			return err
		}
		logf.WithRelatedResource(log, s).V(logf.DebugLevel).Info("Deleted 'next private key' Secret resource")
	}
	return nil
}

func (c *controller) setNextPrivateKeySecretName(ctx context.Context, crt *acmapi.Certificate, name *string) error {
	// skip updates if there has been no change
	if name == nil && crt.Status.NextPrivateKeySecretName == nil {
		return nil
	}
	if name != nil && crt.Status.NextPrivateKeySecretName != nil {
		if *name == *crt.Status.NextPrivateKeySecretName {
			return nil
		}
	}
	crt = crt.DeepCopy()
	crt.Status.NextPrivateKeySecretName = name
	return c.updateOrApplyStatus(ctx, crt)
}

// updateOrApplyStatus will update the controller status.
func (c *controller) updateOrApplyStatus(ctx context.Context, crt *acmapi.Certificate) error {
	_, err := c.client.AnthosCertmanagerV1().Certificates(crt.Namespace).UpdateStatus(ctx, crt, metav1.UpdateOptions{})
	return err

}

func (c *controller) createNewPrivateKeySecret(ctx context.Context, crt *acmapi.Certificate, pk crypto.Signer) (*corev1.Secret, error) {
	// if the 'nextPrivateKeySecretName' field is already set, use this as the
	// name of the Secret resource.
	name := ""
	if crt.Status.NextPrivateKeySecretName != nil {
		name = *crt.Status.NextPrivateKeySecretName
	}

	pkData, err := pki.EncodePrivateKey(pk, acmapi.PKCS8)
	if err != nil {
		return nil, err
	}

	s := &corev1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Namespace:       crt.Namespace,
			Name:            name,
			OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(crt, certificateGvk)},
			Labels: map[string]string{
				"cert-manager.io/next-private-key": "true",
			},
		},
		Data: map[string][]byte{
			corev1.TLSPrivateKeyKey: pkData,
		},
	}
	if s.Name == "" {
		// TODO: handle certificate resources that have especially long names
		s.GenerateName = crt.Name + "-"
	}
	s, err = c.coreClient.CoreV1().Secrets(s.Namespace).Create(ctx, s, metav1.CreateOptions{})
	if err != nil {
		return nil, err
	}
	return s, nil
}

// controllerWrapper wraps the `controller` structure to make it implement
// the controllerpkg.queueingController interface
type controllerWrapper struct {
	*controller
}

func (c *controllerWrapper) Register(ctx *controllerpkg.Context) (workqueue.RateLimitingInterface, []cache.InformerSynced, error) {
	// construct a new named logger to be reused throughout the controller
	log := logf.FromContext(ctx.RootContext, ControllerName)

	ctrl, queue, mustSync := NewController(log,
		ctx.ACMClient,
		ctx.Client,
		ctx.KubeSharedInformerFactory,
		ctx.SharedInformerFactory,
		ctx.Recorder,
		ctx.FieldManager,
	)
	c.controller = ctrl

	return queue, mustSync, nil
}

func init() {
	controllerpkg.Register(ControllerName, func(ctx *controllerpkg.ContextFactory) (controllerpkg.Interface, error) {
		return controllerpkg.NewBuilder(ctx, ControllerName).
			For(&controllerWrapper{}).
			Complete()
	})
}