cmd/k8s-operator,ipn/store/kubestore,kube/kubetypes: share ACME account key per tailnet

Introduce a per-tailnet shared ACME account key so that all ingress
ProxyGroup replicas on a tailnet present the same account identity to
Let's Encrypt. This lets renewals claim the ARI "replaces" exemption
from the 50-certs-per-week rate limit, surviving Pod restarts,
ProxyGroup recreation, and cluster migrations.

The operator provisions a "tailscale-acme-accounts" Secret in its
namespace, guarded by a finalizer and a deletion warning event, and
watched so it is recreated promptly if removed. Proxies migrate any
pre-existing per-pod key into the shared Secret on first boot, adopt
the shared key on subsequent boots, and restore it on cert writes if
the Secret was recreated empty. Certs are stamped with the fingerprint
of the issuing account so renewals skip the "replaces" claim when the
account doesn't match.

Opt-in per-ProxyGroup via the tailscale.com/share-acme-account
annotation, or operator-wide via OPERATOR_SHARED_ACME_ACCOUNT_KEY.

Updates #18251
Updates #20288

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

cmd/k8s-operator/operator.go

@@ -96,6 +96,7 @@ func main() {
 		tsFirewallMode        = defaultEnv("PROXY_FIREWALL_MODE", "")
 		defaultProxyClass     = defaultEnv("PROXY_DEFAULT_CLASS", "")
 		isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
+		sharedACMEAccountKey  = defaultBool("OPERATOR_SHARED_ACME_ACCOUNT_KEY", false)
 		loginServer           = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/")
 		ingressClassName      = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale")
 		operatorSAName        = defaultEnv("OPERATOR_SERVICE_ACCOUNT_NAME", "operator")
@@ -170,6 +171,7 @@ func main() {
 		defaultProxyClass:             defaultProxyClass,
 		loginServer:                   loginServer,
 		ingressClassName:              ingressClassName,
+		sharedACMEAccountKey:          sharedACMEAccountKey,
 	})
 }
 
@@ -752,6 +754,7 @@ func runReconcilers(opts reconcilerOpts) {
 	proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
 	nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog))
 	saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog))
+	acmeSecretFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(acmeAccountsSecretHandlerForProxyGroup(mgr.GetClient(), opts.tailscaleNamespace, opts.sharedACMEAccountKey, startlog))
 	err = builder.ControllerManagedBy(mgr).
 		For(&tsapi.ProxyGroup{}).
 		Named("proxygroup-reconciler").
@@ -760,6 +763,9 @@ func runReconcilers(opts reconcilerOpts) {
 		Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
 		Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup).
 		Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
+		// The shared ACME accounts Secret has no ProxyGroup owner ref, so
+		// watch it by name to react to its deletion/recreation.
+		Watches(&corev1.Secret{}, acmeSecretFilterForProxyGroup).
 		Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
 		Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
 		Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
@@ -780,6 +786,8 @@ func runReconcilers(opts reconcilerOpts) {
 			loginServer:       opts.tsServer.ControlURL,
 			authKeyRateLimits: make(map[string]*rate.Limiter),
 			authKeyReissuing:  make(map[string]bool),
+
+			sharedACMEAccountKey: opts.sharedACMEAccountKey,
 		})
 	if err != nil {
 		startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)
@@ -835,6 +843,13 @@ type reconcilerOpts struct {
 	// ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults
 	// to "tailscale" but can be customised.
 	ingressClassName string
+	// sharedACMEAccountKey is the operator-wide default for the
+	// shared-ACME-account feature. When true, every ProxyGroup uses the
+	// shared per-tailnet account key unless the ProxyGroup explicitly
+	// opts out via tailscale.com/share-acme-account=false. When false,
+	// ProxyGroups opt in individually via
+	// tailscale.com/share-acme-account=true.
+	sharedACMEAccountKey bool
 	// operatorSAName is the name of the ServiceAccount that the operator pod runs as. It is used as the target
 	// ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate using
 	// workload identity federation.
@@ -1230,6 +1245,30 @@ func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLog
 	}
 }
 
+// acmeAccountsSecretHandlerForProxyGroup enqueues ProxyGroups that use the
+// shared ACME account when the shared ACME accounts Secret changes. The
+// Secret carries no owner reference, so the owner-based Secret watch never
+// matches it.
+func acmeAccountsSecretHandlerForProxyGroup(cl client.Client, tsNamespace string, sharedACMEAccountDefault bool, logger *zap.SugaredLogger) handler.MapFunc {
+	return func(ctx context.Context, o client.Object) []reconcile.Request {
+		if o.GetName() != kubetypes.ACMEAccountsSecretName || o.GetNamespace() != tsNamespace {
+			return nil
+		}
+		pgList := new(tsapi.ProxyGroupList)
+		if err := cl.List(ctx, pgList); err != nil {
+			logger.Debugf("error listing ProxyGroups for shared ACME accounts Secret: %v", err)
+			return nil
+		}
+		reqs := make([]reconcile.Request, 0, len(pgList.Items))
+		for _, pg := range pgList.Items {
+			if sharedACMEAccountEnabled(&pg, sharedACMEAccountDefault) {
+				reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
+			}
+		}
+		return reqs
+	}
+}
+
 // serviceHandlerForIngress returns a handler for Service events for ingress
 // reconciler that ensures that if the Service associated with an event is of
 // interest to the reconciler, the associated Ingress(es) gets be reconciled.