cmd/k8s-operator,ipn/store/kubestore,kube/kubetypes: share ACME account key per tailnet

Introduce a per-tailnet shared ACME account key so that all ingress
ProxyGroup replicas on a tailnet present the same account identity to
Let's Encrypt. This lets renewals claim the ARI "replaces" exemption
from the 50-certs-per-week rate limit, surviving Pod restarts,
ProxyGroup recreation, and cluster migrations.

The operator provisions a "tailscale-acme-accounts" Secret in its
namespace, guarded by a finalizer and a deletion warning event, and
watched so it is recreated promptly if removed. Proxies migrate any
pre-existing per-pod key into the shared Secret on first boot, adopt
the shared key on subsequent boots, and restore it on cert writes if
the Secret was recreated empty. Certs are stamped with the fingerprint
of the issuing account so renewals skip the "replaces" claim when the
account doesn't match.

Opt-in per-ProxyGroup via the tailscale.com/share-acme-account
annotation, or operator-wide via OPERATOR_SHARED_ACME_ACCOUNT_KEY.

Updates #18251
Updates #20288

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

feature/acme/cert.go

@@ -360,17 +360,30 @@ var getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBacken
 		return nil, fmt.Errorf("unexpected ACME account status %q", a.Status)
 	}
 
-	// If we have a previous cert, include it in the order. Assuming we're
-	// within the ARI renewal window this should exclude us from LE rate
-	// limits.
-	// Note that this order extension will fail renewals if the ACME account key has changed
-	// since the last issuance, see
-	// https://github.com/tailscale/tailscale/issues/18251
+	// If we have a previous cert, include it in the order via the ARI
+	// "replaces" extension so LE classifies the new cert as a renewal
+	// and exempts it from the per-registered-domain rate limit. Stores
+	// that can tell the current ACME account did not issue the previous
+	// cert opt out via [ARIReplacesAllower]; see #18251.
 	var opts []xacme.OrderOption
 	if previous != nil && !envknob.Bool("TS_DEBUG_ACME_FORCE_RENEWAL") {
 		prevCrt, err := parseCertificate(previous)
 		if err == nil {
-			opts = append(opts, xacme.WithOrderReplacesCert(prevCrt))
+			useReplaces := true
+			if a, ok := cs.(ARIReplacesAllower); ok {
+				if allowed, err := a.ShouldUseARIReplacesForRenewal(domain); err != nil {
+					// Fail open: an error means we couldn't check
+					// eligibility, not that the account is misaligned.
+					logf("acme: failed to check ARI 'replaces' eligibility for %q, defaulting to use it: %v", domain, err)
+				} else {
+					useReplaces = allowed
+				}
+			}
+			if useReplaces {
+				opts = append(opts, xacme.WithOrderReplacesCert(prevCrt))
+			} else {
+				logf("account key mismatch for previous cert; skipping ARI 'replaces' hint")
+			}
 		}
 	}
 

feature/acme/certstore.go

@@ -187,6 +187,25 @@ type TLSCertKeyReader interface {
 	ReadTLSCertAndKey(domain string) ([]byte, []byte, error)
 }
 
+// ARIReplacesAllower is optionally implemented by state stores to opt
+// out of the ARI "replaces" hint on a per-domain basis at renewal time.
+// When implemented and returning false, the renewal path submits a plain
+// newOrder instead of claiming renewal exemption via "replaces".
+//
+// Used by the k8s cert-share store to prevent submitting a "replaces"
+// claim that Let's Encrypt would reject because the current ACME
+// account did not issue the previous cert (which can happen when a
+// shared per-tailnet account key is adopted after a cert was already
+// issued by a per-pod account). See #18251.
+type ARIReplacesAllower interface {
+	// ShouldUseARIReplacesForRenewal reports whether the renewal order
+	// for domain should carry the ARI "replaces" hint. It is advisory:
+	// a non-nil error means eligibility could not be determined, not
+	// that the hint should be skipped, and callers fail open (use the
+	// hint) in that case.
+	ShouldUseARIReplacesForRenewal(domain string) (bool, error)
+}
+
 func (s certStateStore) Read(domain string, now time.Time) (*ipnlocal.TLSCertKeyPair, error) {
 	// If we're using a store that supports atomic reads, use that
 	if kr, ok := s.StateStore.(TLSCertKeyReader); ok {
@@ -215,6 +234,16 @@ func (s certStateStore) Read(domain string, now time.Time) (*ipnlocal.TLSCertKey
 	return &ipnlocal.TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, nil
 }
 
+// ShouldUseARIReplacesForRenewal delegates to the underlying state store
+// if it implements [ARIReplacesAllower]. Stores that don't opt in default
+// to true (attempt "replaces"), preserving the pre-hook behaviour.
+func (s certStateStore) ShouldUseARIReplacesForRenewal(domain string) (bool, error) {
+	if a, ok := s.StateStore.(ARIReplacesAllower); ok {
+		return a.ShouldUseARIReplacesForRenewal(domain)
+	}
+	return true, nil
+}
+
 func (s certStateStore) WriteCert(domain string, cert []byte) error {
 	return ipn.WriteState(s.StateStore, ipn.StateKey(domain+".crt"), cert)
 }