feature/acme: lock ACME per-domain instead of globally (#20303)

The extension's acmeMu was a single lock around getCertPEM. Any
in-flight ACME flow blocked every other domain. With many domains
(ProxyGroup ingress) the queue would back up and per-call timeouts
started firing while we were just waiting on the lock -- the cert
loop treated that as a failure.

Replace with one mutex per domain. Different domains run at the
same time. Same domain still queues so the first run fills the
cache and the rest read from it.

The old global lock also kept ACME account setup safe by accident.
Two goroutines could both find no account key, both generate one,
both write -- last one wins on disk but each carries on with its
own. Add acmeAccountMu around acmeKey and ensureACMEAccount to
keep that path single-file. Otherwise two first-time issuances for
different domains end up with separate accounts at LE.

Updates #20288
Updates tailscale/corp#42164

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>

feature/acme/cert.go

@@ -279,7 +279,7 @@ func (e *extension) domainRenewalTimeByARI(b *ipnlocal.LocalBackend, cs certStor
 	if len(blocks) < 1 {
 		return time.Time{}, fmt.Errorf("could not parse certificate chain from certStore, got %d PEM block(s)", len(blocks))
 	}
-	ac, err := acmeClient(cs)
+	ac, err := e.acmeClient(cs)
 	if err != nil {
 		return time.Time{}, err
 	}
@@ -305,8 +305,9 @@ func (e *extension) domainRenewalTimeByARI(b *ipnlocal.LocalBackend, cs certStor
 // domain is the resolved cert domain (e.g., "*.node.ts.net" for
 // wildcards). It can be overridden in tests.
 var getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBackend, cs certStore, logf logger.Logf, traceACME func(any), domain string, now time.Time, minValidity time.Duration) (*ipnlocal.TLSCertKeyPair, error) {
-	e.acmeMu.Lock()
-	defer e.acmeMu.Unlock()
+	dm := e.lockDomain(domain)
+	dm.Lock()
+	defer dm.Unlock()
 
 	// In case this method was triggered multiple times in parallel (when
 	// serving incoming requests), check whether one of the other goroutines
@@ -335,7 +336,7 @@ var getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBacken
 		defer e.setCertPending(b, domain, false)
 	}
 
-	ac, err := acmeClient(cs)
+	ac, err := e.acmeClient(cs)
 	if err != nil {
 		return nil, err
 	}
@@ -344,25 +345,9 @@ var getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBacken
 		logf("acme: using Directory URL %q", ac.DirectoryURL)
 	}
 
-	a, err := ac.GetReg(ctx, "" /* pre-RFC param */)
-	switch {
-	case err == nil:
-		// Great, already registered.
-		logf("already had ACME account.")
-	case err == xacme.ErrNoAccount:
-		a, err = ac.Register(ctx, new(xacme.Account), xacme.AcceptTOS)
-		if err == xacme.ErrAccountAlreadyExists {
-			// Potential race. Double check.
-			a, err = ac.GetReg(ctx, "" /* pre-RFC param */)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("acme.Register: %w", err)
-		}
-		logf("registered ACME account.")
-		traceACME(a)
-	default:
-		return nil, fmt.Errorf("acme.GetReg: %w", err)
-
+	a, err := e.ensureAccount(ctx, ac, logf, traceACME)
+	if err != nil {
+		return nil, err
 	}
 	if a.Status != xacme.StatusValid {
 		return nil, fmt.Errorf("unexpected ACME account status %q", a.Status)
@@ -410,6 +395,32 @@ var getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBacken
 	return e.issueACMECert(ctx, b, ac, issueArgs)
 }
 
+// ensureAccount returns a valid ACME account, registering one if
+// needed. Locked so two first-time issuances don't both create one.
+func (e *extension) ensureAccount(ctx context.Context, ac *xacme.Client, logf logger.Logf, traceACME func(any)) (*xacme.Account, error) {
+	e.accountMu.Lock()
+	defer e.accountMu.Unlock()
+	a, err := ac.GetReg(ctx, "" /* pre-RFC param */)
+	switch {
+	case err == nil:
+		logf("already had ACME account.")
+		return a, nil
+	case err == xacme.ErrNoAccount:
+		a, err = ac.Register(ctx, new(xacme.Account), xacme.AcceptTOS)
+		if err == xacme.ErrAccountAlreadyExists {
+			a, err = ac.GetReg(ctx, "" /* pre-RFC param */)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("acme.Register: %w", err)
+		}
+		logf("registered ACME account.")
+		traceACME(a)
+		return a, nil
+	default:
+		return nil, fmt.Errorf("acme.GetReg: %w", err)
+	}
+}
+
 type acmeCertIssueArgs struct {
 	cs            certStore          // certificate and ACME account storage
 	logf          logger.Logf        // logs ACME progress and failures