// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause // Package certs implements logic to help multiple Kubernetes replicas share TLS // certs for a common Tailscale Service. package certs import ( "context" "errors" "fmt" "net" "slices" "sync" "syscall" "time" "tailscale.com/client/local" "tailscale.com/ipn" "tailscale.com/kube/localclient" "tailscale.com/types/logger" "tailscale.com/util/goroutines" "tailscale.com/util/mak" ) // CertManager is responsible for issuing certificates for known domains and for // maintaining a loop that re-attempts issuance daily. // Currently cert manager logic is only run on ingress ProxyGroup replicas that are responsible for managing certs for // HA Ingress HTTPS endpoints ('write' replicas). type CertManager struct { lc localclient.LocalClient logf logger.Logf tracker goroutines.Tracker // tracks running goroutines mu sync.Mutex // guards the following // certLoops contains a map of DNS names, for which we currently need to // manage certs to cancel functions that allow stopping a goroutine when // we no longer need to manage certs for the DNS name. certLoops map[string]context.CancelFunc } func NewCertManager(lc localclient.LocalClient, logf logger.Logf) *CertManager { return &CertManager{ lc: lc, logf: logf, } } // EnsureCertLoops ensures that, for all currently managed Service HTTPS // endpoints, there is a cert loop responsible for issuing and ensuring the // renewal of the TLS certs. // ServeConfig must not be nil. func (cm *CertManager) EnsureCertLoops(ctx context.Context, sc *ipn.ServeConfig) error { if sc == nil { return fmt.Errorf("[unexpected] ensureCertLoops called with nil ServeConfig") } currentDomains := make(map[string]bool) const httpsPort = "443" for _, service := range sc.Services { // L7 Web handlers (HA Ingress). for hostPort := range service.Web { domain, port, err := net.SplitHostPort(string(hostPort)) if err != nil { return fmt.Errorf("[unexpected] unable to parse HostPort %s", hostPort) } if port != httpsPort { // HA Ingress' HTTP endpoint continue } currentDomains[domain] = true } // L4 TCP handlers with TLS termination (kube-apiserver proxy). for _, handler := range service.TCP { if handler != nil && handler.TerminateTLS != "" { currentDomains[handler.TerminateTLS] = true } } } cm.mu.Lock() defer cm.mu.Unlock() for domain := range currentDomains { if _, exists := cm.certLoops[domain]; !exists { cancelCtx, cancel := context.WithCancel(ctx) mak.Set(&cm.certLoops, domain, cancel) // Note that most of the issuance anyway happens // serially because the cert client has a shared lock // that's held during any issuance. cm.tracker.Go(func() { cm.runCertLoop(cancelCtx, domain) }) } } // Stop goroutines for domain names that are no longer in the config. for domain, cancel := range cm.certLoops { if !currentDomains[domain] { cancel() delete(cm.certLoops, domain) } } return nil } // isTransientCertErr reports whether err represents a failure that did not // reach the CA (ctx timeout, LocalAPI socket unreachable). Such errors must // not advance the loop's retryCount. func isTransientCertErr(err error) bool { switch { case errors.Is(err, context.DeadlineExceeded), errors.Is(err, context.Canceled), errors.Is(err, syscall.ECONNREFUSED), errors.Is(err, syscall.ECONNRESET), errors.Is(err, syscall.EHOSTUNREACH), errors.Is(err, syscall.EPIPE): return true } var ne net.Error if errors.As(err, &ne) && ne.Timeout() { return true } return false } // nextRetryInterval picks the wait until the next CertPair attempt: // - success: reset retryCount, normalInterval. // - transient (never reached the CA): retrySchedule[0], no retryCount advance. // - retryAfter > 0: honour it, retryCount still advances. // - other: advance retryCount, walk retrySchedule. func nextRetryInterval(err error, retryCount *int, normalInterval, retryAfter time.Duration) time.Duration { switch { case err == nil: *retryCount = 0 return normalInterval case isTransientCertErr(err): return retrySchedule[0] } *retryCount++ idx := *retryCount - 1 if idx >= len(retrySchedule) { idx = len(retrySchedule) - 1 } interval := retrySchedule[idx] if retryAfter > 0 { interval = retryAfter } return interval } // retrySchedule is the wait between successive failed issuance attempts, // following LE's recommended schedule. // https://letsencrypt.org/docs/integration-guide/#retrying-failures var retrySchedule = []time.Duration{ 1 * time.Minute, 10 * time.Minute, 100 * time.Minute, 24 * time.Hour, } // runCertLoop: // - calls localAPI certificate endpoint to ensure that certs are issued for the // given domain name // - calls localAPI certificate endpoint daily to ensure that certs are renewed // - if certificate issuance failed, retries on the schedule defined by // [retrySchedule]; resets to the start once issuance succeeds. // Note that renewal check also happens when the node receives an HTTPS request and it is possible that certs get // renewed at that point. Renewal here is needed to prevent the shared certs from expiry in edge cases where the 'write' // replica does not get any HTTPS requests. func (cm *CertManager) runCertLoop(ctx context.Context, domain string) { const normalInterval = 24 * time.Hour // regular renewal check if err := cm.waitForCertDomain(ctx, domain); err != nil { // Best-effort, log and continue with the issuing loop. cm.logf("error waiting for cert domain %s: %v", domain, err) } timer := time.NewTimer(0) // fire off timer immediately defer timer.Stop() retryCount := 0 for { select { case <-ctx.Done(): return case <-timer.C: // We call the certificate endpoint, but don't do anything with the // returned certs here. The call to the certificate endpoint will // ensure that certs are issued/renewed as needed and stored in the // relevant state store. For example, for HA Ingress 'write' replica, // the cert and key will be stored in a Kubernetes Secret named after // the domain for which we are issuing. // // Note that renewals triggered by the call to the certificates // endpoint here and by renewal check triggered during a call to // node's HTTPS endpoint share the same state/renewal lock mechanism, // so we should not run into redundant issuances during concurrent // renewal checks. // // Long enough to cover queue contention behind tailscaled's // shared cert mutex; if it fires, something is wedged. ctxT, cancel := context.WithTimeout(ctx, 30*time.Minute) _, _, err := cm.lc.CertPair(ctxT, domain) cancel() retryAfter, _ := local.RateLimitRetryAfter(err) nextInterval := nextRetryInterval(err, &retryCount, normalInterval, retryAfter) if err != nil { cm.logf("Error refreshing certificate for %s (retry %d): %v. Will retry in %v\n", domain, retryCount, err, nextInterval) } timer.Reset(nextInterval) } } } // domains before issuing the cert for the first time. It uses the IPN bus // only as a wake-up trigger (Notify.SelfChange) and queries the current // cert domains explicitly via [LocalClient.CertDomains]. func (cm *CertManager) waitForCertDomain(ctx context.Context, domain string) error { w, err := cm.lc.WatchIPNBus(ctx, ipn.NotifyInitialNetMap) if err != nil { return fmt.Errorf("error watching IPN bus: %w", err) } defer w.Close() for { n, err := w.Next() if err != nil { return err } if n.SelfChange == nil { continue } domains, err := cm.lc.CertDomains(ctx) if err != nil { continue } if slices.Contains(domains, domain) { return nil } } }