tailscale/ipn/ipnlocal/watchdog.go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause

package ipnlocal

import (
	"log"
	"runtime"
	"time"

	"tailscale.com/tstime"
)

// deadlockProbeDelay is how long a watched call must be in flight before we
// start probing locks to check for a deadlock. Calls that complete sooner do
// not trigger any probing.
const deadlockProbeDelay = 5 * time.Second

// deadlockTimeout is how long the lock-probing goroutine is allowed to run
// before we declare a deadlock and panic with goroutine stacks. That is, it's
// the maximum total time we allow any of the probed locks to be held.
const deadlockTimeout = 30 * time.Second

// CheckDeadlocks schedules a delayed deadlock probe and returns a function to
// call when the operation being watched is done. Callers typically use it as
// "defer b.CheckDeadlocks()()" to bracket a region of code that should not
// take more than [deadlockProbeDelay].
//
// This is a backstop for detecting and debugging deadlocks in the process, replacing
// the earlier watchdogEngine removed in 2b338dd6a8dbd.
func (b *LocalBackend) CheckDeadlocks() (done func()) {
	// Bump the in-flight count. If a watched region is already open, the
	// probe timer is already armed, so the bump is all we need to do: the
	// matching doneDeadlockCheck will decrement when this caller returns and
	// only the last one out will stop the timer.
	if b.deadlockChecksInFlight.Add(1) != 1 {
		return b.doneDeadlockCheck
	}

	// Fast path to avoid the deadlockTimerMu+Timer.Reset cost when
	// CheckDeadlocks is called many times per second by non-overlapping
	// callers: re-arm the probe timer at most once per wall-clock second.
	// We use a unix-seconds timestamp (+1 so 0 can mean "never") and a CAS
	// so that only one caller per second proceeds to touch the timer; the
	// rest return early.
	nowUnix := tstime.DefaultClock{Clock: b.Clock()}.Now().Unix() + 1
	lastUnix := b.lastDeadlockCheckUnix.Load()
	if lastUnix == nowUnix || !b.lastDeadlockCheckUnix.CompareAndSwap(lastUnix, nowUnix) {
		return b.doneDeadlockCheck
	}

	// Slow path: (re)arm the probe timer. Lazily create it on first use.
	b.deadlockTimerMu.Lock()
	defer b.deadlockTimerMu.Unlock()

	t := b.deadlockProbeTimer
	if t == nil {
		t = time.AfterFunc(deadlockProbeDelay, b.runDeadlockProbe)
		b.deadlockProbeTimer = t
	} else {
		t.Reset(deadlockProbeDelay)
	}
	return b.doneDeadlockCheck
}

func (b *LocalBackend) doneDeadlockCheck() {
	switch n := b.deadlockChecksInFlight.Add(-1); {
	case n > 0:
		return
	case n < 0:
		panic("ipnlocal: doneDeadlockCheck called without matching CheckDeadlocks")
	}

	b.deadlockTimerMu.Lock()
	defer b.deadlockTimerMu.Unlock()
	if b.deadlockProbeTimer == nil {
		return
	}
	b.deadlockProbeTimer.Stop()
}

func (b *LocalBackend) runDeadlockProbe() {
	b.deadlockTimerMu.Lock()
	defer b.deadlockTimerMu.Unlock()

	if b.deadlockChecksInFlight.Load() == 0 {
		return
	}

	t := b.deadlockTimer
	if t == nil {
		t = time.AfterFunc(deadlockTimeout, b.reportDeadlock)
		b.deadlockTimer = t
	} else {
		t.Reset(deadlockTimeout)
	}
	defer t.Stop()

	b.probeLocks()
}

func (b *LocalBackend) probeLocks() {
	b.probeLocalBackendLock()

	sys := b.sys
	if sys == nil {
		return
	}
	if bus, ok := sys.Bus.GetOK(); ok && bus != nil {
		bus.ProbeLocks()
	}
	if dialer, ok := sys.Dialer.GetOK(); ok && dialer != nil {
		dialer.ProbeLocks()
	}
	if dm, ok := sys.DNSManager.GetOK(); ok && dm != nil {
		dm.ProbeLocks()
	}
	if e, ok := sys.Engine.GetOK(); ok && e != nil {
		e.ProbeLocks()
	}
	if nm, ok := sys.NetMon.GetOK(); ok && nm != nil {
		nm.ProbeLocks()
	}
	if mc, ok := sys.MagicSock.GetOK(); ok && mc != nil {
		mc.ProbeLocks()
	}
	if tun, ok := sys.Tun.GetOK(); ok && tun != nil {
		tun.ProbeLocks()
	}
	if ht, ok := sys.HealthTracker.GetOK(); ok && ht != nil {
		ht.ProbeLocks()
	}
}

func (b *LocalBackend) probeLocalBackendLock() {
	b.mu.Lock()
	defer b.mu.Unlock()
}

func (b *LocalBackend) reportDeadlock() {
	logf := b.logf
	if logf == nil {
		logf = log.Printf
	}
	logf("ipnlocal watchdog goroutine stacks:\n%s", goroutineStacks())
	panic("ipnlocal: watchdog timeout")
}

func goroutineStacks() []byte {
	buf := make([]byte, 256<<10)
	for {
		n := runtime.Stack(buf, true)
		if n < len(buf) {
			return buf[:n]
		}
		buf = make([]byte, 2*len(buf))
	}
}