ipn/ipnlocal: add wireguard session state metrics + publish on IPN bus

Updates #19989
Updates tailscale/corp#42874

Change-Id: I843ed95bc7b0f5cd38ba1467332c6b022901e254
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

ipn/backend.go

@@ -171,6 +171,13 @@ const (
 	// LocalBackend.WatchNotificationsAs. LocalAPI WatchIPNBus clients must
 	// not request it.
 	NotifyInProcessNoDisconnect NotifyWatchOpt = 1 << 16
+
+	// NotifyPeerWireGuardState, if set, opts the watcher into
+	// WireGuard session state notifications via [Notify.PeerState].
+	// The first Notify sent to the watcher includes a dump of current
+	// non-zero peer states, and subsequent Notifies include per-peer
+	// state changes.
+	NotifyPeerWireGuardState NotifyWatchOpt = 1 << 18
 )
 
 // NotifyRateLimitIncompatibleBits is the set of new-style IPN bus
@@ -315,6 +322,10 @@ type Notify struct {
 	// the per-field accessors to read them.
 	UserProfiles map[tailcfg.UserID]tailcfg.UserProfileView `json:",omitzero"`
 
+	// PeerState, if non-empty, carries WireGuard session states keyed by stable
+	// node ID. Watchers must opt in via [NotifyPeerWireGuardState].
+	PeerState map[tailcfg.StableNodeID]PeerState `json:",omitzero"`
+
 	Engine      *EngineStatus // if non-nil, the new or current wireguard stats
 	BrowseToURL *string       // if non-nil, UI should open a browser right now
 
@@ -369,6 +380,71 @@ type Notify struct {
 	// type is mirrored in xcode/IPN/Core/LocalAPI/Model/LocalAPIModel.swift
 }
 
+// PeerWireGuardState is the WireGuard session state for a peer.
+//
+// It JSON-marshals as a lowercase string (e.g. "handshake", "established")
+// rather than its integer value, so the wire format does not depend on the
+// numeric constants below.
+type PeerWireGuardState uint8
+
+const (
+	PeerWireGuardStateNone        PeerWireGuardState = 0
+	PeerWireGuardStateHandshake   PeerWireGuardState = 1
+	PeerWireGuardStateEstablished PeerWireGuardState = 2
+	PeerWireGuardStateExpired     PeerWireGuardState = 3
+)
+
+// String returns the lowercase string form of s used by [PeerWireGuardState.MarshalText].
+func (s PeerWireGuardState) String() string {
+	switch s {
+	case PeerWireGuardStateNone:
+		return "none"
+	case PeerWireGuardStateHandshake:
+		return "handshake"
+	case PeerWireGuardStateEstablished:
+		return "established"
+	case PeerWireGuardStateExpired:
+		return "expired"
+	}
+	return fmt.Sprintf("PeerWireGuardState(%d)", uint8(s))
+}
+
+// MarshalText implements [encoding.TextMarshaler].
+func (s PeerWireGuardState) MarshalText() ([]byte, error) {
+	return []byte(s.String()), nil
+}
+
+// UnmarshalText implements [encoding.TextUnmarshaler].
+func (s *PeerWireGuardState) UnmarshalText(b []byte) error {
+	switch string(b) {
+	case "none":
+		*s = PeerWireGuardStateNone
+	case "handshake":
+		*s = PeerWireGuardStateHandshake
+	case "established":
+		*s = PeerWireGuardStateEstablished
+	case "expired":
+		*s = PeerWireGuardStateExpired
+	default:
+		return fmt.Errorf("unknown PeerWireGuardState %q", b)
+	}
+	return nil
+}
+
+// PeerState is the per-peer WireGuard session state delivered in
+// [Notify.PeerState].
+type PeerState struct {
+	// PeerWireGuardState is the current WireGuard session state for the peer.
+	PeerWireGuardState PeerWireGuardState
+
+	// PeerWireGuardStateAt is the wall-clock time at which the peer entered
+	// [PeerState.PeerWireGuardState], as observed by tailscaled.
+	// It is tracked by [LocalBackend] even when no watchers are subscribed,
+	// so a later subscriber's initial snapshot reflects the true transition
+	// time rather than the subscription time.
+	PeerWireGuardStateAt time.Time
+}
+
 func (n Notify) String() string {
 	var sb strings.Builder
 	sb.WriteString("Notify{")
@@ -390,6 +466,9 @@ func (n Notify) String() string {
 	if n.PeerChangedPatch != nil {
 		fmt.Fprintf(&sb, "PeerChangedPatch(%d) ", len(n.PeerChangedPatch))
 	}
+	if len(n.PeerState) > 0 {
+		fmt.Fprintf(&sb, "PeerState(%d) ", len(n.PeerState))
+	}
 	if n.Engine != nil {
 		fmt.Fprintf(&sb, "wg=%v ", *n.Engine)
 	}

ipn/backend_test.go

@@ -4,6 +4,7 @@
 package ipn
 
 import (
+	"encoding/json"
 	"testing"
 
 	"tailscale.com/health"
@@ -41,6 +42,41 @@ func TestNotifyString(t *testing.T) {
 	}
 }
 
+func TestPeerWireGuardStateJSON(t *testing.T) {
+	tests := []struct {
+		state PeerWireGuardState
+		json  string
+	}{
+		{PeerWireGuardStateNone, `"none"`},
+		{PeerWireGuardStateHandshake, `"handshake"`},
+		{PeerWireGuardStateEstablished, `"established"`},
+		{PeerWireGuardStateExpired, `"expired"`},
+	}
+	for _, tt := range tests {
+		t.Run(tt.state.String(), func(t *testing.T) {
+			got, err := json.Marshal(tt.state)
+			if err != nil {
+				t.Fatalf("Marshal: %v", err)
+			}
+			if string(got) != tt.json {
+				t.Errorf("Marshal(%v) = %s; want %s", tt.state, got, tt.json)
+			}
+			var back PeerWireGuardState
+			if err := json.Unmarshal(got, &back); err != nil {
+				t.Fatalf("Unmarshal: %v", err)
+			}
+			if back != tt.state {
+				t.Errorf("round-trip = %v; want %v", back, tt.state)
+			}
+		})
+	}
+
+	var bad PeerWireGuardState
+	if err := json.Unmarshal([]byte(`"bogus"`), &bad); err == nil {
+		t.Errorf("Unmarshal of bogus value did not return an error")
+	}
+}
+
 func TestValidateNotifyWatchOpt(t *testing.T) {
 	tests := []struct {
 		name    string

ipn/ipnlocal/bus.go

@@ -241,6 +241,7 @@ func isNotableNotify(n *ipn.Notify) bool {
 		len(n.PeersChanged) > 0 ||
 		len(n.PeersRemoved) > 0 ||
 		len(n.UserProfiles) > 0 ||
+		len(n.PeerState) > 0 ||
 		!n.DriveShares.IsNil() ||
 		n.Health != nil ||
 		len(n.IncomingFiles) > 0 ||

ipn/ipnlocal/bus_test.go

@@ -35,6 +35,7 @@ func TestIsNotableNotify(t *testing.T) {
 		{"peersremoved", &ipn.Notify{PeersRemoved: []tailcfg.NodeID{1}}, true},
 		{"userprofiles", &ipn.Notify{UserProfiles: map[tailcfg.UserID]tailcfg.UserProfileView{1: (&tailcfg.UserProfile{}).View()}}, true},
 		{"engine", &ipn.Notify{Engine: new(ipn.EngineStatus)}, false},
+		{"peerstate", &ipn.Notify{PeerState: map[tailcfg.StableNodeID]ipn.PeerState{"a": {PeerWireGuardState: ipn.PeerWireGuardStateHandshake}}}, true},
 		{"selfchange", &ipn.Notify{SelfChange: &tailcfg.Node{}}, true},
 	}
 
@@ -45,7 +46,7 @@ func TestIsNotableNotify(t *testing.T) {
 	for sf := range rt.Fields() {
 		n := &ipn.Notify{}
 		switch sf.Name {
-		case "_", "NetMap", "PeerChangedPatch", "SelfChange", "PeersChanged", "PeersRemoved", "UserProfiles", "Engine", "Version":
+		case "_", "NetMap", "PeerChangedPatch", "SelfChange", "PeersChanged", "PeersRemoved", "UserProfiles", "Engine", "PeerState", "Version":
 			// Already covered above or not applicable.
 			continue
 		case "DriveShares":

ipn/ipnlocal/local.go

@@ -14,6 +14,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"maps"
 	"math"
 	"math/rand/v2"
 	"net"
@@ -361,6 +362,21 @@ type LocalBackend struct {
 	lastStatusTime    time.Time                // status.AsOf value of the last processed status update
 	componentLogUntil map[string]componentLogState
 	currentUser       ipnauth.Actor
+	peerWGStateQueue  execqueue.ExecQueue // serializes WireGuard state transitions from wireguard-go
+	// peerWGState is the current non-zero WireGuard session state per peer,
+	// keyed by stable node ID for delivery on the IPN bus.
+	// Entries are added/updated by [LocalBackend.handlePeerWireGuardState]
+	// and removed when wireguard-go reports [wgengine.PeerWireGuardStateNone]
+	// (including the synthetic None fired by wireguard-go's RemovePeer path
+	// when a peer is removed from the WG config).
+	peerWGState map[tailcfg.StableNodeID]ipn.PeerState
+	// peerWGStableIDByKey caches the wireguard-go NodePublic -> StableNodeID
+	// translation so callbacks don't have to re-resolve through the netmap on
+	// every transition, and so the final PeerWireGuardStateNone callback for a
+	// removed peer can still be mapped to its StableNodeID after the peer is
+	// gone from the netmap. Entries are added on the first non-zero transition
+	// for a key and removed alongside the corresponding peerWGState entry.
+	peerWGStableIDByKey map[key.NodePublic]tailcfg.StableNodeID
 
 	// capForcedNetfilter is the netfilter that control instructs Linux clients
 	// to use, unless overridden locally.
@@ -628,6 +644,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo
 	nb.ready()
 
 	e.SetPeerByIPPacketFunc(b.lookupPeerByIP)
+	e.SetPeerSessionStateFunc(b.onPeerWireGuardState)
 
 	if sys.InitialConfig != nil {
 		if err := b.initPrefsFromConfig(sys.InitialConfig); err != nil {
@@ -1307,6 +1324,7 @@ func (b *LocalBackend) Shutdown() {
 	//  2. Event handlers may not guard against undesirable post/in-progress
 	//     LocalBackend.Shutdown() behaviors.
 	b.appcTask.Shutdown()
+	b.peerWGStateQueue.Shutdown()
 	b.eventClient.Close()
 
 	b.em.close()
@@ -3704,6 +3722,9 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 		statusSB = &ipnstate.StatusBuilder{WantPeers: true}
 		b.e.UpdateStatus(statusSB)
 	}
+	if mask&ipn.NotifyPeerWireGuardState != 0 {
+		_ = b.peerWGStateQueue.Wait(ctx)
+	}
 
 	// Watch for deadlocks only during the registration phase below; the rest
 	// of this method blocks on ctx (often for hours) and shouldn't trip the
@@ -3711,7 +3732,7 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 	deadlockDone := b.CheckDeadlocks()
 	b.mu.Lock()
 
-	const initialBits = ipn.NotifyInitialState | ipn.NotifyInitialPrefs | ipn.NotifyInitialNetMap | ipn.NotifyInitialStatus | ipn.NotifyInitialDriveShares | ipn.NotifyInitialSuggestedExitNode | ipn.NotifyInitialClientVersion
+	const initialBits = ipn.NotifyInitialState | ipn.NotifyInitialPrefs | ipn.NotifyInitialNetMap | ipn.NotifyInitialStatus | ipn.NotifyInitialDriveShares | ipn.NotifyInitialSuggestedExitNode | ipn.NotifyInitialClientVersion | ipn.NotifyPeerWireGuardState
 	if mask&initialBits != 0 {
 		cn := b.currentNode()
 		ini = &ipn.Notify{Version: version.Long()}
@@ -3769,6 +3790,9 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A
 		mask:      mask,
 	}
 	mak.Set(&b.notifyWatchers, sessionID, session)
+	if mask&ipn.NotifyPeerWireGuardState != 0 {
+		ini.PeerState = maps.Clone(b.peerWGState)
+	}
 	b.mu.Unlock()
 	deadlockDone()
 
@@ -4073,6 +4097,7 @@ func (b *LocalBackend) notifyForSessionLocked(sess *watchSession, n *ipn.Notify)
 	stripPeersChanged := len(n.PeersChanged) > 0 && !wantsPeerChanges
 	stripPeersRemoved := len(n.PeersRemoved) > 0 && !wantsPeerChanges
 	stripPatches := len(n.PeerChangedPatch) > 0 && !wantsPeerPatches
+	stripPeerState := len(n.PeerState) > 0 && sess.mask&ipn.NotifyPeerWireGuardState == 0
 	promotePatches := len(n.PeerChangedPatch) > 0 && wantsPeerChanges && !wantsPeerPatches
 
 	if sess.selfChangeResetsImplicitState(n.SelfChange.View()) {
@@ -4103,7 +4128,7 @@ func (b *LocalBackend) notifyForSessionLocked(sess *watchSession, n *ipn.Notify)
 	}
 	replaceUserProfiles := !stripUserProfiles && len(sessUserProfiles) != len(n.UserProfiles)
 
-	if !stripNetMap && !stripPeersChanged && !stripPeersRemoved && !stripPatches && !stripUserProfiles && !replaceUserProfiles && !promotePatches {
+	if !stripNetMap && !stripPeersChanged && !stripPeersRemoved && !stripPatches && !stripPeerState && !stripUserProfiles && !replaceUserProfiles && !promotePatches {
 		return n
 	}
 	nCopy := *n
@@ -4132,6 +4157,9 @@ func (b *LocalBackend) notifyForSessionLocked(sess *watchSession, n *ipn.Notify)
 	if stripPatches {
 		nCopy.PeerChangedPatch = nil
 	}
+	if stripPeerState {
+		nCopy.PeerState = nil
+	}
 	if stripUserProfiles {
 		nCopy.UserProfiles = nil
 	} else if replaceUserProfiles {
@@ -5761,6 +5789,98 @@ func (b *LocalBackend) lookupPeerByIP(ip netip.Addr) (key.NodePublic, bool) {
 	return peer.Key(), true
 }
 
+// onPeerWireGuardState is called by wireguard-go, through wgengine, for
+// serialized WireGuard session state transitions. wireguard-go is holding locks
+// while calling this, so this must stay cheap, must not acquire b.mu, and must
+// not call back into wireguard-go. Acquiring b.mu here can deadlock with
+// LocalBackend operations that hold b.mu while waiting for wireguard-go to make
+// progress.
+func (b *LocalBackend) onPeerWireGuardState(peerKey key.NodePublic, state wgengine.PeerWireGuardState) {
+	st := peerWireGuardStateFromEngine(state)
+	// 10ms granularity is plenty for diagnostics and keeps the JSON form short.
+	at := b.clock.Now().Round(10 * time.Millisecond)
+	b.peerWGStateQueue.Add(func() {
+		b.handlePeerWireGuardState(peerKey, st, at)
+	})
+}
+
+func (b *LocalBackend) handlePeerWireGuardState(peerKey key.NodePublic, st ipn.PeerWireGuardState, at time.Time) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	id, ok := b.peerWGStableIDByKey[peerKey]
+	if !ok {
+		nb := b.currentNode()
+		nid, ok := nb.NodeByKey(peerKey)
+		if !ok {
+			b.logf("[unexpected] WireGuard session state for unknown peer %v", peerKey.ShortString())
+			return
+		}
+		peer, ok := nb.NodeByID(nid)
+		if !ok {
+			b.logf("[unexpected] WireGuard session state for unknown node %v", nid)
+			return
+		}
+		id = peer.StableID()
+	}
+
+	if st == ipn.PeerWireGuardStateNone {
+		old, ok := b.peerWGState[id]
+		if !ok {
+			return
+		}
+		delete(b.peerWGState, id)
+		delete(b.peerWGStableIDByKey, peerKey)
+		addPeerWGStateMetric(old.PeerWireGuardState, -1)
+	} else {
+		old := b.peerWGState[id].PeerWireGuardState
+		if old == st {
+			return
+		}
+		if old != ipn.PeerWireGuardStateNone {
+			addPeerWGStateMetric(old, -1)
+		}
+		mak.Set(&b.peerWGState, id, ipn.PeerState{
+			PeerWireGuardState:   st,
+			PeerWireGuardStateAt: at,
+		})
+		mak.Set(&b.peerWGStableIDByKey, peerKey, id)
+		addPeerWGStateMetric(st, 1)
+	}
+	b.sendToLocked(ipn.Notify{
+		PeerState: map[tailcfg.StableNodeID]ipn.PeerState{id: {
+			PeerWireGuardState:   st,
+			PeerWireGuardStateAt: at,
+		}},
+	}, allClients)
+}
+
+func addPeerWGStateMetric(state ipn.PeerWireGuardState, delta int64) {
+	switch state {
+	case ipn.PeerWireGuardStateHandshake:
+		metricPeerWGStateHandshake.Add(delta)
+	case ipn.PeerWireGuardStateEstablished:
+		metricPeerWGStateEstablished.Add(delta)
+	case ipn.PeerWireGuardStateExpired:
+		metricPeerWGStateExpired.Add(delta)
+	}
+}
+
+func peerWireGuardStateFromEngine(state wgengine.PeerWireGuardState) ipn.PeerWireGuardState {
+	switch state {
+	case wgengine.PeerWireGuardStateNone:
+		return ipn.PeerWireGuardStateNone
+	case wgengine.PeerWireGuardStateHandshake:
+		return ipn.PeerWireGuardStateHandshake
+	case wgengine.PeerWireGuardStateEstablished:
+		return ipn.PeerWireGuardStateEstablished
+	case wgengine.PeerWireGuardStateExpired:
+		return ipn.PeerWireGuardStateExpired
+	default:
+		panic(fmt.Sprintf("unexpected wgengine.PeerWireGuardState %d", state))
+	}
+}
+
 func (b *LocalBackend) isEngineBlocked() bool {
 	b.mu.Lock()
 	defer b.mu.Unlock()
@@ -8950,6 +9070,9 @@ func maybeUsernameOf(actor ipnauth.Actor) string {
 var (
 	metricCurrentWatchIPNBus     = clientmetric.NewGauge("localbackend_current_watch_ipn_bus")
 	metricIPForwardingCheckError = clientmetric.NewCounter("localbackend_ip_forwarding_check_error")
+	metricPeerWGStateHandshake   = clientmetric.NewGauge("localbackend_peer_wireguard_state_handshake")
+	metricPeerWGStateEstablished = clientmetric.NewGauge("localbackend_peer_wireguard_state_established")
+	metricPeerWGStateExpired     = clientmetric.NewGauge("localbackend_peer_wireguard_state_expired")
 
 	// Counters for the controlclient's delta-update fast path: each
 	// counts a destination-side call into [LocalBackend] from

ipn/ipnlocal/local_test.go

@@ -2357,6 +2357,210 @@ func TestNotifyForSessionPeerVisibility(t *testing.T) {
 	})
 }
 
+func TestNotifyForSessionPeerWireGuardStateVisibility(t *testing.T) {
+	b := newTestLocalBackend(t)
+	notify := ipn.Notify{
+		PeerState: map[tailcfg.StableNodeID]ipn.PeerState{
+			"stable1": {PeerWireGuardState: ipn.PeerWireGuardStateEstablished},
+		},
+	}
+
+	deliver := func(mask ipn.NotifyWatchOpt) *ipn.Notify {
+		sess := &watchSession{mask: mask}
+		b.mu.Lock()
+		defer b.mu.Unlock()
+		return b.notifyForSessionLocked(sess, &notify)
+	}
+
+	if n := deliver(0); len(n.PeerState) != 0 {
+		t.Fatalf("PeerState without watch bit = %v; want empty", n.PeerState)
+	}
+	if n := deliver(ipn.NotifyPeerWireGuardState); n.PeerState["stable1"].PeerWireGuardState != ipn.PeerWireGuardStateEstablished {
+		t.Fatalf("PeerState with watch bit = %v; want stable1=established", n.PeerState)
+	}
+}
+
+func TestPeerWireGuardStateValuesMatchWGEngine(t *testing.T) {
+	const unknownPeerWireGuardState wgengine.PeerWireGuardState = 255
+
+	tests := []struct {
+		name string
+		in   wgengine.PeerWireGuardState
+		want ipn.PeerWireGuardState
+	}{
+		{"none", wgengine.PeerWireGuardStateNone, ipn.PeerWireGuardStateNone},
+		{"handshake", wgengine.PeerWireGuardStateHandshake, ipn.PeerWireGuardStateHandshake},
+		{"established", wgengine.PeerWireGuardStateEstablished, ipn.PeerWireGuardStateEstablished},
+		{"expired", wgengine.PeerWireGuardStateExpired, ipn.PeerWireGuardStateExpired},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := peerWireGuardStateFromEngine(tt.in); got != tt.want {
+				t.Fatalf("converted state = %v; want %v", got, tt.want)
+			}
+			if got, want := uint8(tt.in), uint8(tt.want); got != want {
+				t.Fatalf("wgengine const = %v; want %v", got, want)
+			}
+		})
+	}
+
+	defer func() {
+		if recover() == nil {
+			t.Fatal("expected panic for unknown wgengine state")
+		}
+	}()
+	_ = peerWireGuardStateFromEngine(unknownPeerWireGuardState)
+}
+
+func TestPeerWireGuardStateWatchInitialThenDeltas(t *testing.T) {
+	b := newTestLocalBackend(t)
+
+	peer1 := &tailcfg.Node{
+		ID:       1,
+		StableID: "stable1",
+		Key:      makeNodeKeyFromID(1),
+	}
+	peer2 := &tailcfg.Node{
+		ID:       2,
+		StableID: "stable2",
+		Key:      makeNodeKeyFromID(2),
+	}
+	b.currentNode().SetNetMap(&netmap.NetworkMap{
+		Peers: []tailcfg.NodeView{peer1.View(), peer2.View()},
+	})
+
+	t1 := time.Unix(1700000000, 0)
+	b.handlePeerWireGuardState(peer1.Key, ipn.PeerWireGuardStateEstablished, t1)
+	b.handlePeerWireGuardState(peer1.Key, ipn.PeerWireGuardStateEstablished, t1.Add(time.Second))
+	defer b.handlePeerWireGuardState(peer1.Key, ipn.PeerWireGuardStateNone, time.Now())
+	defer b.handlePeerWireGuardState(peer2.Key, ipn.PeerWireGuardStateNone, time.Now())
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	gotCh := make(chan *ipn.Notify, 2)
+	doneCh := make(chan struct{})
+	go func() {
+		defer close(doneCh)
+		var got int
+		b.WatchNotificationsAs(ctx, nil, ipn.NotifyPeerWireGuardState, func() {
+			b.onPeerWireGuardState(peer2.Key, wgengine.PeerWireGuardStateHandshake)
+		}, func(n *ipn.Notify) bool {
+			gotCh <- n
+			got++
+			if got == 2 {
+				cancel()
+				return false
+			}
+			return true
+		})
+	}()
+
+	var got []*ipn.Notify
+	for len(got) < 2 {
+		select {
+		case n := <-gotCh:
+			got = append(got, n)
+		case <-time.After(5 * time.Second):
+			t.Fatalf("timed out waiting for notifications; got %v", got)
+		}
+	}
+	<-doneCh
+
+	if got[0].PeerState["stable1"].PeerWireGuardState != ipn.PeerWireGuardStateEstablished {
+		t.Fatalf("initial PeerState = %v; want stable1=established", got[0].PeerState)
+	}
+	if got, want := got[0].PeerState["stable1"].PeerWireGuardStateAt, t1; !got.Equal(want) {
+		t.Fatalf("initial PeerState[stable1].PeerWireGuardStateAt = %v; want %v (no-op transition preserves first-entry time)", got, want)
+	}
+	if _, ok := got[0].PeerState["stable2"]; ok {
+		t.Fatalf("initial PeerState includes post-registration delta: %v", got[0].PeerState)
+	}
+	if len(got[1].PeerState) != 1 || got[1].PeerState["stable2"].PeerWireGuardState != ipn.PeerWireGuardStateHandshake {
+		t.Fatalf("delta PeerState = %v; want stable2=handshake only", got[1].PeerState)
+	}
+	if got[1].PeerState["stable2"].PeerWireGuardStateAt.IsZero() {
+		t.Fatalf("delta PeerState[stable2].PeerWireGuardStateAt is zero; want set by LocalBackend clock")
+	}
+}
+
+func TestPeerWireGuardStateMetrics(t *testing.T) {
+	b := newTestLocalBackend(t)
+	peer := &tailcfg.Node{
+		ID:       1,
+		StableID: "stable1",
+		Key:      makeNodeKeyFromID(1),
+	}
+	b.currentNode().SetNetMap(&netmap.NetworkMap{
+		Peers: []tailcfg.NodeView{peer.View()},
+	})
+	now := time.Now()
+	defer b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateNone, now)
+
+	baseHandshake := metricPeerWGStateHandshake.Value()
+	baseEstablished := metricPeerWGStateEstablished.Value()
+	baseExpired := metricPeerWGStateExpired.Value()
+
+	b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateEstablished, now)
+	if got, want := metricPeerWGStateEstablished.Value(), baseEstablished+1; got != want {
+		t.Fatalf("established gauge = %v; want %v", got, want)
+	}
+	b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateEstablished, now)
+	if got, want := metricPeerWGStateEstablished.Value(), baseEstablished+1; got != want {
+		t.Fatalf("established gauge after no-op = %v; want %v", got, want)
+	}
+
+	b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateHandshake, now)
+	if got, want := metricPeerWGStateEstablished.Value(), baseEstablished; got != want {
+		t.Fatalf("established gauge after transition = %v; want %v", got, want)
+	}
+	if got, want := metricPeerWGStateHandshake.Value(), baseHandshake+1; got != want {
+		t.Fatalf("handshake gauge = %v; want %v", got, want)
+	}
+
+	b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateExpired, now)
+	if got, want := metricPeerWGStateHandshake.Value(), baseHandshake; got != want {
+		t.Fatalf("handshake gauge after transition = %v; want %v", got, want)
+	}
+	if got, want := metricPeerWGStateExpired.Value(), baseExpired+1; got != want {
+		t.Fatalf("expired gauge = %v; want %v", got, want)
+	}
+
+	b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateNone, now)
+	if got, want := metricPeerWGStateExpired.Value(), baseExpired; got != want {
+		t.Fatalf("expired gauge after none = %v; want %v", got, want)
+	}
+}
+
+func TestPeerWireGuardStateCallbackDoesNotBlockOnLocalBackendMu(t *testing.T) {
+	b := newTestLocalBackend(t)
+	peer := &tailcfg.Node{
+		ID:       1,
+		StableID: "stable1",
+		Key:      makeNodeKeyFromID(1),
+	}
+	b.currentNode().SetNetMap(&netmap.NetworkMap{
+		Peers: []tailcfg.NodeView{peer.View()},
+	})
+
+	b.mu.Lock()
+	done := make(chan struct{})
+	go func() {
+		b.onPeerWireGuardState(peer.Key, wgengine.PeerWireGuardStateEstablished)
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		b.mu.Unlock()
+		t.Fatal("onPeerWireGuardState blocked on LocalBackend.mu")
+	}
+	b.mu.Unlock()
+
+	_ = b.peerWGStateQueue.Wait(context.Background())
+	b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateNone, time.Now())
+}
+
 func TestSetControlClientStatusSendsFullNetmapAsPeerChanges(t *testing.T) {
 	b := newTestLocalBackend(t)
 	nw := newNotificationWatcher(t, b, ipnauth.Self)

ipn/ipnlocal/state_test.go

@@ -1982,6 +1982,8 @@ func (e *mockEngine) Ping(ip netip.Addr, pingType tailcfg.PingType, size int, cb
 func (e *mockEngine) InstallCaptureHook(packet.CaptureCallback) {}
 
 func (e *mockEngine) SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool)) {}
+func (e *mockEngine) SetPeerSessionStateFunc(func(key.NodePublic, wgengine.PeerWireGuardState)) {
+}
 
 func (e *mockEngine) Close() {
 	e.mu.Lock()