From 6596d237a3ea90e5457752015b66f08320226701 Mon Sep 17 00:00:00 2001 From: Brad Fitzpatrick Date: Fri, 5 Jun 2026 15:01:24 +0000 Subject: [PATCH] ipn/ipnlocal: add wireguard session state metrics + publish on IPN bus Updates #19989 Updates tailscale/corp#42874 Change-Id: I843ed95bc7b0f5cd38ba1467332c6b022901e254 Signed-off-by: Brad Fitzpatrick --- cmd/tailscale/cli/debug.go | 65 ++++++++++-- ipn/backend.go | 79 ++++++++++++++ ipn/backend_test.go | 36 +++++++ ipn/ipnlocal/bus.go | 1 + ipn/ipnlocal/bus_test.go | 3 +- ipn/ipnlocal/local.go | 127 ++++++++++++++++++++++- ipn/ipnlocal/local_test.go | 204 +++++++++++++++++++++++++++++++++++++ ipn/ipnlocal/state_test.go | 2 + wgengine/userspace.go | 24 +++++ wgengine/userspace_test.go | 33 ++++++ wgengine/wgengine.go | 38 +++++++ 11 files changed, 602 insertions(+), 10 deletions(-) diff --git a/cmd/tailscale/cli/debug.go b/cmd/tailscale/cli/debug.go index 2697ac5d1..a63039be3 100644 --- a/cmd/tailscale/cli/debug.go +++ b/cmd/tailscale/cli/debug.go @@ -270,8 +270,18 @@ func debugCmd() *ffcli.Command { FlagSet: (func() *flag.FlagSet { fs := newFlagSet("watch-ipn") fs.BoolVar(&watchIPNArgs.initial, "initial", false, "include the initial backend State and Prefs in the first message") - fs.BoolVar(&watchIPNArgs.rateLimit, "rate-limit", true, "rate limit messages") fs.IntVar(&watchIPNArgs.count, "count", 0, "exit after printing this many statuses, or 0 to keep going forever") + fs.BoolVar(&watchIPNArgs.engineUpdates, "engine-updates", false, "set NotifyWatchEngineUpdates: send Engine updates") + fs.BoolVar(&watchIPNArgs.initialDriveShares, "initial-drive-shares", false, "set NotifyInitialDriveShares: send current Taildrive Shares in first message") + fs.BoolVar(&watchIPNArgs.initialOutgoingFiles, "initial-outgoing-files", false, "set NotifyInitialOutgoingFiles: send current Taildrop OutgoingFiles in first message") + fs.BoolVar(&watchIPNArgs.initialHealthState, "initial-health", false, "set NotifyInitialHealthState: send current health.State in first message") + fs.BoolVar(&watchIPNArgs.healthActions, "health-actions", false, "set NotifyHealthActions: include PrimaryActions in health.State") + fs.BoolVar(&watchIPNArgs.initialSuggestedExitNode, "initial-suggested-exit-node", false, "set NotifyInitialSuggestedExitNode: send current SuggestedExitNode in first message") + fs.BoolVar(&watchIPNArgs.initialClientVersion, "initial-client-version", false, "set NotifyInitialClientVersion: send current ClientVersion in first message") + fs.BoolVar(&watchIPNArgs.peerChanges, "peer-changes", true, "set NotifyPeerChanges: send PeersChanged and PeersRemoved updates") + fs.BoolVar(&watchIPNArgs.initialStatus, "initial-status", false, "set NotifyInitialStatus: send current ipnstate.Status in first message") + fs.BoolVar(&watchIPNArgs.peerPatches, "peer-patches", true, "set NotifyPeerPatches: send narrow per-field peer patches") + fs.BoolVar(&watchIPNArgs.peerWireGuardState, "peer-wireguard-state", false, "set NotifyPeerWireGuardState: send WireGuard session state notifications") return fs })(), }, @@ -632,18 +642,59 @@ func runPrefs(ctx context.Context, args []string) error { } var watchIPNArgs struct { - initial bool - rateLimit bool - count int + initial bool + count int + + engineUpdates bool + initialDriveShares bool + initialOutgoingFiles bool + initialHealthState bool + healthActions bool + initialSuggestedExitNode bool + initialClientVersion bool + peerChanges bool + initialStatus bool + peerPatches bool + peerWireGuardState bool } func runWatchIPN(ctx context.Context, args []string) error { - mask := ipn.NotifyPeerChanges | ipn.NotifyPeerPatches + mask := ipn.NotifyNoNetMap if watchIPNArgs.initial { mask |= ipn.NotifyInitialState | ipn.NotifyInitialPrefs } - if watchIPNArgs.rateLimit { - mask |= ipn.NotifyRateLimit + if watchIPNArgs.engineUpdates { + mask |= ipn.NotifyWatchEngineUpdates + } + if watchIPNArgs.initialDriveShares { + mask |= ipn.NotifyInitialDriveShares + } + if watchIPNArgs.initialOutgoingFiles { + mask |= ipn.NotifyInitialOutgoingFiles + } + if watchIPNArgs.initialHealthState { + mask |= ipn.NotifyInitialHealthState + } + if watchIPNArgs.healthActions { + mask |= ipn.NotifyHealthActions + } + if watchIPNArgs.initialSuggestedExitNode { + mask |= ipn.NotifyInitialSuggestedExitNode + } + if watchIPNArgs.initialClientVersion { + mask |= ipn.NotifyInitialClientVersion + } + if watchIPNArgs.peerChanges { + mask |= ipn.NotifyPeerChanges + } + if watchIPNArgs.initialStatus { + mask |= ipn.NotifyInitialStatus + } + if watchIPNArgs.peerPatches { + mask |= ipn.NotifyPeerPatches + } + if watchIPNArgs.peerWireGuardState { + mask |= ipn.NotifyPeerWireGuardState } watcher, err := localClient.WatchIPNBus(ctx, mask) if err != nil { diff --git a/ipn/backend.go b/ipn/backend.go index e387e6d96..b38c47099 100644 --- a/ipn/backend.go +++ b/ipn/backend.go @@ -171,6 +171,13 @@ const ( // LocalBackend.WatchNotificationsAs. LocalAPI WatchIPNBus clients must // not request it. NotifyInProcessNoDisconnect NotifyWatchOpt = 1 << 16 + + // NotifyPeerWireGuardState, if set, opts the watcher into + // WireGuard session state notifications via [Notify.PeerState]. + // The first Notify sent to the watcher includes a dump of current + // non-zero peer states, and subsequent Notifies include per-peer + // state changes. + NotifyPeerWireGuardState NotifyWatchOpt = 1 << 18 ) // NotifyRateLimitIncompatibleBits is the set of new-style IPN bus @@ -315,6 +322,10 @@ type Notify struct { // the per-field accessors to read them. UserProfiles map[tailcfg.UserID]tailcfg.UserProfileView `json:",omitzero"` + // PeerState, if non-empty, carries WireGuard session states keyed by stable + // node ID. Watchers must opt in via [NotifyPeerWireGuardState]. + PeerState map[tailcfg.StableNodeID]PeerState `json:",omitzero"` + Engine *EngineStatus // if non-nil, the new or current wireguard stats BrowseToURL *string // if non-nil, UI should open a browser right now @@ -369,6 +380,71 @@ type Notify struct { // type is mirrored in xcode/IPN/Core/LocalAPI/Model/LocalAPIModel.swift } +// PeerWireGuardState is the WireGuard session state for a peer. +// +// It JSON-marshals as a lowercase string (e.g. "handshake", "established") +// rather than its integer value, so the wire format does not depend on the +// numeric constants below. +type PeerWireGuardState uint8 + +const ( + PeerWireGuardStateNone PeerWireGuardState = 0 + PeerWireGuardStateHandshake PeerWireGuardState = 1 + PeerWireGuardStateEstablished PeerWireGuardState = 2 + PeerWireGuardStateExpired PeerWireGuardState = 3 +) + +// String returns the lowercase string form of s used by [PeerWireGuardState.MarshalText]. +func (s PeerWireGuardState) String() string { + switch s { + case PeerWireGuardStateNone: + return "none" + case PeerWireGuardStateHandshake: + return "handshake" + case PeerWireGuardStateEstablished: + return "established" + case PeerWireGuardStateExpired: + return "expired" + } + return fmt.Sprintf("PeerWireGuardState(%d)", uint8(s)) +} + +// MarshalText implements [encoding.TextMarshaler]. +func (s PeerWireGuardState) MarshalText() ([]byte, error) { + return []byte(s.String()), nil +} + +// UnmarshalText implements [encoding.TextUnmarshaler]. +func (s *PeerWireGuardState) UnmarshalText(b []byte) error { + switch string(b) { + case "none": + *s = PeerWireGuardStateNone + case "handshake": + *s = PeerWireGuardStateHandshake + case "established": + *s = PeerWireGuardStateEstablished + case "expired": + *s = PeerWireGuardStateExpired + default: + return fmt.Errorf("unknown PeerWireGuardState %q", b) + } + return nil +} + +// PeerState is the per-peer WireGuard session state delivered in +// [Notify.PeerState]. +type PeerState struct { + // PeerWireGuardState is the current WireGuard session state for the peer. + PeerWireGuardState PeerWireGuardState + + // PeerWireGuardStateAt is the wall-clock time at which the peer entered + // [PeerState.PeerWireGuardState], as observed by tailscaled. + // It is tracked by [LocalBackend] even when no watchers are subscribed, + // so a later subscriber's initial snapshot reflects the true transition + // time rather than the subscription time. + PeerWireGuardStateAt time.Time +} + func (n Notify) String() string { var sb strings.Builder sb.WriteString("Notify{") @@ -390,6 +466,9 @@ func (n Notify) String() string { if n.PeerChangedPatch != nil { fmt.Fprintf(&sb, "PeerChangedPatch(%d) ", len(n.PeerChangedPatch)) } + if len(n.PeerState) > 0 { + fmt.Fprintf(&sb, "PeerState(%d) ", len(n.PeerState)) + } if n.Engine != nil { fmt.Fprintf(&sb, "wg=%v ", *n.Engine) } diff --git a/ipn/backend_test.go b/ipn/backend_test.go index 1957e21e6..f6ad29011 100644 --- a/ipn/backend_test.go +++ b/ipn/backend_test.go @@ -4,6 +4,7 @@ package ipn import ( + "encoding/json" "testing" "tailscale.com/health" @@ -41,6 +42,41 @@ func TestNotifyString(t *testing.T) { } } +func TestPeerWireGuardStateJSON(t *testing.T) { + tests := []struct { + state PeerWireGuardState + json string + }{ + {PeerWireGuardStateNone, `"none"`}, + {PeerWireGuardStateHandshake, `"handshake"`}, + {PeerWireGuardStateEstablished, `"established"`}, + {PeerWireGuardStateExpired, `"expired"`}, + } + for _, tt := range tests { + t.Run(tt.state.String(), func(t *testing.T) { + got, err := json.Marshal(tt.state) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + if string(got) != tt.json { + t.Errorf("Marshal(%v) = %s; want %s", tt.state, got, tt.json) + } + var back PeerWireGuardState + if err := json.Unmarshal(got, &back); err != nil { + t.Fatalf("Unmarshal: %v", err) + } + if back != tt.state { + t.Errorf("round-trip = %v; want %v", back, tt.state) + } + }) + } + + var bad PeerWireGuardState + if err := json.Unmarshal([]byte(`"bogus"`), &bad); err == nil { + t.Errorf("Unmarshal of bogus value did not return an error") + } +} + func TestValidateNotifyWatchOpt(t *testing.T) { tests := []struct { name string diff --git a/ipn/ipnlocal/bus.go b/ipn/ipnlocal/bus.go index fbbfd7dda..e1c060d55 100644 --- a/ipn/ipnlocal/bus.go +++ b/ipn/ipnlocal/bus.go @@ -241,6 +241,7 @@ func isNotableNotify(n *ipn.Notify) bool { len(n.PeersChanged) > 0 || len(n.PeersRemoved) > 0 || len(n.UserProfiles) > 0 || + len(n.PeerState) > 0 || !n.DriveShares.IsNil() || n.Health != nil || len(n.IncomingFiles) > 0 || diff --git a/ipn/ipnlocal/bus_test.go b/ipn/ipnlocal/bus_test.go index 14aaae541..47004a58a 100644 --- a/ipn/ipnlocal/bus_test.go +++ b/ipn/ipnlocal/bus_test.go @@ -35,6 +35,7 @@ func TestIsNotableNotify(t *testing.T) { {"peersremoved", &ipn.Notify{PeersRemoved: []tailcfg.NodeID{1}}, true}, {"userprofiles", &ipn.Notify{UserProfiles: map[tailcfg.UserID]tailcfg.UserProfileView{1: (&tailcfg.UserProfile{}).View()}}, true}, {"engine", &ipn.Notify{Engine: new(ipn.EngineStatus)}, false}, + {"peerstate", &ipn.Notify{PeerState: map[tailcfg.StableNodeID]ipn.PeerState{"a": {PeerWireGuardState: ipn.PeerWireGuardStateHandshake}}}, true}, {"selfchange", &ipn.Notify{SelfChange: &tailcfg.Node{}}, true}, } @@ -45,7 +46,7 @@ func TestIsNotableNotify(t *testing.T) { for sf := range rt.Fields() { n := &ipn.Notify{} switch sf.Name { - case "_", "NetMap", "PeerChangedPatch", "SelfChange", "PeersChanged", "PeersRemoved", "UserProfiles", "Engine", "Version": + case "_", "NetMap", "PeerChangedPatch", "SelfChange", "PeersChanged", "PeersRemoved", "UserProfiles", "Engine", "PeerState", "Version": // Already covered above or not applicable. continue case "DriveShares": diff --git a/ipn/ipnlocal/local.go b/ipn/ipnlocal/local.go index 4ccee52ee..7811612b8 100644 --- a/ipn/ipnlocal/local.go +++ b/ipn/ipnlocal/local.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "maps" "math" "math/rand/v2" "net" @@ -361,6 +362,21 @@ type LocalBackend struct { lastStatusTime time.Time // status.AsOf value of the last processed status update componentLogUntil map[string]componentLogState currentUser ipnauth.Actor + peerWGStateQueue execqueue.ExecQueue // serializes WireGuard state transitions from wireguard-go + // peerWGState is the current non-zero WireGuard session state per peer, + // keyed by stable node ID for delivery on the IPN bus. + // Entries are added/updated by [LocalBackend.handlePeerWireGuardState] + // and removed when wireguard-go reports [wgengine.PeerWireGuardStateNone] + // (including the synthetic None fired by wireguard-go's RemovePeer path + // when a peer is removed from the WG config). + peerWGState map[tailcfg.StableNodeID]ipn.PeerState + // peerWGStableIDByKey caches the wireguard-go NodePublic -> StableNodeID + // translation so callbacks don't have to re-resolve through the netmap on + // every transition, and so the final PeerWireGuardStateNone callback for a + // removed peer can still be mapped to its StableNodeID after the peer is + // gone from the netmap. Entries are added on the first non-zero transition + // for a key and removed alongside the corresponding peerWGState entry. + peerWGStableIDByKey map[key.NodePublic]tailcfg.StableNodeID // capForcedNetfilter is the netfilter that control instructs Linux clients // to use, unless overridden locally. @@ -628,6 +644,7 @@ func NewLocalBackend(logf logger.Logf, logID logid.PublicID, sys *tsd.System, lo nb.ready() e.SetPeerByIPPacketFunc(b.lookupPeerByIP) + e.SetPeerSessionStateFunc(b.onPeerWireGuardState) if sys.InitialConfig != nil { if err := b.initPrefsFromConfig(sys.InitialConfig); err != nil { @@ -1307,6 +1324,7 @@ func (b *LocalBackend) Shutdown() { // 2. Event handlers may not guard against undesirable post/in-progress // LocalBackend.Shutdown() behaviors. b.appcTask.Shutdown() + b.peerWGStateQueue.Shutdown() b.eventClient.Close() b.em.close() @@ -3704,6 +3722,9 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A statusSB = &ipnstate.StatusBuilder{WantPeers: true} b.e.UpdateStatus(statusSB) } + if mask&ipn.NotifyPeerWireGuardState != 0 { + _ = b.peerWGStateQueue.Wait(ctx) + } // Watch for deadlocks only during the registration phase below; the rest // of this method blocks on ctx (often for hours) and shouldn't trip the @@ -3711,7 +3732,7 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A deadlockDone := b.CheckDeadlocks() b.mu.Lock() - const initialBits = ipn.NotifyInitialState | ipn.NotifyInitialPrefs | ipn.NotifyInitialNetMap | ipn.NotifyInitialStatus | ipn.NotifyInitialDriveShares | ipn.NotifyInitialSuggestedExitNode | ipn.NotifyInitialClientVersion + const initialBits = ipn.NotifyInitialState | ipn.NotifyInitialPrefs | ipn.NotifyInitialNetMap | ipn.NotifyInitialStatus | ipn.NotifyInitialDriveShares | ipn.NotifyInitialSuggestedExitNode | ipn.NotifyInitialClientVersion | ipn.NotifyPeerWireGuardState if mask&initialBits != 0 { cn := b.currentNode() ini = &ipn.Notify{Version: version.Long()} @@ -3769,6 +3790,9 @@ func (b *LocalBackend) WatchNotificationsAs(ctx context.Context, actor ipnauth.A mask: mask, } mak.Set(&b.notifyWatchers, sessionID, session) + if mask&ipn.NotifyPeerWireGuardState != 0 { + ini.PeerState = maps.Clone(b.peerWGState) + } b.mu.Unlock() deadlockDone() @@ -4073,6 +4097,7 @@ func (b *LocalBackend) notifyForSessionLocked(sess *watchSession, n *ipn.Notify) stripPeersChanged := len(n.PeersChanged) > 0 && !wantsPeerChanges stripPeersRemoved := len(n.PeersRemoved) > 0 && !wantsPeerChanges stripPatches := len(n.PeerChangedPatch) > 0 && !wantsPeerPatches + stripPeerState := len(n.PeerState) > 0 && sess.mask&ipn.NotifyPeerWireGuardState == 0 promotePatches := len(n.PeerChangedPatch) > 0 && wantsPeerChanges && !wantsPeerPatches if sess.selfChangeResetsImplicitState(n.SelfChange.View()) { @@ -4103,7 +4128,7 @@ func (b *LocalBackend) notifyForSessionLocked(sess *watchSession, n *ipn.Notify) } replaceUserProfiles := !stripUserProfiles && len(sessUserProfiles) != len(n.UserProfiles) - if !stripNetMap && !stripPeersChanged && !stripPeersRemoved && !stripPatches && !stripUserProfiles && !replaceUserProfiles && !promotePatches { + if !stripNetMap && !stripPeersChanged && !stripPeersRemoved && !stripPatches && !stripPeerState && !stripUserProfiles && !replaceUserProfiles && !promotePatches { return n } nCopy := *n @@ -4132,6 +4157,9 @@ func (b *LocalBackend) notifyForSessionLocked(sess *watchSession, n *ipn.Notify) if stripPatches { nCopy.PeerChangedPatch = nil } + if stripPeerState { + nCopy.PeerState = nil + } if stripUserProfiles { nCopy.UserProfiles = nil } else if replaceUserProfiles { @@ -5761,6 +5789,98 @@ func (b *LocalBackend) lookupPeerByIP(ip netip.Addr) (key.NodePublic, bool) { return peer.Key(), true } +// onPeerWireGuardState is called by wireguard-go, through wgengine, for +// serialized WireGuard session state transitions. wireguard-go is holding locks +// while calling this, so this must stay cheap, must not acquire b.mu, and must +// not call back into wireguard-go. Acquiring b.mu here can deadlock with +// LocalBackend operations that hold b.mu while waiting for wireguard-go to make +// progress. +func (b *LocalBackend) onPeerWireGuardState(peerKey key.NodePublic, state wgengine.PeerWireGuardState) { + st := peerWireGuardStateFromEngine(state) + // 10ms granularity is plenty for diagnostics and keeps the JSON form short. + at := b.clock.Now().Round(10 * time.Millisecond) + b.peerWGStateQueue.Add(func() { + b.handlePeerWireGuardState(peerKey, st, at) + }) +} + +func (b *LocalBackend) handlePeerWireGuardState(peerKey key.NodePublic, st ipn.PeerWireGuardState, at time.Time) { + b.mu.Lock() + defer b.mu.Unlock() + + id, ok := b.peerWGStableIDByKey[peerKey] + if !ok { + nb := b.currentNode() + nid, ok := nb.NodeByKey(peerKey) + if !ok { + b.logf("[unexpected] WireGuard session state for unknown peer %v", peerKey.ShortString()) + return + } + peer, ok := nb.NodeByID(nid) + if !ok { + b.logf("[unexpected] WireGuard session state for unknown node %v", nid) + return + } + id = peer.StableID() + } + + if st == ipn.PeerWireGuardStateNone { + old, ok := b.peerWGState[id] + if !ok { + return + } + delete(b.peerWGState, id) + delete(b.peerWGStableIDByKey, peerKey) + addPeerWGStateMetric(old.PeerWireGuardState, -1) + } else { + old := b.peerWGState[id].PeerWireGuardState + if old == st { + return + } + if old != ipn.PeerWireGuardStateNone { + addPeerWGStateMetric(old, -1) + } + mak.Set(&b.peerWGState, id, ipn.PeerState{ + PeerWireGuardState: st, + PeerWireGuardStateAt: at, + }) + mak.Set(&b.peerWGStableIDByKey, peerKey, id) + addPeerWGStateMetric(st, 1) + } + b.sendToLocked(ipn.Notify{ + PeerState: map[tailcfg.StableNodeID]ipn.PeerState{id: { + PeerWireGuardState: st, + PeerWireGuardStateAt: at, + }}, + }, allClients) +} + +func addPeerWGStateMetric(state ipn.PeerWireGuardState, delta int64) { + switch state { + case ipn.PeerWireGuardStateHandshake: + metricPeerWGStateHandshake.Add(delta) + case ipn.PeerWireGuardStateEstablished: + metricPeerWGStateEstablished.Add(delta) + case ipn.PeerWireGuardStateExpired: + metricPeerWGStateExpired.Add(delta) + } +} + +func peerWireGuardStateFromEngine(state wgengine.PeerWireGuardState) ipn.PeerWireGuardState { + switch state { + case wgengine.PeerWireGuardStateNone: + return ipn.PeerWireGuardStateNone + case wgengine.PeerWireGuardStateHandshake: + return ipn.PeerWireGuardStateHandshake + case wgengine.PeerWireGuardStateEstablished: + return ipn.PeerWireGuardStateEstablished + case wgengine.PeerWireGuardStateExpired: + return ipn.PeerWireGuardStateExpired + default: + panic(fmt.Sprintf("unexpected wgengine.PeerWireGuardState %d", state)) + } +} + func (b *LocalBackend) isEngineBlocked() bool { b.mu.Lock() defer b.mu.Unlock() @@ -8950,6 +9070,9 @@ func maybeUsernameOf(actor ipnauth.Actor) string { var ( metricCurrentWatchIPNBus = clientmetric.NewGauge("localbackend_current_watch_ipn_bus") metricIPForwardingCheckError = clientmetric.NewCounter("localbackend_ip_forwarding_check_error") + metricPeerWGStateHandshake = clientmetric.NewGauge("localbackend_peer_wireguard_state_handshake") + metricPeerWGStateEstablished = clientmetric.NewGauge("localbackend_peer_wireguard_state_established") + metricPeerWGStateExpired = clientmetric.NewGauge("localbackend_peer_wireguard_state_expired") // Counters for the controlclient's delta-update fast path: each // counts a destination-side call into [LocalBackend] from diff --git a/ipn/ipnlocal/local_test.go b/ipn/ipnlocal/local_test.go index 2c5a39dd3..b0d24cf7a 100644 --- a/ipn/ipnlocal/local_test.go +++ b/ipn/ipnlocal/local_test.go @@ -2357,6 +2357,210 @@ func TestNotifyForSessionPeerVisibility(t *testing.T) { }) } +func TestNotifyForSessionPeerWireGuardStateVisibility(t *testing.T) { + b := newTestLocalBackend(t) + notify := ipn.Notify{ + PeerState: map[tailcfg.StableNodeID]ipn.PeerState{ + "stable1": {PeerWireGuardState: ipn.PeerWireGuardStateEstablished}, + }, + } + + deliver := func(mask ipn.NotifyWatchOpt) *ipn.Notify { + sess := &watchSession{mask: mask} + b.mu.Lock() + defer b.mu.Unlock() + return b.notifyForSessionLocked(sess, ¬ify) + } + + if n := deliver(0); len(n.PeerState) != 0 { + t.Fatalf("PeerState without watch bit = %v; want empty", n.PeerState) + } + if n := deliver(ipn.NotifyPeerWireGuardState); n.PeerState["stable1"].PeerWireGuardState != ipn.PeerWireGuardStateEstablished { + t.Fatalf("PeerState with watch bit = %v; want stable1=established", n.PeerState) + } +} + +func TestPeerWireGuardStateValuesMatchWGEngine(t *testing.T) { + const unknownPeerWireGuardState wgengine.PeerWireGuardState = 255 + + tests := []struct { + name string + in wgengine.PeerWireGuardState + want ipn.PeerWireGuardState + }{ + {"none", wgengine.PeerWireGuardStateNone, ipn.PeerWireGuardStateNone}, + {"handshake", wgengine.PeerWireGuardStateHandshake, ipn.PeerWireGuardStateHandshake}, + {"established", wgengine.PeerWireGuardStateEstablished, ipn.PeerWireGuardStateEstablished}, + {"expired", wgengine.PeerWireGuardStateExpired, ipn.PeerWireGuardStateExpired}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := peerWireGuardStateFromEngine(tt.in); got != tt.want { + t.Fatalf("converted state = %v; want %v", got, tt.want) + } + if got, want := uint8(tt.in), uint8(tt.want); got != want { + t.Fatalf("wgengine const = %v; want %v", got, want) + } + }) + } + + defer func() { + if recover() == nil { + t.Fatal("expected panic for unknown wgengine state") + } + }() + _ = peerWireGuardStateFromEngine(unknownPeerWireGuardState) +} + +func TestPeerWireGuardStateWatchInitialThenDeltas(t *testing.T) { + b := newTestLocalBackend(t) + + peer1 := &tailcfg.Node{ + ID: 1, + StableID: "stable1", + Key: makeNodeKeyFromID(1), + } + peer2 := &tailcfg.Node{ + ID: 2, + StableID: "stable2", + Key: makeNodeKeyFromID(2), + } + b.currentNode().SetNetMap(&netmap.NetworkMap{ + Peers: []tailcfg.NodeView{peer1.View(), peer2.View()}, + }) + + t1 := time.Unix(1700000000, 0) + b.handlePeerWireGuardState(peer1.Key, ipn.PeerWireGuardStateEstablished, t1) + b.handlePeerWireGuardState(peer1.Key, ipn.PeerWireGuardStateEstablished, t1.Add(time.Second)) + defer b.handlePeerWireGuardState(peer1.Key, ipn.PeerWireGuardStateNone, time.Now()) + defer b.handlePeerWireGuardState(peer2.Key, ipn.PeerWireGuardStateNone, time.Now()) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + gotCh := make(chan *ipn.Notify, 2) + doneCh := make(chan struct{}) + go func() { + defer close(doneCh) + var got int + b.WatchNotificationsAs(ctx, nil, ipn.NotifyPeerWireGuardState, func() { + b.onPeerWireGuardState(peer2.Key, wgengine.PeerWireGuardStateHandshake) + }, func(n *ipn.Notify) bool { + gotCh <- n + got++ + if got == 2 { + cancel() + return false + } + return true + }) + }() + + var got []*ipn.Notify + for len(got) < 2 { + select { + case n := <-gotCh: + got = append(got, n) + case <-time.After(5 * time.Second): + t.Fatalf("timed out waiting for notifications; got %v", got) + } + } + <-doneCh + + if got[0].PeerState["stable1"].PeerWireGuardState != ipn.PeerWireGuardStateEstablished { + t.Fatalf("initial PeerState = %v; want stable1=established", got[0].PeerState) + } + if got, want := got[0].PeerState["stable1"].PeerWireGuardStateAt, t1; !got.Equal(want) { + t.Fatalf("initial PeerState[stable1].PeerWireGuardStateAt = %v; want %v (no-op transition preserves first-entry time)", got, want) + } + if _, ok := got[0].PeerState["stable2"]; ok { + t.Fatalf("initial PeerState includes post-registration delta: %v", got[0].PeerState) + } + if len(got[1].PeerState) != 1 || got[1].PeerState["stable2"].PeerWireGuardState != ipn.PeerWireGuardStateHandshake { + t.Fatalf("delta PeerState = %v; want stable2=handshake only", got[1].PeerState) + } + if got[1].PeerState["stable2"].PeerWireGuardStateAt.IsZero() { + t.Fatalf("delta PeerState[stable2].PeerWireGuardStateAt is zero; want set by LocalBackend clock") + } +} + +func TestPeerWireGuardStateMetrics(t *testing.T) { + b := newTestLocalBackend(t) + peer := &tailcfg.Node{ + ID: 1, + StableID: "stable1", + Key: makeNodeKeyFromID(1), + } + b.currentNode().SetNetMap(&netmap.NetworkMap{ + Peers: []tailcfg.NodeView{peer.View()}, + }) + now := time.Now() + defer b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateNone, now) + + baseHandshake := metricPeerWGStateHandshake.Value() + baseEstablished := metricPeerWGStateEstablished.Value() + baseExpired := metricPeerWGStateExpired.Value() + + b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateEstablished, now) + if got, want := metricPeerWGStateEstablished.Value(), baseEstablished+1; got != want { + t.Fatalf("established gauge = %v; want %v", got, want) + } + b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateEstablished, now) + if got, want := metricPeerWGStateEstablished.Value(), baseEstablished+1; got != want { + t.Fatalf("established gauge after no-op = %v; want %v", got, want) + } + + b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateHandshake, now) + if got, want := metricPeerWGStateEstablished.Value(), baseEstablished; got != want { + t.Fatalf("established gauge after transition = %v; want %v", got, want) + } + if got, want := metricPeerWGStateHandshake.Value(), baseHandshake+1; got != want { + t.Fatalf("handshake gauge = %v; want %v", got, want) + } + + b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateExpired, now) + if got, want := metricPeerWGStateHandshake.Value(), baseHandshake; got != want { + t.Fatalf("handshake gauge after transition = %v; want %v", got, want) + } + if got, want := metricPeerWGStateExpired.Value(), baseExpired+1; got != want { + t.Fatalf("expired gauge = %v; want %v", got, want) + } + + b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateNone, now) + if got, want := metricPeerWGStateExpired.Value(), baseExpired; got != want { + t.Fatalf("expired gauge after none = %v; want %v", got, want) + } +} + +func TestPeerWireGuardStateCallbackDoesNotBlockOnLocalBackendMu(t *testing.T) { + b := newTestLocalBackend(t) + peer := &tailcfg.Node{ + ID: 1, + StableID: "stable1", + Key: makeNodeKeyFromID(1), + } + b.currentNode().SetNetMap(&netmap.NetworkMap{ + Peers: []tailcfg.NodeView{peer.View()}, + }) + + b.mu.Lock() + done := make(chan struct{}) + go func() { + b.onPeerWireGuardState(peer.Key, wgengine.PeerWireGuardStateEstablished) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + b.mu.Unlock() + t.Fatal("onPeerWireGuardState blocked on LocalBackend.mu") + } + b.mu.Unlock() + + _ = b.peerWGStateQueue.Wait(context.Background()) + b.handlePeerWireGuardState(peer.Key, ipn.PeerWireGuardStateNone, time.Now()) +} + func TestSetControlClientStatusSendsFullNetmapAsPeerChanges(t *testing.T) { b := newTestLocalBackend(t) nw := newNotificationWatcher(t, b, ipnauth.Self) diff --git a/ipn/ipnlocal/state_test.go b/ipn/ipnlocal/state_test.go index 4eb8ac136..8190aee03 100644 --- a/ipn/ipnlocal/state_test.go +++ b/ipn/ipnlocal/state_test.go @@ -1982,6 +1982,8 @@ func (e *mockEngine) Ping(ip netip.Addr, pingType tailcfg.PingType, size int, cb func (e *mockEngine) InstallCaptureHook(packet.CaptureCallback) {} func (e *mockEngine) SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool)) {} +func (e *mockEngine) SetPeerSessionStateFunc(func(key.NodePublic, wgengine.PeerWireGuardState)) { +} func (e *mockEngine) Close() { e.mu.Lock() diff --git a/wgengine/userspace.go b/wgengine/userspace.go index 58a09a50f..d10c7464b 100644 --- a/wgengine/userspace.go +++ b/wgengine/userspace.go @@ -21,6 +21,7 @@ import ( "github.com/gaissmai/bart" "github.com/tailscale/wireguard-go/device" "github.com/tailscale/wireguard-go/tun" + "go4.org/mem" "tailscale.com/control/controlknobs" "tailscale.com/drive" "tailscale.com/envknob" @@ -726,6 +727,29 @@ func (e *userspaceEngine) SetPeerByIPPacketFunc(fn func(netip.Addr) (_ key.NodeP }) } +func (e *userspaceEngine) SetPeerSessionStateFunc(fn func(key.NodePublic, PeerWireGuardState)) { + e.wgdev.SetSessionStateFunc(func(pk device.NoisePublicKey, state device.PeerSessionState) { + if fn != nil { + fn(key.NodePublicFromRaw32(mem.B(pk[:])), peerWireGuardStateFromDevice(state)) + } + }) +} + +func peerWireGuardStateFromDevice(state device.PeerSessionState) PeerWireGuardState { + switch state { + case device.PeerSessionNone: + return PeerWireGuardStateNone + case device.PeerSessionHandshake: + return PeerWireGuardStateHandshake + case device.PeerSessionEstablished: + return PeerWireGuardStateEstablished + case device.PeerSessionExpired: + return PeerWireGuardStateExpired + default: + panic(fmt.Sprintf("unexpected wireguard-go PeerSessionState %d", state)) + } +} + // hasOverlap checks if there is a IPPrefix which is common amongst the two // provided slices. func hasOverlap(aips, rips views.Slice[netip.Prefix]) bool { diff --git a/wgengine/userspace_test.go b/wgengine/userspace_test.go index b2f40fada..4cadbcb69 100644 --- a/wgengine/userspace_test.go +++ b/wgengine/userspace_test.go @@ -13,6 +13,7 @@ import ( "sync" "testing" + "github.com/tailscale/wireguard-go/device" "go4.org/mem" "tailscale.com/cmd/testwrapper/flakytest" "tailscale.com/control/controlknobs" @@ -35,6 +36,38 @@ import ( "tailscale.com/wgengine/wgcfg" ) +func TestPeerWireGuardStateValuesMatchWireguardGo(t *testing.T) { + const unknownPeerSessionState device.PeerSessionState = 255 + + tests := []struct { + name string + wg device.PeerSessionState + want PeerWireGuardState + }{ + {"none", device.PeerSessionNone, PeerWireGuardStateNone}, + {"handshake", device.PeerSessionHandshake, PeerWireGuardStateHandshake}, + {"established", device.PeerSessionEstablished, PeerWireGuardStateEstablished}, + {"expired", device.PeerSessionExpired, PeerWireGuardStateExpired}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := peerWireGuardStateFromDevice(tt.wg); got != tt.want { + t.Fatalf("converted state = %v; want %v", got, tt.want) + } + if got, want := uint8(tt.wg), uint8(tt.want); got != want { + t.Fatalf("wireguard-go const = %v; want %v", got, want) + } + }) + } + + defer func() { + if recover() == nil { + t.Fatal("expected panic for unknown wireguard-go state") + } + }() + _ = peerWireGuardStateFromDevice(unknownPeerSessionState) +} + func nodeViews(v []*tailcfg.Node) []tailcfg.NodeView { nv := make([]tailcfg.NodeView, len(v)) for i, n := range v { diff --git a/wgengine/wgengine.go b/wgengine/wgengine.go index 5ca4b75cf..a2504f84c 100644 --- a/wgengine/wgengine.go +++ b/wgengine/wgengine.go @@ -41,6 +41,28 @@ type StatusCallback func(*Status, error) // into network map updates. type NetworkMapCallback func(*netmap.NetworkMap) +// PeerWireGuardState is the current WireGuard session state for a peer. +type PeerWireGuardState uint8 + +const ( + // PeerWireGuardStateNone means there is no handshake in progress and no + // session key material retained for this peer. + PeerWireGuardStateNone PeerWireGuardState = 0 + + // PeerWireGuardStateHandshake means a handshake is in progress for this + // peer, but there is not currently a usable WireGuard session. + PeerWireGuardStateHandshake PeerWireGuardState = 1 + + // PeerWireGuardStateEstablished means the peer has a completed WireGuard + // session with usable session key material. + PeerWireGuardStateEstablished PeerWireGuardState = 2 + + // PeerWireGuardStateExpired means the peer's session key material is no + // longer considered usable, but final key cleanup or lazy peer removal may + // not have happened yet. + PeerWireGuardStateExpired PeerWireGuardState = 3 +) + // ErrNoChanges is returned by Engine.Reconfig if no changes were made. var ErrNoChanges = errors.New("no changes made to Engine config") @@ -141,4 +163,20 @@ type Engine interface { // SetPeerByIPPacketFunc installs a callback used by wireguard-go to // look up which peer should handle an outbound packet by destination IP. SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool)) + + // SetPeerSessionStateFunc installs a callback used to observe WireGuard + // peer session state transitions. + // + // Calls are serialized per Engine and delivered in transition order from + // wireguard-go, while wireguard-go is holding locks. The callback must be + // cheap and must not call back into wireguard-go. + // + // It does not replay current state. Callers that need a complete view should + // set it before peers are started or lazily created, and maintain any + // snapshots, sequence numbers, and pubsub state outside wireguard-go. + // + // In Tailscale, the usual implementation is + // ipnlocal.LocalBackend.onPeerWireGuardState, installed early in + // LocalBackend construction. + SetPeerSessionStateFunc(func(key.NodePublic, PeerWireGuardState)) }