wgengine,ipn/ipnlocal: sync wireguard-go peers incrementally on netmap deltas

Previously, any peer added or removed by an incremental netmap delta
was only visible to wireguard-go after a full authReconfig: wgcfg's
ReconfigDevice re-installed a PeerLookupFunc closing over a freshly
built map of every peer's allowed IPs, doing O(n) work per change.

Instead, install the wireguard-go device hooks once, backed by live
state. Engine.SetPeerConfigFunc installs a single long-lived
PeerLookupFunc that queries LocalBackend's per-node RouteManager on
demand, and Engine.SyncDevicePeer does O(1) per-peer device sync
(remove, or update allowed IPs) as each delta mutation is applied.
Full reconfigs keep an O(n peers) device sync for now, but with no
lookup closure to reinstall and no removed-peer resurrection race; a
later change removes full-config peer syncing entirely.

The RouteManager's PeerAllowedIPs accessor backs the new hooks: its
sorted output makes unchanged state a no-op update, and its peer
filtering mirrors nmcfg.WGCfg, so expired peers and peers predating
both DERP and disco contribute no prefixes and thus cannot be lazily
created in the device, which matters because wireguard-go validates
inbound source IPs against per-peer allowed IPs.

The engine's SetPeerByIPPacketFunc callback is now authoritative when
installed, since LocalBackend's implementation covers subnet routes
and exit-node routes via the RouteManager's outbound table; the
engine's own reconfig-time BART table only serves engines running
without a LocalBackend.

The forced authReconfig on peer add/remove stays for now: the
WireGuard device no longer needs it, but OS routes, the quad-100
resolver's MagicDNS hosts map, and tstun's masquerade/jailed peer
config are still derived from the full peer set. Making those
delta-aware is the next step before gating it.

Updates #12542

Change-Id: I3ba8c7c324bca0ad0269279d03f53b1f17fb63a2
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

wgengine/userspace.go

@@ -122,6 +122,12 @@ type userspaceEngine struct {
 	// for the cold-path control lookups (Ping, TSMP, pendopen, etc).
 	peerForIP atomic.Pointer[func(netip.Addr) (_ PeerForIP, ok bool)]
 
+	// peerConfigFn, if non-nil, is the live per-peer allowed-IPs
+	// source installed via [userspaceEngine.SetPeerConfigFunc]. When
+	// set, wgdev's PeerLookupFunc queries it directly, so reconfigs
+	// no longer install per-config lookup closures.
+	peerConfigFn atomic.Pointer[func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool)]
+
 	lastCfgFull        wgcfg.Config
 	lastRouter         *router.Config
 	lastDNSConfig      dns.ConfigView // or invalid if none
@@ -726,20 +732,84 @@ func (e *userspaceEngine) maybeReconfigWireguardLocked() error {
 	e.peerByIPRoute.Store(rt)
 
 	e.logf("wgengine: Reconfig: configuring userspace WireGuard config (with %d peers)", len(full.Peers))
-	if err := wgcfg.ReconfigDevice(e.wgdev, &full, e.logf); err != nil {
+	if e.peerConfigFn.Load() != nil {
+		// The device has a long-lived PeerLookupFunc backed by the
+		// live config source, so only the peer set needs syncing;
+		// there is no per-config lookup closure to (re)install, and
+		// no removed peer can be resurrected with stale state.
+		//
+		// TODO(bradfitz): remove this O(n peers) sync. It's redundant
+		// with the incremental SyncDevicePeer calls that LocalBackend
+		// makes for exactly the peers whose allowed IPs changed. It
+		// only remains because peer changes still force a full
+		// Reconfig; once that's gated on actual router/DNS changes,
+		// this sync (and full-config peer syncing generally) can go.
+		peers := make(map[device.NoisePublicKey][]netip.Prefix, len(full.Peers))
+		for _, p := range full.Peers {
+			peers[p.PublicKey.Raw32()] = p.AllowedIPs
+		}
+		e.wgdev.RemoveMatchingPeers(func(pk device.NoisePublicKey) bool {
+			_, exists := peers[pk]
+			return !exists
+		})
+		// Update AllowedIPs on any already-active peers whose config
+		// may have changed. Peers that don't exist yet will get the
+		// correct AllowedIPs from the device's PeerLookupFunc when
+		// they are lazily created.
+		for pk, allowedIPs := range peers {
+			if peer, ok := e.wgdev.LookupActivePeer(pk); ok {
+				peer.SetAllowedIPs(allowedIPs)
+			}
+		}
+	} else if err := wgcfg.ReconfigDevice(e.wgdev, &full, e.logf); err != nil {
 		e.logf("wgdev.Reconfig: %v", err)
 		return err
 	}
 	return nil
 }
 
+// SetPeerConfigFunc implements [Engine.SetPeerConfigFunc]. It stores
+// fn and installs a single wgdev PeerLookupFunc wrapping it, so
+// lazily-created peers always get current allowed IPs and the lookup
+// func never needs to be reinstalled as the peer set changes.
+func (e *userspaceEngine) SetPeerConfigFunc(fn func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool)) {
+	if fn == nil {
+		panic("SetPeerConfigFunc: nil fn")
+	}
+	e.peerConfigFn.Store(&fn)
+	e.wgdev.SetPeerLookupFunc(wgcfg.NewPeerLookupFunc(e.wgdev.Bind(), e.logf, func(pubk device.NoisePublicKey) ([]netip.Prefix, bool) {
+		return fn(key.NodePublicFromRaw32(mem.B(pubk[:])))
+	}))
+}
+
+// SyncDevicePeer implements [Engine.SyncDevicePeer].
+func (e *userspaceEngine) SyncDevicePeer(k key.NodePublic) {
+	fn := e.peerConfigFn.Load()
+	if fn == nil {
+		return
+	}
+	e.wgLock.Lock()
+	defer e.wgLock.Unlock()
+	allowedIPs, ok := (*fn)(k)
+	if !ok {
+		e.wgdev.RemovePeer(k.Raw32())
+		return
+	}
+	if peer, ok := e.wgdev.LookupActivePeer(k.Raw32()); ok {
+		peer.SetAllowedIPs(allowedIPs)
+	}
+}
+
 // SetPeerByIPPacketFunc installs a callback used by wireguard-go to look up
 // which peer should handle an outbound packet by destination IP.
 //
-// fn is an optional fast path for exact node-address matches (e.g. dst is a
-// Tailscale IP). On miss (or if fn is nil), the engine's own BART table
-// ([userspaceEngine.peerByIPRoute], built from the wireguard-filtered peer
-// list) is consulted to handle subnet routes and exit-node default routes.
+// If fn is non-nil it is authoritative: LocalBackend's implementation
+// consults both the exact node-address fast path and the RouteManager's
+// outbound table (covering subnet routes and exit-node default routes),
+// and stays correct under incremental netmap deltas. The engine's own
+// BART table ([userspaceEngine.peerByIPRoute], rebuilt only on full
+// reconfigs) is used only when no fn is installed (e.g. engines running
+// without a LocalBackend).
 //
 // [NewUserspaceEngine] installs a BART-only default at engine creation time,
 // so callers that don't call SetPeerByIPPacketFunc (e.g. those not running
@@ -750,6 +820,7 @@ func (e *userspaceEngine) SetPeerByIPPacketFunc(fn func(netip.Addr) (_ key.NodeP
 			if pk, ok := fn(dst); ok {
 				return pk.Raw32(), true
 			}
+			return device.NoisePublicKey{}, false
 		}
 		if rt := e.peerByIPRoute.Load(); rt != nil {
 			if pk, ok := rt.Lookup(dst); ok {

wgengine/wgcfg/device.go

@@ -18,6 +18,27 @@ func NewDevice(tunDev tun.Device, bind conn.Bind, logger *device.Logger) *device
 	return device.NewDevice(tunDev, bind, logger)
 }
 
+// NewPeerLookupFunc returns a [device.PeerLookupFunc] that lazily
+// creates peers using allowedIPs as the source of each peer's allowed
+// IPs. The peer's endpoint is derived from its public key via bind.
+func NewPeerLookupFunc(bind conn.Bind, logf logger.Logf, allowedIPs func(device.NoisePublicKey) ([]netip.Prefix, bool)) device.PeerLookupFunc {
+	return func(pubk device.NoisePublicKey) (_ *device.NewPeerConfig, ok bool) {
+		ips, ok := allowedIPs(pubk)
+		if !ok {
+			return nil, false
+		}
+		ep, err := bind.ParseEndpoint(fmt.Sprintf("%x", pubk[:]))
+		if err != nil {
+			logf("wgcfg: failed to parse endpoint for peer %x: %v", pubk[:8], err)
+			return nil, false
+		}
+		return &device.NewPeerConfig{
+			AllowedIPs: ips,
+			Endpoint:   ep,
+		}, true
+	}
+}
+
 // ReconfigDevice replaces the existing device configuration with cfg.
 //
 // Instead of using the UAPI text protocol, it uses the wireguard-go direct API

wgengine/wgengine.go

@@ -197,6 +197,33 @@ type Engine interface {
 	// look up which peer should handle an outbound packet by destination IP.
 	SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool))
 
+	// SetPeerConfigFunc installs the live source of per-peer WireGuard
+	// configuration: given a peer's public key, fn returns the prefixes
+	// the peer is currently allowed to originate traffic from, or
+	// ok=false if the peer is unknown (in which case it must not exist
+	// in the WireGuard device). The engine installs a single
+	// [device.PeerLookupFunc] wrapping fn, so lazily-created peers
+	// always see current state and the lookup func never needs to be
+	// reinstalled as peers come and go.
+	//
+	// It is expected to be called once during LocalBackend construction,
+	// before the first [Engine.Reconfig]. fn is called rarely (when
+	// wireguard-go first hears from a peer it doesn't have) and may
+	// acquire locks.
+	SetPeerConfigFunc(fn func(key.NodePublic) (allowedIPs []netip.Prefix, ok bool))
+
+	// SyncDevicePeer synchronizes the WireGuard device's state for a
+	// single peer with the config source installed via
+	// [Engine.SetPeerConfigFunc]: if the source no longer knows the
+	// peer, it is removed from the device; if the peer is active in the
+	// device, its allowed IPs are updated. It does O(1) work (plus the
+	// config source lookup) and is intended to be called for each peer
+	// added, updated, or removed by an incremental netmap delta,
+	// avoiding a full [Engine.Reconfig].
+	//
+	// It is a no-op if no config source is installed.
+	SyncDevicePeer(key.NodePublic)
+
 	// SetNetLogSource installs the [NetLogSource] consulted by the
 	// engine's network flow logger for node lookups and the current
 	// audit logging identity.