ipn/ipnlocal, control/controlclient: process node adds/removes in constant time

For large tailnets (~50k+ nodes) with frequent peer churn (ephemeral GitHub Actions workers etc.), tailscaled used to rebuild the full netmap and fan it out on the IPN bus on every MapResponse that added or removed a peer. There were two O(N) costs per delta: the full netmap rebuild + every Notify.NetMap encode to every bus watcher. This change tackles both: 1. Plumb O(1) peer add/remove through the delta path. PeersChanged and PeersRemoved no longer prevent the delta happy path; instead, they mutate the per-node-backend peer map in place. 2. Restrict ipn.Notify.NetMap emission to the platforms whose host GUIs still depend on it (Windows, macOS, iOS) and migrate in-tree consumers off it everywhere else: - Migrate reactive consumers (containerboot, kube agents, sniproxy, tsconsensus, etc.) off Notify.NetMap to the previously-added Notify.SelfChange signal so they no longer have to subscribe to the full netmap. - Add ipn.NotifyNoNetMap so GUI clients on "legacy-emit" platforms that have already migrated can opt out of the per-watcher NetMap encode. - Gate Notify.NetMap emission on the producer side by a compile- time GOOS check, so the supporting code is dead-code-eliminated on Linux and other geese where no GUI consumer needs it. Re-running BenchmarkGiantTailnet from tstest/largetailnet, which was added along with baseline numbers on unmodified main in ad5436af0d, the per-delta cost (one peer add+remove pair) is now ~O(1) regardless of tailnet size N: N no-watcher (ms/op) bus-watcher (ms/op) before now factor before now factor 10000 32 0.11 300x 166 0.13 1300x 50000 222 0.11 2000x 865 0.13 6700x 100000 504 0.12 4100x 1765 0.13 13400x 250000 1551 0.12 12500x 4696 0.15 32400x Updates #12542 Change-Id: I94e34b37331d1a8ec74c299deffadf4d061fda9e Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
2026-05-21 09:26:19 -07:00
parent 2703f91174
commit aa5da2e5f2
23 changed files with 1521 additions and 211 deletions
@@ -34,7 +34,6 @@ import (
 	"github.com/miekg/dns"
 	"go4.org/mem"
 	"tailscale.com/client/local"
-	"tailscale.com/client/tailscale"
 	"tailscale.com/cmd/testwrapper/flakytest"
 	"tailscale.com/feature"
 	_ "tailscale.com/feature/clientupdate"
@@ -69,6 +68,14 @@ func TestMain(m *testing.M) {
 	os.Exit(0)
 }

+// fetchNetMapForTest fetches the current netmap from tailscaled via the
+// "current-netmap" debug action. The debug action's payload shape is
+// intentionally not part of any stable API; tests use it to inspect
+// internal state.
+func fetchNetMapForTest(ctx context.Context, lc *local.Client) (*netmap.NetworkMap, error) {
+	return local.GetDebugResultJSON[*netmap.NetworkMap](ctx, lc, "current-netmap")
+}
+
 // Tests that tailscaled starts up in TUN mode, and also without data races:
 // https://github.com/tailscale/tailscale/issues/7894
 func TestTUNMode(t *testing.T) {
@@ -1189,20 +1196,18 @@ func TestClientSideJailing(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	waitPeerIsJailed := func(t *testing.T, b *tailscale.IPNBusWatcher, jailed bool) {
+	waitPeerIsJailed := func(t *testing.T, b *local.IPNBusWatcher, lc *local.Client, jailed bool) {
 		t.Helper()
 		for {
-			n, err := b.Next()
+			_, err := b.Next()
 			if err != nil {
 				t.Fatal(err)
 			}
-			if n.NetMap == nil {
+			nm, err := fetchNetMapForTest(context.Background(), lc)
+			if err != nil || nm == nil || len(nm.Peers) == 0 {
 				continue
 			}
-			if len(n.NetMap.Peers) == 0 {
-				continue
-			}
-			if j := n.NetMap.Peers[0].IsJailed(); j == jailed {
+			if j := nm.Peers[0].IsJailed(); j == jailed {
 				break
 			}
 		}
@@ -1213,8 +1218,8 @@ func TestClientSideJailing(t *testing.T) {
 			env.Control.SetJailed(k2, k1, tc.n1JailedForN2)

 			// Wait for the jailed status to propagate.
-			waitPeerIsJailed(t, b1, tc.n2JailedForN1)
-			waitPeerIsJailed(t, b2, tc.n1JailedForN2)
+			waitPeerIsJailed(t, b1, lc1, tc.n2JailedForN1)
+			waitPeerIsJailed(t, b2, lc2, tc.n1JailedForN2)

 			testDial(t, lc1, ip2, port, tc.n1JailedForN2)
 			testDial(t, lc2, ip1, port, tc.n2JailedForN1)
@@ -0,0 +1,279 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package largetailnet_test
+
+import (
+	"context"
+	"fmt"
+	"net/http/httptest"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"tailscale.com/ipn/store/mem"
+	"tailscale.com/tailcfg"
+	"tailscale.com/tsnet"
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/integration"
+	"tailscale.com/tstest/integration/testcontrol"
+	"tailscale.com/tstest/largetailnet"
+	"tailscale.com/types/ipproto"
+	"tailscale.com/types/logger"
+	"tailscale.com/util/clientmetric"
+	"tailscale.com/wgengine/filter"
+)
+
+// metricByName returns the [clientmetric.Metric] with the given name,
+// failing the test if not found.
+func metricByName(t testing.TB, name string) *clientmetric.Metric {
+	t.Helper()
+	for _, m := range clientmetric.Metrics() {
+		if m.Name() == name {
+			return m
+		}
+	}
+	t.Fatalf("metric %q not found", name)
+	return nil
+}
+
+// TestNetmapDeltaFastPath drives a sequence of MapResponses against an
+// in-process tsnet + testcontrol harness via [largetailnet.Streamer]'s
+// AltMapStream hook, exercising every delta-message kind the
+// incremental netmap path handles. After each delta it asserts both:
+//
+//   - the appropriate fast-path metric counters incremented (i.e. we
+//     stayed on the incremental path and did not fall through to a full
+//     netmap rebuild); and
+//
+//   - the corresponding side effect is observable on the [LocalBackend]
+//     (a fresh peer resolvable via PeerByID, a UserProfile resolvable
+//     via UserProfile, a packet filter rule reflected in
+//     GetFilterForTest, a per-field patch reflected in PeerByID, etc.).
+//
+// This is the destination-side companion to
+// [tstest/largetailnet/BenchmarkGiantTailnet], which only measures cost
+// of the same fast path — this test verifies correctness.
+func TestNetmapDeltaFastPath(t *testing.T) {
+	tstest.Shard(t)
+
+	logf := logger.Discard
+	if testing.Verbose() {
+		logf = t.Logf
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second)
+	t.Cleanup(cancel)
+
+	derpMap := integration.RunDERPAndSTUN(t, logf, "127.0.0.1")
+
+	// Start with one initial peer (NodeID 2) so the initial netmap is
+	// realistic. The fast path will not fire for the initial response —
+	// it always goes through UpdateFullNetmap — but every subsequent
+	// SendDelta should.
+	streamer := largetailnet.New(1, derpMap)
+	ctrl := &testcontrol.Server{
+		DERPMap:      derpMap,
+		DNSConfig:    &tailcfg.DNSConfig{},
+		AltMapStream: streamer.AltMapStream(),
+		Logf:         logf,
+	}
+	ctrl.HTTPTestServer = httptest.NewUnstartedServer(ctrl)
+	ctrl.HTTPTestServer.Start()
+	t.Cleanup(ctrl.HTTPTestServer.Close)
+
+	tmp := filepath.Join(t.TempDir(), "tsnet")
+	if err := os.MkdirAll(tmp, 0o755); err != nil {
+		t.Fatal(err)
+	}
+	s := &tsnet.Server{
+		Dir:        tmp,
+		ControlURL: ctrl.HTTPTestServer.URL,
+		Hostname:   "delta-test",
+		Store:      new(mem.Store),
+		Ephemeral:  true,
+		Logf:       logf,
+	}
+	t.Cleanup(func() { s.Close() })
+	if _, err := s.Up(ctx); err != nil {
+		t.Fatalf("tsnet.Server.Up: %v", err)
+	}
+	lb := tsnet.TestHooks.LocalBackend(s)
+
+	// Snapshot baseline metric values; we'll assert deltas against
+	// these. Globals make per-test isolation impossible, but deltas
+	// are robust against interleaving (assuming no other test runs in
+	// parallel here, hence tstest.Shard above).
+	mFast := metricByName(t, "controlclient_map_response_handled_incrementally")
+	mFull := metricByName(t, "controlclient_map_response_handled_full_rebuild")
+	mAdd := metricByName(t, "localbackend_netmap_delta_peer_added")
+	mRem := metricByName(t, "localbackend_netmap_delta_peer_removed")
+	mPatch := metricByName(t, "localbackend_netmap_delta_peer_patched")
+	mFilter := metricByName(t, "localbackend_update_packet_filter")
+	mUsers := metricByName(t, "localbackend_update_user_profiles")
+	baseline := map[*clientmetric.Metric]int64{
+		mFast: mFast.Value(), mFull: mFull.Value(),
+		mAdd: mAdd.Value(), mRem: mRem.Value(), mPatch: mPatch.Value(),
+		mFilter: mFilter.Value(), mUsers: mUsers.Value(),
+	}
+	dumpMetrics := func(t *testing.T) {
+		t.Helper()
+		for _, m := range []*clientmetric.Metric{mFast, mFull, mAdd, mRem, mPatch, mFilter, mUsers} {
+			t.Logf("metric %s = %d (baseline %d, delta %d)", m.Name(), m.Value(), baseline[m], m.Value()-baseline[m])
+		}
+	}
+	waitDelta := func(t *testing.T, m *clientmetric.Metric, want int64) {
+		t.Helper()
+		err := tstest.WaitFor(2*time.Second, func() error {
+			got := m.Value() - baseline[m]
+			if got >= want {
+				return nil
+			}
+			return fmt.Errorf("%s delta = %d, want >= %d", m.Name(), got, want)
+		})
+		if err != nil {
+			dumpMetrics(t)
+			t.Fatalf("%s: %v", m.Name(), err)
+		}
+		if got := m.Value() - baseline[m]; got != want {
+			t.Errorf("%s delta = %d, want exactly %d", m.Name(), got, want)
+		}
+		baseline[m] = m.Value()
+	}
+
+	// Helper to send a MapResponse and wait for it to be processed by
+	// the client. We use the metric deltas as our synchronization
+	// point: SendDelta is synchronous from the streamer side, but the
+	// client processes the response on its own goroutine, so we wait
+	// for the fast-path counter to tick.
+	sendDelta := func(t *testing.T, mr *tailcfg.MapResponse) {
+		t.Helper()
+		if err := streamer.SendDelta(ctx, mr); err != nil {
+			t.Fatalf("SendDelta: %v", err)
+		}
+	}
+
+	// Self IPv4, used as the destination in packet filter checks below.
+	// largetailnet derives self addresses from SelfNodeID via node4/node6;
+	// for SelfNodeID=1 that's 100.100.0.1.
+	selfIP4 := netip.MustParseAddr("100.100.0.1")
+
+	// addedPeerID is set by the peer_added_with_filter_and_user_profile
+	// subtest and consumed later by peer_removed.
+	var addedPeerID tailcfg.NodeID
+
+	t.Run("peer_added_with_filter_and_user_profile", func(t *testing.T) {
+		// Add a fresh peer. Bundle a new PacketFilter rule allowing
+		// TCP from that peer's IP to a port we'll later probe, and a
+		// new UserProfile for the user that owns the new peer.
+		newPeer := streamer.AllocPeer()
+		newUser := tailcfg.UserID(42)
+		newPeer.User = newUser
+		newPeer.Addresses = []netip.Prefix{netip.MustParsePrefix("100.64.0.42/32")}
+		addedPeerID = newPeer.ID
+		sendDelta(t, &tailcfg.MapResponse{
+			PeersChanged: []*tailcfg.Node{newPeer},
+			PacketFilter: []tailcfg.FilterRule{{
+				SrcIPs:   []string{"100.64.0.42/32"},
+				IPProto:  []int{int(ipproto.TCP)},
+				DstPorts: []tailcfg.NetPortRange{{IP: "*", Ports: tailcfg.PortRange{First: 22, Last: 22}}},
+			}},
+			UserProfiles: []tailcfg.UserProfile{{
+				ID:          newUser,
+				LoginName:   "alice@example.com",
+				DisplayName: "Alice",
+			}},
+		})
+
+		waitDelta(t, mFast, 1)
+		waitDelta(t, mAdd, 1)
+		waitDelta(t, mFilter, 1)
+		waitDelta(t, mUsers, 1)
+		waitDelta(t, mFull, 0)
+
+		// Side effects.
+		nv, ok := lb.PeerByID(newPeer.ID)
+		if !ok || nv.ID() != newPeer.ID {
+			t.Errorf("PeerByID(%d) ok=%v node=%v", newPeer.ID, ok, nv)
+		}
+		uv, ok := lb.UserProfile(newUser)
+		if !ok || uv.LoginName() != "alice@example.com" {
+			t.Errorf("UserProfile(%d) ok=%v login=%q", newUser, ok, uv.LoginName())
+		}
+		pf := lb.GetFilterForTest()
+		if got := pf.Check(netip.MustParseAddr("100.64.0.42"), selfIP4, 22, ipproto.TCP); got != filter.Accept {
+			t.Errorf("packet filter Check from new peer = %s; want Accept", got)
+		}
+	})
+
+	t.Run("peer_patch_derp_home", func(t *testing.T) {
+		// Patch the initial peer's DERPRegion via PeersChangedPatch.
+		// This rides as NodeMutationDERPHome.
+		sendDelta(t, &tailcfg.MapResponse{
+			PeersChangedPatch: []*tailcfg.PeerChange{{
+				NodeID:     2,
+				DERPRegion: 7,
+			}},
+		})
+
+		waitDelta(t, mFast, 1)
+		waitDelta(t, mPatch, 1)
+		waitDelta(t, mFull, 0)
+
+		nv, ok := lb.PeerByID(2)
+		if !ok {
+			t.Fatalf("PeerByID(2) not found")
+		}
+		if got := nv.HomeDERP(); got != 7 {
+			t.Errorf("HomeDERP = %d, want 7", got)
+		}
+	})
+
+	t.Run("peer_online_and_last_seen", func(t *testing.T) {
+		// Online + LastSeen on the same delta. PeerSeenChange's value
+		// is true to set LastSeen, false to clear it; the time it gets
+		// is now() at the time MutationsFromMapResponse runs on the
+		// client, not a wire value.
+		sendDelta(t, &tailcfg.MapResponse{
+			OnlineChange:   map[tailcfg.NodeID]bool{2: true},
+			PeerSeenChange: map[tailcfg.NodeID]bool{2: true},
+		})
+
+		waitDelta(t, mFast, 1)
+		// Two mutations: one NodeMutationOnline + one NodeMutationLastSeen.
+		waitDelta(t, mPatch, 2)
+		waitDelta(t, mFull, 0)
+
+		nv, ok := lb.PeerByID(2)
+		if !ok {
+			t.Fatalf("PeerByID(2) not found")
+		}
+		if o := nv.Online(); !o.Valid() || !o.Get() {
+			t.Errorf("Online = %v, want true", o)
+		}
+	})
+
+	t.Run("peer_removed", func(t *testing.T) {
+		if addedPeerID == 0 {
+			t.Fatal("peer_added_with_filter_and_user_profile must run first")
+		}
+		// Sanity check: the peer should currently exist.
+		if _, ok := lb.PeerByID(addedPeerID); !ok {
+			t.Fatalf("PeerByID(%d) missing before removal", addedPeerID)
+		}
+
+		sendDelta(t, &tailcfg.MapResponse{
+			PeersRemoved: []tailcfg.NodeID{addedPeerID},
+		})
+
+		waitDelta(t, mFast, 1)
+		waitDelta(t, mRem, 1)
+		waitDelta(t, mFull, 0)
+
+		if _, ok := lb.PeerByID(addedPeerID); ok {
+			t.Errorf("PeerByID(%d) still present after PeersRemoved", addedPeerID)
+		}
+	})
+}
@@ -40,7 +40,7 @@ var (
 // processing peer-add/peer-remove deltas in steady state, with no IPN bus
 // subscribers attached. This represents the headless-tailscaled workload
 // (Linux subnet routers, container sidecars, ...) where the LocalBackend
-// does not pay for fanning Notify.NetMap out to GUI watchers.
+// does not pay for fanning Notify events out to GUI watchers.
 //
 // Use [BenchmarkGiantTailnetBusWatcher] for the GUI-client workload.
 //
@@ -54,9 +54,9 @@ func BenchmarkGiantTailnet(b *testing.B) {

 // BenchmarkGiantTailnetBusWatcher is like [BenchmarkGiantTailnet] but
 // attaches one [local.Client.WatchIPNBus] subscriber for the duration of the
-// benchmark. The Notify-fan-out cost (notably Notify.NetMap encoding to
-// every watcher on every full-rebuild path) is therefore included in the
-// per-delta measurement, which approximates the GUI-client workload.
+// benchmark. The Notify-fan-out cost (per-watcher encoding done on every
+// full-rebuild path) is therefore included in the per-delta measurement,
+// which approximates the GUI-client workload.
 //
 // The benchmark is opt-in via --actually-test-giant-tailnet.
 func BenchmarkGiantTailnetBusWatcher(b *testing.B) {
@@ -160,15 +160,17 @@ func benchGiantTailnet(b *testing.B, busWatcher bool) {
 		notifyCh = make(chan struct{}, 1024)
 		go func() {
 			for {
-				n, err := bw.Next()
-				if err != nil {
+				if _, err := bw.Next(); err != nil {
 					return
 				}
-				if n.NetMap != nil || len(n.PeerChanges) > 0 {
-					select {
-					case notifyCh <- struct{}{}:
-					default:
-					}
+				// Any notify counts as a per-delta ack: peer add/remove
+				// in the delta path emits Notify.PeersChanged /
+				// Notify.PeersRemoved, peer patches emit
+				// Notify.PeerChanges, and self-node updates emit
+				// Notify.SelfChange.
+				select {
+				case notifyCh <- struct{}{}:
+				default:
 				}
 			}
 		}()