wgengine,wgcfg,feature/netlog: move network flow logging behind a feature hook

wgcfg.Config.NetworkLogging carried the network flow logging identity
inside the WireGuard config, where it was unrelated to WireGuard; it
lived there mainly so that identity changes would defeat Reconfig's
ErrNoChanges check and reach the netlog startup/shutdown logic.

Remove the field and move the whole netlog lifecycle into a new
feature/netlog package, installed on the engine via the new
wgengine.HookNewNetLogger hook, like other feature/* packages. The
logging identity now comes from LocalBackend's current netmap via the
widened NetLogSource interface (replacing Engine.SetNetLogNodeSource),
so nmcfg no longer parses audit log IDs into the config. The engine
still calls the hook before its ErrNoChanges return and before
router.Set (to capture initial packets), and again after router.Set
(to capture final packets), preserving the previous ordering.

Core wgengine no longer imports wgengine/netlog, so minimal builds
drop it entirely. tailscaled keeps netlog via feature/condregister,
and tsnet imports feature/condregister/netlog explicitly to keep
netlog enabled by default in tsnet-based binaries (tsidp,
k8s-operator).

This is pulled out of a future change that removes wgcfg.Config.Peers,
to make that PR smaller.

Updates #12542
Updates #12614

Change-Id: I41ca7dfe43c51e977c41b5f8e934bd1f0e6e6e24
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

wgengine/netlog_hooks.go

@@ -0,0 +1,86 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package wgengine
+
+import (
+	"net/netip"
+
+	"tailscale.com/feature"
+	"tailscale.com/health"
+	"tailscale.com/net/netmon"
+	"tailscale.com/net/tstun"
+	"tailscale.com/tailcfg"
+	"tailscale.com/types/logger"
+	"tailscale.com/types/logid"
+	"tailscale.com/util/eventbus"
+	"tailscale.com/wgengine/magicsock"
+	"tailscale.com/wgengine/router"
+)
+
+// NetLogSource provides the network flow logging feature what it needs
+// from the rest of the system (in practice, ipnlocal.LocalBackend):
+// on-demand node lookups and the current audit logging identity.
+type NetLogSource interface {
+	// SelfNode returns the current self node and its owning user profile.
+	// The views are invalid if there is no current node.
+	SelfNode() (tailcfg.NodeView, tailcfg.UserProfileView)
+
+	// NodeByAddr returns the node assigned the given address along with
+	// its owning user profile.
+	// ok is false if no node is known to own addr.
+	NodeByAddr(addr netip.Addr) (node tailcfg.NodeView, user tailcfg.UserProfileView, ok bool)
+
+	// NetLogIDs returns the audit logging IDs from the current netmap,
+	// along with whether exit node flows should be logged.
+	// ok is false if network flow logging should not run.
+	NetLogIDs() (nodeID, domainID logid.PrivateID, logExitFlows bool, ok bool)
+}
+
+// NetLogger is the engine's handle on the network flow logger
+// lifecycle, implemented by the feature/netlog package.
+//
+// Reconfig and ReconfigDone are only called from [Engine.Reconfig],
+// serialized by the engine's internal lock. Shutdown may be called
+// concurrently with them from [Engine.Close].
+type NetLogger interface {
+	// Reconfig is called near the top of every [Engine.Reconfig],
+	// before its ErrNoChanges early return (so identity-only changes
+	// still take effect) and before the router is configured (so a
+	// starting logger captures initial packets). It reports whether
+	// network logging state changed, which suppresses the engine's
+	// ErrNoChanges early return.
+	Reconfig(routerCfg *router.Config, routerChanged bool) (changed bool)
+
+	// ReconfigDone is called at the end of [Engine.Reconfig], after
+	// the router is configured, so that a logger stopping as a result
+	// of the preceding Reconfig call captures final packets. It may
+	// block to flush pending messages.
+	ReconfigDone()
+
+	// Shutdown stops the logger. It may block to flush pending
+	// messages, bounded by an internal upload timeout.
+	Shutdown()
+}
+
+// NetLoggerDeps are the engine-owned dependencies passed to
+// [HookNewNetLogger] to construct a [NetLogger].
+type NetLoggerDeps struct {
+	Logf   logger.Logf
+	Tun    *tstun.Wrapper
+	Sock   *magicsock.Conn
+	NetMon *netmon.Monitor
+	Health *health.Tracker
+	Bus    *eventbus.Bus
+
+	// Source returns the [NetLogSource] installed via
+	// [Engine.SetNetLogSource], or nil if none has been installed yet.
+	// It is a func because the source is installed after the engine
+	// (and thus the NetLogger) is constructed.
+	Source func() NetLogSource
+}
+
+// HookNewNetLogger is set by the feature/netlog package to construct
+// each engine's [NetLogger]. If unset, network flow logging is not
+// linked into the binary and the engine skips all NetLogger calls.
+var HookNewNetLogger feature.Hook[func(NetLoggerDeps) NetLogger]

wgengine/userspace.go

@@ -58,7 +58,6 @@ import (
 	"tailscale.com/version"
 	"tailscale.com/wgengine/filter"
 	"tailscale.com/wgengine/magicsock"
-	"tailscale.com/wgengine/netlog"
 	"tailscale.com/wgengine/netstack/gro"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
@@ -66,10 +65,6 @@ import (
 	"tailscale.com/wgengine/wglog"
 )
 
-// networkLoggerUploadTimeout is the maximum timeout to wait when
-// shutting down the network logger as it uploads the last network log messages.
-const networkLoggerUploadTimeout = 5 * time.Second
-
 type userspaceEngine struct {
 	// eventBus will eventually become required, but for now may be nil.
 	eventBus    *eventbus.Bus
@@ -156,13 +151,16 @@ type userspaceEngine struct {
 	// value of the ICMP identifier and sequence number concatenated.
 	icmpEchoResponseCallback map[uint32]func()
 
-	// networkLogger logs statistics about network connections.
-	networkLogger netlog.Logger
+	// netlogger is the network flow logger handle constructed via
+	// [HookNewNetLogger], or nil if the feature/netlog package is not
+	// linked into the binary.
+	netlogger NetLogger
 
-	// netLogSource is the [netlog.NodeSource] installed via
-	// [Engine.SetNetLogNodeSource]; it is read when starting up the network
-	// logger from inside Reconfig. It may be nil if no source was installed.
-	netLogSource syncs.AtomicValue[netlog.NodeSource]
+	// netLogSource is the [NetLogSource] installed via
+	// [Engine.SetNetLogSource]; the netlogger reads it on each
+	// Reconfig via [NetLoggerDeps.Source]. It may be nil if no source
+	// was installed.
+	netLogSource syncs.AtomicValue[NetLogSource]
 
 	// wgPeerLookup is the lookup function installed via
 	// [Engine.SetWGPeerLookup]; it is consulted by [wgLogger] to translate
@@ -454,6 +452,18 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
 	closePool.add(e.magicConn)
 	e.magicConn.SetNetworkUp(e.netMon.InterfaceState().AnyInterfaceUp())
 
+	if newNetLogger, ok := HookNewNetLogger.GetOk(); ok {
+		e.netlogger = newNetLogger(NetLoggerDeps{
+			Logf:   logf,
+			Tun:    e.tundev,
+			Sock:   e.magicConn,
+			NetMon: e.netMon,
+			Health: e.health,
+			Bus:    e.eventBus,
+			Source: e.netLogSource.Load,
+		})
+	}
+
 	tsTUNDev.SetDiscoKey(e.magicConn.DiscoPublicKey())
 
 	if conf.RespondToPing {
@@ -758,9 +768,9 @@ func (e *userspaceEngine) SetPeerSessionStateFunc(fn func(key.NodePublic, PeerWi
 	})
 }
 
-// SetNetLogNodeSource installs the [netlog.NodeSource] used by the engine's
-// network logger.
-func (e *userspaceEngine) SetNetLogNodeSource(src netlog.NodeSource) {
+// SetNetLogSource installs the [NetLogSource] consulted by the engine's
+// network flow logger.
+func (e *userspaceEngine) SetNetLogSource(src NetLogSource) {
 	e.netLogSource.Store(src)
 }
 
@@ -864,17 +874,18 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 
 	listenPortChanged := listenPort != e.magicConn.LocalPort()
 	peerMTUChanged := peerMTUEnable != e.magicConn.PeerMTUEnabled()
-	if !engineChanged && !routerChanged && !dnsChanged && !listenPortChanged && !isSubnetRouterChanged && !peerMTUChanged {
-		return ErrNoChanges
+
+	// Let the network flow logger react before the early return below,
+	// so that logging identity changes take effect even when nothing
+	// else changed, and before the router is configured, so that a
+	// starting logger captures initial packets.
+	netlogChanged := false
+	if e.netlogger != nil {
+		netlogChanged = e.netlogger.Reconfig(routerCfg, routerChanged)
 	}
-	newLogIDs := cfg.NetworkLogging
-	oldLogIDs := e.lastCfgFull.NetworkLogging
-	netLogIDsNowValid := !newLogIDs.NodeID.IsZero() && !newLogIDs.DomainID.IsZero()
-	netLogIDsWasValid := !oldLogIDs.NodeID.IsZero() && !oldLogIDs.DomainID.IsZero()
-	netLogIDsChanged := netLogIDsNowValid && netLogIDsWasValid && newLogIDs != oldLogIDs
-	netLogRunning := netLogIDsNowValid && !routerCfg.Equal(&router.Config{})
-	if !buildfeatures.HasNetLog || envknob.NoLogsNoSupport() {
-		netLogRunning = false
+
+	if !engineChanged && !routerChanged && !dnsChanged && !listenPortChanged && !isSubnetRouterChanged && !peerMTUChanged && !netlogChanged {
+		return ErrNoChanges
 	}
 
 	// TODO(bradfitz,danderson): maybe delete this isDNSIPOverTailscale
@@ -982,36 +993,8 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 		}
 	}
 
-	// Shutdown the network logger because the IDs changed.
-	// Let it be started back up by subsequent logic.
-	if buildfeatures.HasNetLog && netLogIDsChanged && e.networkLogger.Running() {
-		e.logf("wgengine: Reconfig: shutting down network logger")
-		ctx, cancel := context.WithTimeout(context.Background(), networkLoggerUploadTimeout)
-		defer cancel()
-		if err := e.networkLogger.Shutdown(ctx); err != nil {
-			e.logf("wgengine: Reconfig: error shutting down network logger: %v", err)
-		}
-	}
-
-	// Startup the network logger.
-	// Do this before configuring the router so that we capture initial packets.
-	if buildfeatures.HasNetLog && netLogRunning && !e.networkLogger.Running() {
-		nid := cfg.NetworkLogging.NodeID
-		tid := cfg.NetworkLogging.DomainID
-		logExitFlowEnabled := cfg.NetworkLogging.LogExitFlowEnabled
-		e.logf("wgengine: Reconfig: starting up network logger (node:%s tailnet:%s)", nid.Public(), tid.Public())
-		src := e.netLogSource.Load()
-		if src == nil {
-			e.logf("wgengine: Reconfig: no NodeSource installed; network logger not started")
-		} else if err := e.networkLogger.Startup(e.logf, src, nid, tid, e.tundev, e.magicConn, e.netMon, e.health, e.eventBus, logExitFlowEnabled); err != nil {
-			e.logf("wgengine: Reconfig: error starting up network logger: %v", err)
-		}
-		e.networkLogger.ReconfigRoutes(routerCfg)
-	}
-
 	if routerChanged {
 		e.logf("wgengine: Reconfig: configuring router")
-		e.networkLogger.ReconfigRoutes(routerCfg)
 		err := e.router.Set(routerCfg)
 		e.health.SetRouterHealth(err)
 		if err != nil {
@@ -1050,16 +1033,11 @@ func (e *userspaceEngine) Reconfig(cfg *wgcfg.Config, routerCfg *router.Config,
 		}
 	}
 
-	// Shutdown the network logger.
-	// Do this after configuring the router so that we capture final packets.
-	// This attempts to flush out any log messages and may block.
-	if !netLogRunning && e.networkLogger.Running() {
-		e.logf("wgengine: Reconfig: shutting down network logger")
-		ctx, cancel := context.WithTimeout(context.Background(), networkLoggerUploadTimeout)
-		defer cancel()
-		if err := e.networkLogger.Shutdown(ctx); err != nil {
-			e.logf("wgengine: Reconfig: error shutting down network logger: %v", err)
-		}
+	// Let the network flow logger finish reacting after the router is
+	// configured, so that a stopping logger captures final packets.
+	// This may block to flush pending log messages.
+	if e.netlogger != nil {
+		e.netlogger.ReconfigDone()
 	}
 
 	if buildfeatures.HasBird && isSubnetRouterChanged && e.birdClient != nil {
@@ -1252,10 +1230,8 @@ func (e *userspaceEngine) Close() {
 	}
 	close(e.waitCh)
 
-	ctx, cancel := context.WithTimeout(context.Background(), networkLoggerUploadTimeout)
-	defer cancel()
-	if err := e.networkLogger.Shutdown(ctx); err != nil {
-		e.logf("wgengine: Close: error shutting down network logger: %v", err)
+	if e.netlogger != nil {
+		e.netlogger.Shutdown()
 	}
 }
 

wgengine/wgcfg/config.go

@@ -9,7 +9,6 @@ import (
 	"slices"
 
 	"tailscale.com/types/key"
-	"tailscale.com/types/logid"
 )
 
 //go:generate go run tailscale.com/cmd/cloner -type=Config,Peer
@@ -20,15 +19,6 @@ type Config struct {
 	PrivateKey key.NodePrivate
 	Addresses  []netip.Prefix
 	Peers      []Peer
-
-	// NetworkLogging enables network logging.
-	// It is disabled if either ID is the zero value.
-	// LogExitFlowEnabled indicates whether or not exit flows should be logged.
-	NetworkLogging struct {
-		NodeID             logid.PrivateID
-		DomainID           logid.PrivateID
-		LogExitFlowEnabled bool
-	}
 }
 
 func (c *Config) Equal(o *Config) bool {
@@ -36,7 +26,6 @@ func (c *Config) Equal(o *Config) bool {
 		return c == o
 	}
 	return c.PrivateKey.Equal(o.PrivateKey) &&
-		c.NetworkLogging == o.NetworkLogging &&
 		slices.Equal(c.Addresses, o.Addresses) &&
 		slices.EqualFunc(c.Peers, o.Peers, Peer.Equal)
 }

wgengine/wgcfg/config_test.go

@@ -14,8 +14,7 @@ func TestConfigEqual(t *testing.T) {
 	rt := reflect.TypeFor[Config]()
 	for sf := range rt.Fields() {
 		switch sf.Name {
-		case "Name", "NodeID", "PrivateKey", "Addresses", "Peers",
-			"NetworkLogging":
+		case "Name", "NodeID", "PrivateKey", "Addresses", "Peers":
 			// These are compared in [Config.Equal].
 		default:
 			t.Errorf("Have you added field %q to Config.Equal? Do so if not, and then update TestConfigEqual", sf.Name)

wgengine/wgcfg/nmcfg/nmcfg.go

@@ -15,7 +15,6 @@ import (
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/logger"
-	"tailscale.com/types/logid"
 	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine/wgcfg"
 )
@@ -53,27 +52,6 @@ func WGCfg(pk key.NodePrivate, nm *netmap.NetworkMap, logf logger.Logf, flags ne
 		Peers:      make([]wgcfg.Peer, 0, len(nm.Peers)),
 	}
 
-	// Setup log IDs for data plane audit logging.
-	if nm.SelfNode.Valid() {
-		canNetworkLog := nm.SelfNode.HasCap(tailcfg.CapabilityDataPlaneAuditLogs)
-		logExitFlowEnabled := nm.SelfNode.HasCap(tailcfg.NodeAttrLogExitFlows)
-		if canNetworkLog && nm.SelfNode.DataPlaneAuditLogID() != "" && nm.DomainAuditLogID != "" {
-			nodeID, errNode := logid.ParsePrivateID(nm.SelfNode.DataPlaneAuditLogID())
-			if errNode != nil {
-				logf("[v1] wgcfg: unable to parse node audit log ID: %v", errNode)
-			}
-			domainID, errDomain := logid.ParsePrivateID(nm.DomainAuditLogID)
-			if errDomain != nil {
-				logf("[v1] wgcfg: unable to parse domain audit log ID: %v", errDomain)
-			}
-			if errNode == nil && errDomain == nil {
-				cfg.NetworkLogging.NodeID = nodeID
-				cfg.NetworkLogging.DomainID = domainID
-				cfg.NetworkLogging.LogExitFlowEnabled = logExitFlowEnabled
-			}
-		}
-	}
-
 	var skippedExitNode, skippedSubnetRouter, skippedExpired []tailcfg.NodeView
 
 	for _, peer := range nm.Peers {

wgengine/wgcfg/wgcfg_clone.go

@@ -9,7 +9,6 @@ import (
 	"net/netip"
 
 	"tailscale.com/types/key"
-	"tailscale.com/types/logid"
 )
 
 // Clone makes a deep copy of Config.
@@ -32,14 +31,9 @@ func (src *Config) Clone() *Config {
 
 // A compilation failure here means this code must be regenerated, with the command at the top of this file.
 var _ConfigCloneNeedsRegeneration = Config(struct {
-	PrivateKey     key.NodePrivate
-	Addresses      []netip.Prefix
-	Peers          []Peer
-	NetworkLogging struct {
-		NodeID             logid.PrivateID
-		DomainID           logid.PrivateID
-		LogExitFlowEnabled bool
-	}
+	PrivateKey key.NodePrivate
+	Addresses  []netip.Prefix
+	Peers      []Peer
 }{})
 
 // Clone makes a deep copy of Peer.

wgengine/wgengine.go

@@ -16,7 +16,6 @@ import (
 	"tailscale.com/types/key"
 	"tailscale.com/types/netmap"
 	"tailscale.com/wgengine/filter"
-	"tailscale.com/wgengine/netlog"
 	"tailscale.com/wgengine/router"
 	"tailscale.com/wgengine/wgcfg"
 	"tailscale.com/wgengine/wgint"
@@ -198,12 +197,13 @@ type Engine interface {
 	// look up which peer should handle an outbound packet by destination IP.
 	SetPeerByIPPacketFunc(func(netip.Addr) (_ key.NodePublic, ok bool))
 
-	// SetNetLogNodeSource installs the [netlog.NodeSource] used by the engine's
-	// network logger to look up Tailscale node info on demand.
+	// SetNetLogSource installs the [NetLogSource] consulted by the
+	// engine's network flow logger for node lookups and the current
+	// audit logging identity.
 	//
 	// It is expected to be called once during LocalBackend construction,
 	// before any [Engine.Reconfig] call that starts up the network logger.
-	SetNetLogNodeSource(netlog.NodeSource)
+	SetNetLogSource(NetLogSource)
 
 	// SetWGPeerLookup installs the function used by the engine's
 	// wireguard-go log wrapper to rewrite peer references in log lines