feature/conn25: expire idle flows from FlowTable

Track lastSeen on each cached flow and add a sweeper goroutine
that periodically removes flows idle past the idle timeout.

Introduce tunables for idle timeout, maximum flows removed per sweep (to
limit mutex hold time), and the sweeper interval.

Also cap the previously-unlimited tables: 10k client flows, 100k
connector flows.

Updates tailscale/corp#38630

Signed-off-by: Michael Ben-Ami <mzb@tailscale.com>

feature/conn25/flowtable.go

@@ -5,10 +5,13 @@ package conn25
 
 import (
 	"container/list"
+	"context"
 	"sync"
+	"time"
 
 	"tailscale.com/net/flowtrack"
 	"tailscale.com/net/packet"
+	"tailscale.com/tstime/mono"
 )
 
 // PacketAction may modify the packet.
@@ -48,7 +51,7 @@ const (
 type cachedFlow struct {
 	data FlowData // user-defined tuples and actions for both directions
 
-	// lastSeen time.Time // tracks when the flow was last hit for expiration management
+	lastSeen mono.Time // tracks when the flow was last hit for expiration management
 	// onRemove func()    // fires on removal/expiration (e.g. update watchers, send RST to client)
 }
 
@@ -59,22 +62,81 @@ type cachedFlow struct {
 // of entries is specified in calls to [NewFlowTable]. FlowTable has
 // its own mutex and is safe for concurrent use.
 type FlowTable struct {
+	maxEntries         int
+	idleTimeout        time.Duration
+	sweepInterval      time.Duration
+	maxRemovedPerSweep int
+
 	mu           sync.Mutex
 	fromTunCache map[flowtrack.Tuple]*list.Element
 	fromWGCache  map[flowtrack.Tuple]*list.Element
 	lru          *list.List
-	maxEntries   int
+}
+
+const (
+	// DefaultFlowIdleTimeout is the default idle timeout for a flow.
+	// See also [WithFlowIdleTimeout].
+	DefaultFlowIdleTimeout = 5 * time.Minute
+	// DefaultFlowSweepInterval is the default sweep interval for
+	// automatically removing expired flows. See also [WithFlowSweepInterval].
+	DefaultFlowSweepInterval = 3 * time.Minute
+	// DefaultMaxRemovedFlowsPerSweep is the default maximum number of
+	// flows removed per sweep. It can be used to tune how long the table
+	// mutex is held during sweeps. See also [WithMaxRemovedFlowsPerSweep].
+	DefaultMaxRemovedFlowsPerSweep = 1000
+)
+
+// FlowTableOption configures options for use with [NewFlowTable].
+type FlowTableOption func(ft *FlowTable)
+
+// WithFlowIdleTimeout sets the threshold duration for flow idle time
+// before it is eligible for removal. A flow is considered idle for the
+// time that elapses since its creation or last lookup. A duration of
+// 0 means that expiration is disabled. If WithFlowIdleTimeout is not
+// passed to [NewFlowTable], then [DefaultFlowIdleTimeout] is used.
+func WithFlowIdleTimeout(timeout time.Duration) FlowTableOption {
+	return func(ft *FlowTable) {
+		ft.idleTimeout = timeout
+	}
+}
+
+// WithFlowSweepInterval sets the interval to automatically
+// remove idle flows that exceed the idle timeout. A value of 0
+// disables automatic sweeping. If WithFlowSweepInterval is not
+// passed to [NewFlowTable], then [DefaultFlowSweepInterval] is used.
+func WithFlowSweepInterval(ival time.Duration) FlowTableOption {
+	return func(ft *FlowTable) {
+		ft.sweepInterval = ival
+	}
+}
+
+// WithMaxRemovedFlowsPerSweep sets maximum number of expired
+// flows that can be removed per sweep. A value of 0 means no
+// maximum. If WithMaxRemovedFlowsPerSweep is not passed to
+// [NewFlowTable], then [DefaultMaxRemovedFlowsPerSweep] is used.
+func WithMaxRemovedFlowsPerSweep(maxPer int) FlowTableOption {
+	return func(ft *FlowTable) {
+		ft.maxRemovedPerSweep = maxPer
+	}
 }
 
 // NewFlowTable returns a [FlowTable] with maxEntries maximum entries.
 // A maxEntries of 0 indicates no maximum. See also [FlowTable].
-func NewFlowTable(maxEntries int) *FlowTable {
-	return &FlowTable{
-		fromTunCache: make(map[flowtrack.Tuple]*list.Element, maxEntries),
-		fromWGCache:  make(map[flowtrack.Tuple]*list.Element, maxEntries),
-		lru:          list.New(),
-		maxEntries:   maxEntries,
+func NewFlowTable(maxEntries int, opts ...FlowTableOption) *FlowTable {
+	ft := &FlowTable{
+		maxEntries:         maxEntries,
+		idleTimeout:        DefaultFlowIdleTimeout,
+		sweepInterval:      DefaultFlowSweepInterval,
+		maxRemovedPerSweep: DefaultMaxRemovedFlowsPerSweep,
+		fromTunCache:       make(map[flowtrack.Tuple]*list.Element, maxEntries),
+		fromWGCache:        make(map[flowtrack.Tuple]*list.Element, maxEntries),
+		lru:                list.New(),
 	}
+
+	for _, o := range opts {
+		o(ft)
+	}
+	return ft
 }
 
 // LookupFromTunDevice looks up a [PacketAction] that is valid to run on packets
@@ -123,7 +185,7 @@ func (t *FlowTable) lookup(k flowtrack.Tuple, dir Origin) (PacketAction, bool) {
 	// Support LRU.
 	t.lru.MoveToFront(ele)
 
-	// TODO(mzb): Update flow.lastSeen.
+	flow.lastSeen = mono.Now()
 
 	return action, true
 }
@@ -143,8 +205,8 @@ func (t *FlowTable) NewFlow(data FlowData) error {
 	t.removeFlowLocked(t.fromWGCache[data.FromWG.Tuple])
 
 	flow := &cachedFlow{
-		data: data,
-		// Populate lastSeen
+		data:     data,
+		lastSeen: mono.Now(),
 		// Populate onRemove()
 	}
 
@@ -159,6 +221,51 @@ func (t *FlowTable) NewFlow(data FlowData) error {
 	return nil
 }
 
+// StartExpiredSweeper starts a sweeper that removes idle flows that have
+// not been created or looked up for a duration greater than the configured
+// idle timeout. See [WithFlowIdleTimeout]. The sweep runs at the configured
+// sweep interval. See [WithFlowSweepInterval]. The sweeper stops when ctx
+// is canceled.
+func (t *FlowTable) StartExpiredSweeper(ctx context.Context) {
+	if t.sweepInterval == 0 || t.idleTimeout == 0 {
+		return
+	}
+
+	ticker := time.NewTicker(t.sweepInterval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			t.removeIdle(mono.Now())
+		}
+	}
+}
+
+func (t *FlowTable) removeIdle(now mono.Time) int {
+	if t.idleTimeout == 0 {
+		return 0
+	}
+
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	removed := 0
+	for ele := t.lru.Back(); ele != nil; ele = t.lru.Back() {
+		if t.maxRemovedPerSweep > 0 && removed >= t.maxRemovedPerSweep {
+			break
+		}
+		flow := ele.Value.(*cachedFlow)
+		if now.Sub(flow.lastSeen) <= t.idleTimeout {
+			break
+		}
+		t.removeFlowLocked(ele)
+		removed++
+	}
+	return removed
+}
+
 func (t *FlowTable) removeFlowLocked(ele *list.Element) {
 	if ele == nil {
 		return