feature/conn25/datapath.go

@@ -108,17 +108,17 @@ func (dh *datapathHandler) HandlePacketFromWireGuard(p *packet.Parsed) filter.Re
 
 	// Check if this is an existing (return) flow on a client.
 	// If found, perform the action for the existing client flow and return.
-	existing, ok := dh.clientFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
+	action, ok := dh.clientFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
 	if ok {
-		existing.Action(p)
+		action(p)
 		return filter.Accept
 	}
 
 	// Check if this is an existing connector outbound flow.
 	// If found, perform the action for the existing connector outbound flow and return.
-	existing, ok = dh.connectorFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
+	action, ok = dh.connectorFlowTable.LookupFromWireGuard(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
 	if ok {
-		existing.Action(p)
+		action(p)
 		return filter.Accept
 	}
 
@@ -145,15 +145,18 @@ func (dh *datapathHandler) HandlePacketFromWireGuard(p *packet.Parsed) filter.Re
 	// This is a new outbound flow on a connector. Install a DNAT TransitIP-to-RealIP action
 	// for the outgoing direction, and an SNAT RealIP-to-TransitIP action for the
 	// return direction.
-	outgoing := FlowData{
+	outgoing := TupleAndAction{
 		Tuple:  flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst),
 		Action: dh.dnatAction(realIP),
 	}
-	incoming := FlowData{
+	incoming := TupleAndAction{
 		Tuple:  flowtrack.MakeTuple(p.IPProto, netip.AddrPortFrom(realIP, p.Dst.Port()), p.Src),
 		Action: dh.snatAction(transitIP),
 	}
-	if err := dh.connectorFlowTable.NewFlowFromWireGuard(outgoing, incoming); err != nil {
+	if err := dh.connectorFlowTable.NewFlow(FlowData{
+		FromTun: incoming,
+		FromWG:  outgoing,
+	}); err != nil {
 		dh.debugLogf("error installing flow, passing packet unmodified: %v", err)
 		return filter.Accept
 	}
@@ -174,17 +177,17 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed) filter.Re
 
 	// Check if this is an existing client outbound flow.
 	// If found, perform the action for the existing client flow and return.
-	existing, ok := dh.clientFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
+	action, ok := dh.clientFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
 	if ok {
-		existing.Action(p)
+		action(p)
 		return filter.Accept
 	}
 
 	// Check if this is an existing connector return flow.
 	// If found, perform the action for the existing connector return flow and return.
-	existing, ok = dh.connectorFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
+	action, ok = dh.connectorFlowTable.LookupFromTunDevice(flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst))
 	if ok {
-		existing.Action(p)
+		action(p)
 		return filter.Accept
 	}
 
@@ -211,15 +214,18 @@ func (dh *datapathHandler) HandlePacketFromTunDevice(p *packet.Parsed) filter.Re
 	// This is a new outbound client flow. Install a DNAT MagicIP-to-TransitIP action
 	// for the outgoing direction, and an SNAT TransitIP-to-MagicIP action for the
 	// return direction.
-	outgoing := FlowData{
+	outgoing := TupleAndAction{
 		Tuple:  flowtrack.MakeTuple(p.IPProto, p.Src, p.Dst),
 		Action: dh.dnatAction(transitIP),
 	}
-	incoming := FlowData{
+	incoming := TupleAndAction{
 		Tuple:  flowtrack.MakeTuple(p.IPProto, netip.AddrPortFrom(transitIP, p.Dst.Port()), p.Src),
 		Action: dh.snatAction(magicIP),
 	}
-	if err := dh.clientFlowTable.NewFlowFromTunDevice(outgoing, incoming); err != nil {
+	if err := dh.clientFlowTable.NewFlow(FlowData{
+		FromTun: outgoing,
+		FromWG:  incoming,
+	}); err != nil {
 		dh.debugLogf("error installing flow from tun device, passing packet unmodified: %v", err)
 		return filter.Accept
 	}

feature/conn25/flowtable.go

@@ -4,7 +4,7 @@
 package conn25
 
 import (
-	"errors"
+	"container/list"
 	"sync"
 
 	"tailscale.com/net/flowtrack"
@@ -14,12 +14,23 @@ import (
 // PacketAction may modify the packet.
 type PacketAction func(*packet.Parsed)
 
-// FlowData is an entry stored in the [FlowTable].
-type FlowData struct {
+// TupleAndAction wraps the [flowtrack.Tuple] and
+// the [PacketAction] to return on lookups to that
+// tuple.
+type TupleAndAction struct {
 	Tuple  flowtrack.Tuple
 	Action PacketAction
 }
 
+// FlowData is an entry stored in the [FlowTable]
+// constructed by the consumer of the table.
+// It specifies tuples and actions for each direction
+// of the flow.
+type FlowData struct {
+	FromTun TupleAndAction
+	FromWG  TupleAndAction
+}
+
 // Origin is used to track the direction of a flow.
 type Origin uint8
 
@@ -31,119 +42,131 @@ const (
 	FromWireGuard
 )
 
+// cachedFlow is the main unit of storage in the table.
+// It wraps the [FlowData] passed in by the consumer, as well
+// as internal metadata and callbacks.
 type cachedFlow struct {
-	flow   FlowData
-	paired flowtrack.Tuple // tuple for the other direction
+	data FlowData // user-defined tuples and actions for both directions
+
+	// lastSeen time.Time // tracks when the flow was last hit for expiration management
+	// onRemove func()    // fires on removal/expiration (e.g. update watchers, send RST to client)
 }
 
 // FlowTable stores and retrieves [FlowData] that can be looked up
-// by 5-tuple. New entries specify the tuple to use for both directions
+// by 5-tuple [flowtrack.Tuple] and direction.
+// New entries specify the tuple to use for both directions
 // of traffic flow. The underlying cache is LRU, and the maximum number
 // of entries is specified in calls to [NewFlowTable]. FlowTable has
 // its own mutex and is safe for concurrent use.
 type FlowTable struct {
 	mu           sync.Mutex
-	fromTunCache *flowtrack.Cache[cachedFlow] // guarded by mu
-	fromWGCache  *flowtrack.Cache[cachedFlow] // guarded by mu
+	fromTunCache map[flowtrack.Tuple]*list.Element
+	fromWGCache  map[flowtrack.Tuple]*list.Element
+	lru          *list.List
+	maxEntries   int
 }
 
-// NewFlowTable returns a [FlowTable] maxEntries maximum entries.
+// NewFlowTable returns a [FlowTable] with maxEntries maximum entries.
 // A maxEntries of 0 indicates no maximum. See also [FlowTable].
 func NewFlowTable(maxEntries int) *FlowTable {
 	return &FlowTable{
-		fromTunCache: &flowtrack.Cache[cachedFlow]{
-			MaxEntries: maxEntries,
-		},
-		fromWGCache: &flowtrack.Cache[cachedFlow]{
-			MaxEntries: maxEntries,
-		},
+		fromTunCache: make(map[flowtrack.Tuple]*list.Element, maxEntries),
+		fromWGCache:  make(map[flowtrack.Tuple]*list.Element, maxEntries),
+		lru:          list.New(),
+		maxEntries:   maxEntries,
 	}
 }
 
-// LookupFromTunDevice looks up a [FlowData] entry that is valid to run for packets
+// LookupFromTunDevice looks up a [PacketAction] that is valid to run on packets
 // observed as coming from the tun device. The tuple must match the direction it was
 // stored with.
-func (t *FlowTable) LookupFromTunDevice(k flowtrack.Tuple) (FlowData, bool) {
+func (t *FlowTable) LookupFromTunDevice(k flowtrack.Tuple) (PacketAction, bool) {
 	return t.lookup(k, FromTun)
 }
 
-// LookupFromWireGuard looks up a [FlowData] entry that is valid to run for packets
+// LookupFromWireGuard looks up a [PacketAction] that is valid to run for packets
 // observed as coming from the WireGuard tunnel. The tuple must match the direction it was
 // stored with.
-func (t *FlowTable) LookupFromWireGuard(k flowtrack.Tuple) (FlowData, bool) {
+func (t *FlowTable) LookupFromWireGuard(k flowtrack.Tuple) (PacketAction, bool) {
 	return t.lookup(k, FromWireGuard)
 }
 
-func (t *FlowTable) lookup(k flowtrack.Tuple, want Origin) (FlowData, bool) {
-	var cache *flowtrack.Cache[cachedFlow]
-	switch want {
+func (t *FlowTable) lookup(k flowtrack.Tuple, dir Origin) (PacketAction, bool) {
+	var cache map[flowtrack.Tuple]*list.Element
+	switch dir {
 	case FromTun:
 		cache = t.fromTunCache
 	case FromWireGuard:
 		cache = t.fromWGCache
 	default:
-		return FlowData{}, false
+		return nil, false
 	}
 
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	v, ok := cache.Get(k)
+	ele, ok := cache[k]
 	if !ok {
-		return FlowData{}, false
-	}
-	return v.flow, true
-}
-
-// NewFlowFromTunDevice installs (or overwrites) both the forward and return entries.
-// The forward tuple is tagged as FromTun, and the return tuple is tagged as FromWireGuard.
-// If overwriting, it removes the old paired tuple for the forward key to avoid stale reverse mappings.
-func (t *FlowTable) NewFlowFromTunDevice(fwd, rev FlowData) error {
-	return t.newFlow(FromTun, fwd, rev)
-}
-
-// NewFlowFromWireGuard installs (or overwrites) both the forward and return entries.
-// The forward tuple is tagged as FromWireGuard, and the return tuple is tagged as FromTun.
-// If overwriting, it removes the old paired tuple for the forward key to avoid stale reverse mappings.
-func (t *FlowTable) NewFlowFromWireGuard(fwd, rev FlowData) error {
-	return t.newFlow(FromWireGuard, fwd, rev)
-}
-
-func (t *FlowTable) newFlow(fwdOrigin Origin, fwd, rev FlowData) error {
-	if fwd.Action == nil || rev.Action == nil {
-		return errors.New("nil action received for flow")
+		return nil, false
 	}
 
-	var fwdCache, revCache *flowtrack.Cache[cachedFlow]
-	switch fwdOrigin {
+	flow := ele.Value.(*cachedFlow)
+
+	var action PacketAction
+	switch dir {
 	case FromTun:
-		fwdCache, revCache = t.fromTunCache, t.fromWGCache
+		action = flow.data.FromTun.Action
 	case FromWireGuard:
-		fwdCache, revCache = t.fromWGCache, t.fromTunCache
-	default:
-		return errors.New("newFlow called with unknown direction")
+		action = flow.data.FromWG.Action
 	}
 
+	// Support LRU.
+	t.lru.MoveToFront(ele)
+
+	// TODO(mzb): Update flow.lastSeen.
+
+	return action, true
+}
+
+// NewFlow installs data as an flow in the table, and evicts any flow that
+// either tuple already points at. This can result in two flows being evicted
+// if each of the new tuples point at distinct existing flows. If the new flow
+// would cause the table to exceed its maximum size, the least recently used
+// (looked-up or created) flow is evicted. data is not validated, the caller must
+// supply non-nil packet actions.
+func (t *FlowTable) NewFlow(data FlowData) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()
 
-	// If overwriting an existing entry, remove its previously-paired mapping so
-	// we don't leave stale tuples around.
-	if old, ok := fwdCache.Get(fwd.Tuple); ok {
-		revCache.Remove(old.paired)
-	}
-	if old, ok := revCache.Get(rev.Tuple); ok {
-		fwdCache.Remove(old.paired)
+	// If either tuple leads to anything existing, remove it.
+	t.removeFlowLocked(t.fromTunCache[data.FromTun.Tuple])
+	t.removeFlowLocked(t.fromWGCache[data.FromWG.Tuple])
+
+	flow := &cachedFlow{
+		data: data,
+		// Populate lastSeen
+		// Populate onRemove()
 	}
 
-	fwdCache.Add(fwd.Tuple, cachedFlow{
-		flow:   fwd,
-		paired: rev.Tuple,
-	})
-	revCache.Add(rev.Tuple, cachedFlow{
-		flow:   rev,
-		paired: fwd.Tuple,
-	})
+	ele := t.lru.PushFront(flow)
+	if t.maxEntries > 0 && t.lru.Len() > t.maxEntries {
+		t.removeFlowLocked(t.lru.Back())
+	}
+
+	t.fromTunCache[data.FromTun.Tuple] = ele
+	t.fromWGCache[data.FromWG.Tuple] = ele
 
 	return nil
 }
+
+func (t *FlowTable) removeFlowLocked(ele *list.Element) {
+	if ele == nil {
+		return
+	}
+
+	flow := t.lru.Remove(ele).(*cachedFlow)
+	delete(t.fromTunCache, flow.data.FromTun.Tuple)
+	delete(t.fromWGCache, flow.data.FromWG.Tuple)
+
+	// TODO(mzb): run flow.onRemove()
+}

feature/conn25/flowtable_test.go

@@ -12,114 +12,220 @@ import (
 	"tailscale.com/types/ipproto"
 )
 
-func TestFlowTable(t *testing.T) {
-	ft := NewFlowTable(0)
+var nilPacket *packet.Parsed // nil packet to perform actions against
 
-	fwdTuple := flowtrack.MakeTuple(
+// mkTuple wraps flowtrack.MakeTuple with the UDP proto.
+func mkTuple(src, dst string) flowtrack.Tuple {
+	return flowtrack.MakeTuple(
 		ipproto.UDP,
-		netip.MustParseAddrPort("1.2.3.4:1000"),
-		netip.MustParseAddrPort("4.3.2.1:80"),
-	)
-	// Reverse tuple is defined by caller. Doesn't have to be mirror image of fwd.
-	// To account for intentional modifications, like NAT.
-	revTuple := flowtrack.MakeTuple(
-		ipproto.UDP,
-		netip.MustParseAddrPort("4.3.2.2:80"),
-		netip.MustParseAddrPort("1.2.3.4:1000"),
+		netip.MustParseAddrPort(src),
+		netip.MustParseAddrPort(dst),
 	)
+}
 
-	fwdAction, revAction := 0, 0
-	fwdData := FlowData{
-		Tuple:  fwdTuple,
-		Action: func(_ *packet.Parsed) { fwdAction++ },
-	}
-	revData := FlowData{
-		Tuple:  revTuple,
-		Action: func(_ *packet.Parsed) { revAction++ },
-	}
+func mkFlowWithActions(fromTun, fromWG flowtrack.Tuple) (FlowData, *int, *int) {
+	var tunCalls, wgCalls int
+	fd := mkFlow(fromTun, fromWG)
+	fd.FromTun.Action = func(_ *packet.Parsed) { tunCalls++ }
+	fd.FromWG.Action = func(_ *packet.Parsed) { wgCalls++ }
+	return fd, &tunCalls, &wgCalls
+}
 
-	// For this test setup, from the tun device will be "forward",
-	// and from WG will be "reverse".
-	if err := ft.NewFlowFromTunDevice(fwdData, revData); err != nil {
-		t.Fatalf("got non-nil error for new flow from tun device")
-	}
-
-	// Test basic lookups.
-	lookupFwd, ok := ft.LookupFromTunDevice(fwdTuple)
-	if !ok {
-		t.Fatalf("got not found on first lookup from tun device")
-	}
-	lookupFwd.Action(nil)
-	if fwdAction != 1 {
-		t.Errorf("action for fwd tuple key was not executed")
-	}
-
-	lookupRev, ok := ft.LookupFromWireGuard(revTuple)
-	if !ok {
-		t.Fatalf("got not found on first lookup from WireGuard")
-	}
-	lookupRev.Action(nil)
-	if revAction != 1 {
-		t.Errorf("action for rev tuple key was not executed")
-	}
-
-	// Test not found error.
-	notFoundTuple := flowtrack.MakeTuple(
-		ipproto.UDP,
-		netip.MustParseAddrPort("1.2.3.4:1000"),
-		netip.MustParseAddrPort("4.0.4.4:80"),
-	)
-	if _, ok := ft.LookupFromTunDevice(notFoundTuple); ok {
-		t.Errorf("expected not found for foreign tuple")
-	}
-
-	// Wrong direction is also not found.
-	if _, ok := ft.LookupFromWireGuard(fwdTuple); ok {
-		t.Errorf("expected not found for wrong direction tuple")
-	}
-
-	// Overwriting forward tuple removes its reverse pair as well.
-	newRevData := FlowData{
-		Tuple: flowtrack.MakeTuple(
-			ipproto.UDP,
-			netip.MustParseAddrPort("9.9.9.9:99"),
-			netip.MustParseAddrPort("8.8.8.8:88"),
-		),
-		Action: func(_ *packet.Parsed) {},
-	}
-	if err := ft.NewFlowFromTunDevice(
-		fwdData,
-		newRevData,
-	); err != nil {
-		t.Fatalf("got non-nil error for new flow from tun device")
-	}
-	if _, ok := ft.LookupFromWireGuard(revTuple); ok {
-		t.Errorf("expected not found for removed reverse tuple")
-	}
-
-	// Overwriting reverse tuple removes its forward pair as well.
-	if err := ft.NewFlowFromTunDevice(
-		FlowData{
-			Tuple: flowtrack.MakeTuple(
-				ipproto.UDP,
-				netip.MustParseAddrPort("8.8.8.8:88"),
-				netip.MustParseAddrPort("9.9.9.9:99"),
-			),
+func mkFlow(fromTun, fromWG flowtrack.Tuple) FlowData {
+	return FlowData{
+		FromTun: TupleAndAction{
+			Tuple:  fromTun,
+			Action: func(_ *packet.Parsed) {},
+		},
+		FromWG: TupleAndAction{
+			Tuple:  fromWG,
 			Action: func(_ *packet.Parsed) {},
 		},
-		newRevData, // This is the same "reverse" data installed in previous test.
-	); err != nil {
-		t.Fatalf("got non-nil error for new flow from tun device")
-	}
-	if _, ok := ft.LookupFromTunDevice(fwdTuple); ok {
-		t.Errorf("expected not found for removed forward tuple")
-	}
-
-	// Nil action returns an error.
-	if err := ft.NewFlowFromTunDevice(
-		FlowData{},
-		FlowData{},
-	); err == nil {
-		t.Errorf("expected non-nil error for nil data")
 	}
 }
+
+func mustInstallFlow(t *testing.T, ft *FlowTable, flow FlowData) {
+	t.Helper()
+	if err := ft.NewFlow(flow); err != nil {
+		t.Fatalf("error installing flow: %v", err)
+	}
+}
+
+func assertFlowHit(t *testing.T, ft *FlowTable, dir Origin, tuple flowtrack.Tuple) PacketAction {
+	t.Helper()
+	return assertFlowLookup(t, ft, dir, tuple, true)
+}
+
+func assertFlowMiss(t *testing.T, ft *FlowTable, dir Origin, tuple flowtrack.Tuple) {
+	t.Helper()
+	assertFlowLookup(t, ft, dir, tuple, false)
+}
+
+func assertFlowLookup(t *testing.T, ft *FlowTable, dir Origin, tuple flowtrack.Tuple, wantHit bool) PacketAction {
+	t.Helper()
+	var action PacketAction
+	var ok bool
+
+	switch dir {
+	case FromTun:
+		action, ok = ft.LookupFromTunDevice(tuple)
+	case FromWireGuard:
+		action, ok = ft.LookupFromWireGuard(tuple)
+	default:
+		t.Fatalf("invalid direction: %v", dir)
+	}
+
+	if wantHit && !ok {
+		t.Fatalf("expected flow hit for tuple: %v, dir: %v", tuple, dir)
+	}
+	if !wantHit && ok {
+		t.Fatalf("expected flow miss for tuple: %v, dir: %v", tuple, dir)
+	}
+
+	if wantHit {
+		return action
+	}
+	return nil
+}
+
+func TestFlowTable_NewFlow_Lookup(t *testing.T) {
+	ft := NewFlowTable(0)
+
+	// The tuples in both directions are defined by the caller.
+	// The don't have to be mirror images of each other,
+	// to account for intentional modifications, like NAT.
+	fromTunTuple := mkTuple("1.2.3.4:1000", "4.3.2.1:80")
+	fromWGTuple := mkTuple("4.3.2.2:80", "1.2.3.4:1000")
+
+	flow1, tunCount1, wgCount1 := mkFlowWithActions(fromTunTuple, fromWGTuple)
+	mustInstallFlow(t, ft, flow1)
+
+	// Test basic lookups, and perform actions on packet.
+	assertFlowHit(t, ft, FromTun, fromTunTuple)(nilPacket)
+	assertFlowHit(t, ft, FromWireGuard, fromWGTuple)(nilPacket)
+
+	if *tunCount1 != 1 {
+		t.Fatal("action for from-tun tuple key was not executed")
+	}
+	if *wgCount1 != 1 {
+		t.Fatal("action for from-wg tuple key was not executed")
+	}
+
+	// Test tuple not found.
+	notFoundTuple := mkTuple("1.2.3.4:1000", "4.0.4.4:80")
+	assertFlowMiss(t, ft, FromTun, notFoundTuple)
+
+	// Wrong direction is also not found.
+	assertFlowMiss(t, ft, FromWireGuard, fromTunTuple)
+
+	// Overwriting from-tun tuple removes the from-wg tuple as well.
+	newFromWGTuple := mkTuple("9.9.9.9:99", "8.8.8.8:88")
+	flow2 := mkFlow(fromTunTuple, newFromWGTuple)
+	mustInstallFlow(t, ft, flow2)
+	assertFlowMiss(t, ft, FromWireGuard, fromWGTuple)
+
+	// Overwriting the from-wg tuple removes the from-tun tuple as well.
+	newFromTunTuple := mkTuple("8.8.8.8:88", "9.9.9.9:99")
+	flow3 := mkFlow(newFromTunTuple, newFromWGTuple)
+	mustInstallFlow(t, ft, flow3)
+	assertFlowMiss(t, ft, FromTun, fromTunTuple)
+}
+
+// TestFlowTable_OneReplacesTwo targets a specific case
+// in which a single new flow replaces two existing flows
+// because each tuple of the new flow matches one tuple
+// of an existing flow.
+func TestFlowTable_OneReplacesTwo(t *testing.T) {
+	ft := NewFlowTable(0)
+
+	tunTuple1 := mkTuple("1.2.3.4:1000", "4.3.2.1:80")
+	wgTuple1 := mkTuple("4.3.2.2:80", "1.2.3.4:1000")
+	flow1, tunCount1, wgCount1 := mkFlowWithActions(tunTuple1, wgTuple1)
+
+	tunTuple2 := mkTuple("8.8.8.8:88", "9.9.9.9:99")
+	wgTuple2 := mkTuple("9.9.9.9:99", "8.8.8.8:88")
+	flow2, tunCount2, wgCount2 := mkFlowWithActions(tunTuple2, wgTuple2)
+
+	// Install the first two flows.
+	mustInstallFlow(t, ft, flow1)
+	mustInstallFlow(t, ft, flow2)
+
+	// Confirm they are properly installed through lookups.
+	assertFlowHit(t, ft, FromTun, tunTuple1)
+	assertFlowHit(t, ft, FromWireGuard, wgTuple1)
+	assertFlowHit(t, ft, FromTun, tunTuple2)
+	assertFlowHit(t, ft, FromWireGuard, wgTuple2)
+
+	// flow3 tuples overlap with flow1 and flow2.
+	flow3, tunCount3, wgCount3 := mkFlowWithActions(tunTuple1, wgTuple2)
+	mustInstallFlow(t, ft, flow3)
+
+	// flow3 lookups hit on both of their tuples.
+	tunAction3 := assertFlowHit(t, ft, FromTun, tunTuple1)
+	wgAction3 := assertFlowHit(t, ft, FromWireGuard, wgTuple2)
+
+	// The non-overlapping tuples from flow1 and flow2 should now miss.
+	assertFlowMiss(t, ft, FromTun, tunTuple2)
+	assertFlowMiss(t, ft, FromWireGuard, wgTuple1)
+
+	// Perform both actions on a nil packet to bump counters.
+	tunAction3(nilPacket)
+	wgAction3(nilPacket)
+
+	// Only flow3 counters should have been bumped.
+	if *tunCount1 != 0 || *wgCount1 != 0 {
+		t.Fatalf("flow1 counters (tun, wg), want: (0,0), got: (%d,%d)", *tunCount1, *wgCount1)
+	}
+	if *tunCount2 != 0 || *wgCount2 != 0 {
+		t.Fatalf("flow2 counters (tun, wg), want: (0,0), got: (%d,%d)", *tunCount2, *wgCount2)
+	}
+	if *tunCount3 != 1 || *wgCount3 != 1 {
+		t.Fatalf("flow3 counters (tun, wg), want: (1,1), got: (%d,%d)", *tunCount3, *wgCount3)
+	}
+}
+
+func TestFlowTable_Eviction(t *testing.T) {
+	// Table only has two spots.
+	ft := NewFlowTable(2)
+	aTun, aWG := mkTuple("3.0.0.1:1000", "3.0.0.2:80"), mkTuple("3.0.0.2:80", "3.0.0.1:1000")
+	bTun, bWG := mkTuple("3.0.0.3:1000", "3.0.0.4:80"), mkTuple("3.0.0.4:80", "3.0.0.3:1000")
+	cTun, cWG := mkTuple("3.0.0.5:1000", "3.0.0.6:80"), mkTuple("3.0.0.6:80", "3.0.0.5:1000")
+	dTun, dWG := mkTuple("3.0.0.7:1000", "3.0.0.8:80"), mkTuple("3.0.0.8:80", "3.0.0.7:1000")
+
+	a := mkFlow(aTun, aWG)
+	b := mkFlow(bTun, bWG)
+	c := mkFlow(cTun, cWG)
+	d := mkFlow(dTun, dWG)
+
+	// Install a and b.
+	mustInstallFlow(t, ft, a)
+	mustInstallFlow(t, ft, b)
+
+	// Move a to the front from tun side, b is ready for eviction.
+	assertFlowHit(t, ft, FromTun, aTun)
+
+	// Install c.
+	mustInstallFlow(t, ft, c)
+
+	// Check b is out.
+	assertFlowMiss(t, ft, FromTun, bTun)
+	assertFlowMiss(t, ft, FromWireGuard, bWG)
+
+	// Check c is in.
+	assertFlowHit(t, ft, FromTun, cTun)
+	assertFlowHit(t, ft, FromWireGuard, cWG)
+
+	// Move a to the front again, now from WG side.
+	assertFlowHit(t, ft, FromWireGuard, aWG)
+
+	// Install d.
+	mustInstallFlow(t, ft, d)
+
+	// Check c is out.
+	assertFlowMiss(t, ft, FromTun, cTun)
+	assertFlowMiss(t, ft, FromWireGuard, cWG)
+
+	// Check d is in.
+	assertFlowHit(t, ft, FromTun, dTun)
+	assertFlowHit(t, ft, FromWireGuard, dWG)
+}