net/netns: add functionality to bind outgoing sockets based on route table

When turned on via environment variable (off by default), this will use
the BSD routing APIs to query what interface index a socket should be
bound to, rather than binding to the default interface in all cases.

Updates #5719
Updates #5940

Signed-off-by: Andrew Dunham <andrew@du.nham.ca>
Change-Id: Ib4c919471f377b7a08cd3413f8e8caacb29fee0b

ipn/ipnlocal/local.go

@@ -48,6 +48,7 @@ import (
 	"tailscale.com/net/dnscache"
 	"tailscale.com/net/dnsfallback"
 	"tailscale.com/net/interfaces"
+	"tailscale.com/net/netns"
 	"tailscale.com/net/netutil"
 	"tailscale.com/net/tsaddr"
 	"tailscale.com/net/tsdial"
@@ -3823,6 +3824,9 @@ func (b *LocalBackend) setNetMapLocked(nm *netmap.NetworkMap) {
 
 	b.setDebugLogsByCapabilityLocked(nm)
 
+	// See the netns package for documentation on what this capability does.
+	netns.SetBindToInterfaceByRoute(hasCapability(nm, tailcfg.CapabilityBindToInterfaceByRoute))
+
 	b.setTCPPortsInterceptedFromNetmapAndPrefsLocked(b.pm.CurrentPrefs())
 	if nm == nil {
 		b.nodeByAddr = nil

net/netns/netns.go

@@ -32,6 +32,17 @@ func SetEnabled(on bool) {
 	disabled.Store(!on)
 }
 
+var bindToInterfaceByRoute atomic.Bool
+
+// SetBindToInterfaceByRoute enables or disables whether we use the system's
+// route information to bind to a particular interface. It is the same as
+// setting the TS_BIND_TO_INTERFACE_BY_ROUTE.
+//
+// Currently, this only changes the behaviour on macOS.
+func SetBindToInterfaceByRoute(v bool) {
+	bindToInterfaceByRoute.Store(v)
+}
+
 // Listener returns a new net.Listener with its Control hook func
 // initialized as necessary to run in logical network namespace that
 // doesn't route back into Tailscale.

net/netns/netns_darwin.go

@@ -11,10 +11,14 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"net/netip"
+	"os"
 	"strings"
 	"syscall"
 
+	"golang.org/x/net/route"
 	"golang.org/x/sys/unix"
+	"tailscale.com/envknob"
 	"tailscale.com/net/interfaces"
 	"tailscale.com/types/logger"
 )
@@ -25,6 +29,10 @@ func control(logf logger.Logf) func(network, address string, c syscall.RawConn)
 	}
 }
 
+var bindToInterfaceByRouteEnv = envknob.RegisterBool("TS_BIND_TO_INTERFACE_BY_ROUTE")
+
+var errInterfaceIndexInvalid = errors.New("interface index invalid")
+
 // controlLogf marks c as necessary to dial in a separate network namespace.
 //
 // It's intentionally the same signature as net.Dialer.Control
@@ -34,15 +42,145 @@ func controlLogf(logf logger.Logf, network, address string, c syscall.RawConn) e
 		// Don't bind to an interface for localhost connections.
 		return nil
 	}
-	idx, err := interfaces.DefaultRouteInterfaceIndex()
+
+	idx, err := getInterfaceIndex(logf, address)
 	if err != nil {
-		logf("[unexpected] netns: DefaultRouteInterfaceIndex: %v", err)
+		// callee logged
 		return nil
 	}
 
 	return bindConnToInterface(c, network, address, idx, logf)
 }
 
+func getInterfaceIndex(logf logger.Logf, address string) (int, error) {
+	// Helper so we can log errors.
+	defaultIdx := func() (int, error) {
+		idx, err := interfaces.DefaultRouteInterfaceIndex()
+		if err != nil {
+			logf("[unexpected] netns: DefaultRouteInterfaceIndex: %v", err)
+			return -1, err
+		}
+		return idx, nil
+	}
+
+	useRoute := bindToInterfaceByRoute.Load() || bindToInterfaceByRouteEnv()
+	if !useRoute {
+		return defaultIdx()
+	}
+
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		// No port number; use the string directly.
+		host = address
+	}
+
+	// If the address doesn't parse, use the default index.
+	addr, err := netip.ParseAddr(host)
+	if err != nil {
+		logf("[unexpected] netns: error parsing address %q: %v", host, err)
+		return defaultIdx()
+	}
+
+	idx, err := interfaceIndexFor(addr, true /* canRecurse */)
+	if err != nil {
+		logf("netns: error in interfaceIndexFor: %v", err)
+		return defaultIdx()
+	}
+
+	// Verify that we didn't just choose the Tailscale interface;
+	// if so, we fall back to binding from the default.
+	_, tsif, err2 := interfaces.Tailscale()
+	if err2 == nil && tsif.Index == idx {
+		logf("[unexpected] netns: interfaceIndexFor returned Tailscale interface")
+		return defaultIdx()
+	}
+
+	return idx, err
+}
+
+// interfaceIndexFor returns the interface index that we should bind to in
+// order to send traffic to the provided address.
+func interfaceIndexFor(addr netip.Addr, canRecurse bool) (int, error) {
+	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return 0, fmt.Errorf("creating AF_ROUTE socket: %w", err)
+	}
+	defer unix.Close(fd)
+
+	var routeAddr route.Addr
+	if addr.Is4() {
+		routeAddr = &route.Inet4Addr{IP: addr.As4()}
+	} else {
+		routeAddr = &route.Inet6Addr{IP: addr.As16()}
+	}
+
+	rm := route.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_GET,
+		Flags:   unix.RTF_UP,
+		ID:      uintptr(os.Getpid()),
+		Seq:     1,
+		Addrs: []route.Addr{
+			unix.RTAX_DST: routeAddr,
+		},
+	}
+	b, err := rm.Marshal()
+	if err != nil {
+		return 0, fmt.Errorf("marshaling RouteMessage: %w", err)
+	}
+	_, err = unix.Write(fd, b)
+	if err != nil {
+		return 0, fmt.Errorf("writing message: %w", err)
+	}
+	var buf [2048]byte
+	n, err := unix.Read(fd, buf[:])
+	if err != nil {
+		return 0, fmt.Errorf("reading message: %w", err)
+	}
+	msgs, err := route.ParseRIB(route.RIBTypeRoute, buf[:n])
+	if err != nil {
+		return 0, fmt.Errorf("route.ParseRIB: %w", err)
+	}
+	if len(msgs) == 0 {
+		return 0, fmt.Errorf("no messages")
+	}
+
+	for _, msg := range msgs {
+		rm, ok := msg.(*route.RouteMessage)
+		if !ok {
+			continue
+		}
+		if rm.Version < 3 || rm.Version > 5 || rm.Type != unix.RTM_GET {
+			continue
+		}
+		if len(rm.Addrs) < unix.RTAX_GATEWAY {
+			continue
+		}
+
+		switch addr := rm.Addrs[unix.RTAX_GATEWAY].(type) {
+		case *route.LinkAddr:
+			return addr.Index, nil
+		case *route.Inet4Addr:
+			// We can get a gateway IP; recursively call ourselves
+			// (exactly once) to get the link (and thus index) for
+			// the gateway IP.
+			if canRecurse {
+				return interfaceIndexFor(netip.AddrFrom4(addr.IP), false)
+			}
+		case *route.Inet6Addr:
+			// As above.
+			if canRecurse {
+				return interfaceIndexFor(netip.AddrFrom16(addr.IP), false)
+			}
+		default:
+			// Unknown type; skip it
+			continue
+		}
+	}
+
+	return 0, fmt.Errorf("no valid address found")
+}
+
 // SetListenConfigInterfaceIndex sets lc.Control such that sockets are bound
 // to the provided interface index.
 func SetListenConfigInterfaceIndex(lc *net.ListenConfig, ifIndex int) error {

net/netns/netns_darwin_test.go

@@ -0,0 +1,85 @@
+// Copyright (c) 2023 Tailscale Inc & AUTHORS All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package netns
+
+import (
+	"testing"
+
+	"tailscale.com/net/interfaces"
+)
+
+func TestGetInterfaceIndex(t *testing.T) {
+	oldVal := bindToInterfaceByRoute.Load()
+	t.Cleanup(func() { bindToInterfaceByRoute.Store(oldVal) })
+	bindToInterfaceByRoute.Store(true)
+
+	tests := []struct {
+		name string
+		addr string
+		err  string
+	}{
+		{
+			name: "IP_and_port",
+			addr: "8.8.8.8:53",
+		},
+		{
+			name: "bare_ip",
+			addr: "8.8.8.8",
+		},
+		{
+			name: "invalid",
+			addr: "!!!!!",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			idx, err := getInterfaceIndex(t.Logf, tc.addr)
+			if err != nil {
+				if tc.err == "" {
+					t.Fatalf("got unexpected error: %v", err)
+				}
+				if errstr := err.Error(); errstr != tc.err {
+					t.Errorf("expected error %q, got %q", errstr, tc.err)
+				}
+			} else {
+				t.Logf("getInterfaceIndex(%q) = %d", tc.addr, idx)
+				if tc.err != "" {
+					t.Fatalf("wanted error %q", tc.err)
+				}
+				if idx < 0 {
+					t.Fatalf("got invalid index %d", idx)
+				}
+			}
+		})
+	}
+
+	t.Run("NoTailscale", func(t *testing.T) {
+		_, tsif, err := interfaces.Tailscale()
+		if err != nil {
+			t.Fatal(err)
+		}
+		if tsif == nil {
+			t.Skip("no tailscale interface on this machine")
+		}
+
+		defaultIdx, err := interfaces.DefaultRouteInterfaceIndex()
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		idx, err := getInterfaceIndex(t.Logf, "100.100.100.100:53")
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Logf("tailscaleIdx=%d defaultIdx=%d idx=%d", tsif.Index, defaultIdx, idx)
+
+		if idx == tsif.Index {
+			t.Fatalf("got idx=%d; wanted not Tailscale interface", idx)
+		} else if idx != defaultIdx {
+			t.Fatalf("got idx=%d, want %d", idx, defaultIdx)
+		}
+	})
+}

tailcfg/tailcfg.go

@@ -94,7 +94,8 @@ type CapabilityVersion int
 //   - 54: 2023-01-19: Node.Cap added, PeersChangedPatch.Cap, uses Node.Cap for ExitDNS before Hostinfo.Services fallback
 //   - 55: 2023-01-23: start of c2n GET+POST /update handler
 //   - 56: 2023-01-24: Client understands CapabilityDebugTSDNSResolution
-const CurrentCapabilityVersion CapabilityVersion = 56
+//   - 57: 2023-01-25: Client understands CapabilityBindToInterfaceByRoute
+const CurrentCapabilityVersion CapabilityVersion = 57
 
 type StableID string
 
@@ -1726,6 +1727,11 @@ const (
 	CapabilityDataPlaneAuditLogs = "https://tailscale.com/cap/data-plane-audit-logs" // feature enabled
 	CapabilityDebug              = "https://tailscale.com/cap/debug"                 // exposes debug endpoints over the PeerAPI
 
+	// CapabilityBindToInterfaceByRoute changes how Darwin nodes create
+	// sockets (in the net/netns package). See that package for more
+	// details on the behaviour of this capability.
+	CapabilityBindToInterfaceByRoute = "https://tailscale.com/cap/bind-to-interface-by-route"
+
 	// CapabilityTailnetLockAlpha indicates the node is in the tailnet lock alpha,
 	// and initialization of tailnet lock may proceed.
 	//