tstest/natlab/vnet: add multi-NIC node support, DHCP fixes, and VIPs

Multi-NIC support:
  - Add nodeNIC type and node.extraNICs for secondary network interfaces
  - Add netForMAC/macForNet to route packets to the correct network by MAC
  - Update initFromConfig to allocate a MAC + LAN IP per network
  - Fix handleEthernetFrameFromVM, ServeUnixConn to use netForMAC
  - Fix MACOfIP, writeEth, WriteUDPPacketNoNAT, gVisor write path, and
    createARPResponse to use macForNet (return the MAC actually on that
    network, not the node's primary MAC)
  - Fix createDHCPResponse for multi-NIC (correct client IP and subnet)
  - Add nodeNICMac for secondary NIC MAC generation
  - Add Node accessors: NumNICs, NICMac, Networks, LanIP

DHCP fixes:
  - Include LeaseTime, SubnetMask, Router, DNS in DHCP Offer (not just
    Ack). systemd-networkd requires these to accept an Offer.
  - Fix DHCP response source IP: use gateway IP instead of echoing
    the request's destination (which was 255.255.255.255 for discovers)

New VIPs:
  - cloud-init.tailscale: serves per-node cloud-init meta-data, user-data,
    and network-config for VMs booting with nocloud datasource
  - files.tailscale: serves binary files (tta, tailscale, tailscaled)
    registered via RegisterFile for cloud VM provisioning
  - Add ControlServer() accessor for test control server

This is necessary for a three-VM natlab subnet router
integration test, coming later.

Updates #13038

Change-Id: I59f9f356bae9b5509c117265237983972dfdd5af
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

tstest/natlab/vnet/conf.go

@@ -72,6 +72,13 @@ func nodeMac(n int) MAC {
 	return MAC{0x52, 0xcc, 0xcc, 0xcc, 0xcc, byte(n)}
 }
 
+// nodeNICMac returns the MAC for the nicIdx-th secondary NIC (1-indexed) of node n.
+// Primary NICs (index 0) use nodeMac. Secondary NICs use a different scheme:
+// 52:cc:cc:cc:KK:NN where KK is the NIC index and NN is the node number.
+func nodeNICMac(nodeNum, nicIdx int) MAC {
+	return MAC{0x52, 0xcc, 0xcc, 0xcc, byte(nicIdx), byte(nodeNum)}
+}
+
 func routerMac(n int) MAC {
 	// 52=TS then 0xee for 'etwork
 	return MAC{0x52, 0xee, 0xee, 0xee, 0xee, byte(n)}
@@ -236,11 +243,31 @@ func (n *Node) String() string {
 	return fmt.Sprintf("node%d", n.num)
 }
 
-// MAC returns the MAC address of the node.
+// MAC returns the MAC address of the node's primary NIC.
 func (n *Node) MAC() MAC {
 	return n.mac
 }
 
+// NumNICs returns the number of network interfaces on the node
+// (one per network the node is on).
+func (n *Node) NumNICs() int {
+	return len(n.nets)
+}
+
+// NICMac returns the MAC address for the i-th NIC (0-indexed).
+// NIC 0 is the primary NIC (same as MAC()). NIC 1+ are extra NICs.
+func (n *Node) NICMac(i int) MAC {
+	if i == 0 {
+		return n.mac
+	}
+	return nodeNICMac(n.num, i)
+}
+
+// Networks returns the list of networks this node is on.
+func (n *Node) Networks() []*Network {
+	return n.nets
+}
+
 func (n *Node) Env() []TailscaledEnv {
 	return n.env
 }
@@ -306,6 +333,26 @@ func (n *Node) IsV6Only() bool {
 	return false
 }
 
+// LanIP returns the node's LAN IPv4 address on the given network.
+// It requires the [Server] to have been initialized (i.e., [New] was called).
+// Returns an invalid addr if the node has no IP on that network.
+func (n *Node) LanIP(net *Network) netip.Addr {
+	if n.n == nil {
+		return netip.Addr{}
+	}
+	for i, nn := range n.nets {
+		if nn == net {
+			if i == 0 {
+				return n.n.lanIP
+			}
+			if i-1 < len(n.n.extraNICs) {
+				return n.n.extraNICs[i-1].lanIP
+			}
+		}
+	}
+	return netip.Addr{}
+}
+
 // Network returns the first network this node is connected to,
 // or nil if none.
 func (n *Node) Network() *Network {
@@ -486,10 +533,11 @@ func (s *Server) initFromConfig(c *Config) error {
 		if conf.err != nil {
 			return conf.err
 		}
+		primaryNet := netOfConf[conf.Network()]
 		n := &node{
 			num:           conf.num,
 			mac:           conf.mac,
-			net:           netOfConf[conf.Network()],
+			net:           primaryNet,
 			verboseSyslog: conf.VerboseSyslog(),
 		}
 		n.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
@@ -503,18 +551,52 @@ func (s *Server) initFromConfig(c *Config) error {
 		s.nodes = append(s.nodes, n)
 		s.nodeByMAC[n.mac] = n
 
-		if n.net.v4 {
+		if n.net != nil && n.net.v4 {
 			// Allocate a lanIP for the node. Use the network's CIDR and use final
 			// octet 101 (for first node), 102, etc. The node number comes from the
-			// last octent of the MAC address (0-based)
+			// last octet of the MAC address (0-based)
 			ip4 := n.net.lanIP4.Addr().As4()
 			ip4[3] = 100 + n.mac[5]
 			n.lanIP = netip.AddrFrom4(ip4)
 			n.net.nodesByIP4[n.lanIP] = n
 		}
+		if n.net != nil {
 			n.net.nodesByMAC[n.mac] = n
 		}
 
+		// Set up extra NICs for multi-homed nodes (nodes on more than one network).
+		for nicIdx, confNet := range conf.nets[1:] {
+			extraNet := netOfConf[confNet]
+			if extraNet == nil {
+				continue
+			}
+			mac := nodeNICMac(conf.num, nicIdx+1)
+			nic := nodeNIC{
+				mac: mac,
+				net: extraNet,
+			}
+			nic.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{
+				Name:     fmt.Sprintf("%s-nic%d", n.String(), nicIdx+1),
+				LinkType: layers.LinkTypeEthernet,
+			}))
+			// Allocate a lanIP for the node. Use the network's CIDR and use final
+			// octet 101 (for first node), 102, etc. The node number comes from the
+			// last octet of the MAC address (0-based)
+			if extraNet.v4 {
+				ip4 := extraNet.lanIP4.Addr().As4()
+				ip4[3] = 100 + mac[5]
+				nic.lanIP = netip.AddrFrom4(ip4)
+				extraNet.nodesByIP4[nic.lanIP] = n
+			}
+			extraNet.nodesByMAC[mac] = n
+			if _, ok := s.nodeByMAC[mac]; ok {
+				return fmt.Errorf("two nodes have the same MAC %v", mac)
+			}
+			s.nodeByMAC[mac] = n
+			n.extraNICs = append(n.extraNICs, nic)
+		}
+	}
+
 	// Now that nodes are populated, set up NAT:
 	for _, conf := range c.networks {
 		n := netOfConf[conf]

tstest/natlab/vnet/vip.go

@@ -19,6 +19,8 @@ var (
 	fakeDERP2             = newVIP("derp2.tailscale", "33.4.0.2") // 3340=DERP; 2=derp 2
 	fakeLogCatcher        = newVIP("log.tailscale.com", 4)
 	fakeSyslog            = newVIP("syslog.tailscale", 9)
+	fakeCloudInit         = newVIP("cloud-init.tailscale", 5) // serves cloud-init metadata/userdata per node
+	fakeFiles             = newVIP("files.tailscale", 6)      // serves binary files (tta, tailscale, tailscaled) to VMs
 )
 
 type virtualIP struct {

tstest/natlab/vnet/vnet.go

@@ -30,6 +30,7 @@ import (
 	"net/netip"
 	"os/exec"
 	"strconv"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -267,10 +268,13 @@ func (n *network) handleIPPacketFromGvisor(ipRaw []byte) {
 		n.logf("gvisor: serialize error: %v", err)
 		return
 	}
-	if nw, ok := n.writers.Load(node.mac); ok {
+	// Use the MAC address for this specific network (important for multi-NIC nodes
+	// where the primary MAC may be on a different network).
+	mac := node.macForNet(n)
+	if nw, ok := n.writers.Load(mac); ok {
 		nw.write(resPkt)
 	} else {
-		n.logf("gvisor write: no writeFunc for %v", node.mac)
+		n.logf("gvisor write: no writeFunc for %v (node %v on net %v)", mac, node, n.mac)
 	}
 }
 
@@ -371,6 +375,22 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) {
 		return
 	}
 
+	if destPort == 80 && fakeCloudInit.Match(destIP) {
+		r.Complete(false)
+		tc := gonet.NewTCPConn(&wq, ep)
+		hs := &http.Server{Handler: n.s.cloudInitHandler()}
+		go hs.Serve(netutil.NewOneConnListener(tc, nil))
+		return
+	}
+
+	if destPort == 80 && fakeFiles.Match(destIP) {
+		r.Complete(false)
+		tc := gonet.NewTCPConn(&wq, ep)
+		hs := &http.Server{Handler: n.s.fileServerHandler()}
+		go hs.Serve(netutil.NewOneConnListener(tc, nil))
+		return
+	}
+
 	var targetDial string
 	if n.s.derpIPs.Contains(destIP) {
 		targetDial = destIP.String() + ":" + strconv.Itoa(int(destPort))
@@ -573,8 +593,10 @@ func (n *network) MACOfIP(ip netip.Addr) (_ MAC, ok bool) {
 	if n.lanIP4.Addr() == ip {
 		return n.mac, true
 	}
-	if n, ok := n.nodesByIP4[ip]; ok {
+	if node, ok := n.nodesByIP4[ip]; ok {
-		return n.mac, true
+		// Use the MAC for this specific network (important for multi-NIC nodes
+		// where the primary MAC may be on a different network).
+		return node.macForNet(n), true
 	}
 	return MAC{}, false
 }
@@ -585,6 +607,15 @@ func (n *network) SetControlBlackholed(v bool) {
 	n.blackholeControl = v
 }
 
+// nodeNIC represents a single network interface on a node.
+// For multi-homed nodes, additional NICs beyond the primary are stored in node.extraNICs.
+type nodeNIC struct {
+	mac         MAC
+	net         *network
+	lanIP       netip.Addr
+	interfaceID int
+}
+
 type node struct {
 	mac           MAC
 	num           int // 1-based node number
@@ -593,6 +624,8 @@ type node struct {
 	lanIP         netip.Addr // must be in net.lanIP prefix + unique in net
 	verboseSyslog bool
 
+	extraNICs []nodeNIC // secondary NICs for multi-homed nodes
+
 	// logMu guards logBuf.
 	// TODO(bradfitz): conditionally write these out to separate files at the end?
 	// Currently they only hold logcatcher logs.
@@ -601,6 +634,35 @@ type node struct {
 	logCatcherWrites int
 }
 
+// netForMAC returns the network associated with the given MAC address on this node.
+// It checks the primary NIC first, then any extra NICs.
+func (n *node) netForMAC(mac MAC) *network {
+	if mac == n.mac {
+		return n.net
+	}
+	for _, nic := range n.extraNICs {
+		if nic.mac == mac {
+			return nic.net
+		}
+	}
+	return nil
+}
+
+// macForNet returns the MAC address that this node uses on the given network.
+// For the primary network, this is node.mac. For secondary networks, it's the
+// extra NIC's MAC.
+func (n *node) macForNet(net *network) MAC {
+	if n.net == net {
+		return n.mac
+	}
+	for _, nic := range n.extraNICs {
+		if nic.net == net {
+			return nic.mac
+		}
+	}
+	return n.mac // fallback to primary
+}
+
 // String returns the string "nodeN" where N is the 1-based node number.
 func (n *node) String() string {
 	return fmt.Sprintf("node%d", n.num)
@@ -657,6 +719,9 @@ type Server struct {
 	agentConnWaiter map[*node]chan<- struct{} // signaled after added to set
 	agentConns      set.Set[*agentConn]       //  not keyed by node; should be small/cheap enough to scan all
 	agentDialer     map[*node]netx.DialFunc
+
+	cloudInitData map[int]*CloudInitData // node num → cloud-init config
+	fileContents  map[string][]byte      // filename → file bytes
 }
 
 func (s *Server) logf(format string, args ...any) {
@@ -741,6 +806,96 @@ func New(c *Config) (*Server, error) {
 	return s, nil
 }
 
+// ControlServer returns the test control server used by this vnet.
+func (s *Server) ControlServer() *testcontrol.Server {
+	return s.control
+}
+
+// CloudInitData holds the cloud-init configuration for a node.
+type CloudInitData struct {
+	MetaData      string
+	UserData      string
+	NetworkConfig string // optional; if set, served as network-config
+}
+
+// SetCloudInitData registers cloud-init configuration for the given node number.
+// This data is served via the cloud-init.tailscale VIP when the VM boots.
+func (s *Server) SetCloudInitData(nodeNum int, data *CloudInitData) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.cloudInitData, nodeNum, data)
+}
+
+// RegisterFile registers a file to be served by the files.tailscale VIP.
+// The path is the URL path (e.g., "tta" is served at http://files.tailscale/tta).
+func (s *Server) RegisterFile(path string, data []byte) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	mak.Set(&s.fileContents, path, data)
+}
+
+// cloudInitHandler returns an HTTP handler that serves cloud-init
+// meta-data and user-data for VMs that boot with
+// ds=nocloud;s=http://cloud-init.tailscale/node-N/.
+func (s *Server) cloudInitHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Parse node number from URL path like "/node-2/meta-data"
+		path := strings.TrimPrefix(r.URL.Path, "/")
+		parts := strings.SplitN(path, "/", 2)
+		if len(parts) != 2 {
+			http.Error(w, "bad path", http.StatusNotFound)
+			return
+		}
+		nodeNum := 0
+		if _, err := fmt.Sscanf(parts[0], "node-%d", &nodeNum); err != nil {
+			http.Error(w, "bad node number", http.StatusNotFound)
+			return
+		}
+		s.mu.Lock()
+		data := s.cloudInitData[nodeNum]
+		s.mu.Unlock()
+		if data == nil {
+			http.Error(w, "no cloud-init data for node", http.StatusNotFound)
+			return
+		}
+		switch parts[1] {
+		case "meta-data":
+			w.Header().Set("Content-Type", "text/yaml")
+			io.WriteString(w, data.MetaData)
+		case "user-data":
+			w.Header().Set("Content-Type", "text/yaml")
+			io.WriteString(w, data.UserData)
+		case "network-config":
+			if data.NetworkConfig == "" {
+				http.Error(w, "not found", http.StatusNotFound)
+				return
+			}
+			w.Header().Set("Content-Type", "text/yaml")
+			io.WriteString(w, data.NetworkConfig)
+		default:
+			http.Error(w, "not found", http.StatusNotFound)
+		}
+	})
+}
+
+// fileServerHandler returns an HTTP handler that serves files registered
+// via RegisterFile. Files are served at http://files.tailscale/<path>.
+func (s *Server) fileServerHandler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		path := strings.TrimPrefix(r.URL.Path, "/")
+		s.mu.Lock()
+		data, ok := s.fileContents[path]
+		s.mu.Unlock()
+		if !ok {
+			http.Error(w, "not found", http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Type", "application/octet-stream")
+		w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+		w.Write(data)
+	})
+}
+
 func (s *Server) Close() {
 	if shutdown := s.shuttingDown.Swap(true); !shutdown {
 		s.shutdownCancel()
@@ -905,9 +1060,14 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) {
 		}
 		if !didReg[srcMAC] {
 			didReg[srcMAC] = true
+			srcNet := srcNode.netForMAC(srcMAC)
+			if srcNet == nil {
+				s.logf("[conn %p] node %v has no network for MAC %v", c.uc, srcNode, srcMAC)
+				continue
+			}
 			s.logf("[conn %p] Registering writer for MAC %v, node %v", c.uc, srcMAC, srcNode.lanIP)
-			srcNode.net.registerWriter(srcMAC, c)
+			srcNet.registerWriter(srcMAC, c)
-			defer srcNode.net.unregisterWriter(srcMAC)
+			defer srcNet.unregisterWriter(srcMAC)
 		}
 
 		if err := s.handleEthernetFrameFromVM(packetRaw); err != nil {
@@ -930,13 +1090,18 @@ func (s *Server) handleEthernetFrameFromVM(packetRaw []byte) error {
 		return fmt.Errorf("got frame from unknown MAC %v", srcMAC)
 	}
 
+	srcNet := srcNode.netForMAC(srcMAC)
+	if srcNet == nil {
+		return fmt.Errorf("node %v has no network for MAC %v", srcNode, srcMAC)
+	}
+
 	must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{
 		Timestamp:      time.Now(),
 		CaptureLength:  len(packetRaw),
 		Length:         len(packetRaw),
 		InterfaceIndex: srcNode.interfaceID,
 	}, packetRaw))
-	srcNode.net.HandleEthernetPacket(ep)
+	srcNet.HandleEthernetPacket(ep)
 	return nil
 }
 
@@ -1191,8 +1356,8 @@ func (n *network) WriteUDPPacketNoNAT(p UDPPacket) {
 	}
 
 	eth := &layers.Ethernet{
-		SrcMAC: n.mac.HWAddr(), // of gateway
+		SrcMAC: n.mac.HWAddr(),             // of gateway; on the specific network
-		DstMAC: node.mac.HWAddr(),
+		DstMAC: node.macForNet(n).HWAddr(), // use the MAC for this network
 	}
 	ethRaw, err := n.serializedUDPPacket(src, dst, p.Payload, eth)
 	if err != nil {
@@ -1531,12 +1696,28 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) {
 		log.Printf("DHCP request from unknown node %v; ignoring", srcMAC)
 		return nil, nil
 	}
-	gwIP := node.net.lanIP4.Addr()
+	// Use the network associated with this MAC (important for multi-NIC nodes).
+	srcNet := node.netForMAC(srcMAC)
+	if srcNet == nil {
+		log.Printf("DHCP request from MAC %v with no associated network; ignoring", srcMAC)
+		return nil, nil
+	}
+	gwIP := srcNet.lanIP4.Addr()
 
-	ipLayer := request.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
 	udpLayer := request.Layer(layers.LayerTypeUDP).(*layers.UDP)
 	dhcpLayer := request.Layer(layers.LayerTypeDHCPv4).(*layers.DHCPv4)
 
+	// Determine the client's LAN IP for this specific NIC.
+	clientIP := node.lanIP
+	if srcMAC != node.mac {
+		for _, nic := range node.extraNICs {
+			if nic.mac == srcMAC {
+				clientIP = nic.lanIP
+				break
+			}
+		}
+	}
+
 	response := &layers.DHCPv4{
 		Operation:    layers.DHCPOpReply,
 		HardwareType: layers.LinkTypeEthernet,
@@ -1544,7 +1725,7 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) {
 		Xid:          dhcpLayer.Xid,
 		ClientHWAddr: dhcpLayer.ClientHWAddr,
 		Flags:        dhcpLayer.Flags,
-		YourClientIP: node.lanIP.AsSlice(),
+		YourClientIP: clientIP.AsSlice(),
 		Options: []layers.DHCPOption{
 			{
 				Type:   layers.DHCPOptServerID,
@@ -1562,11 +1743,33 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) {
 	}
 	switch msgType {
 	case layers.DHCPMsgTypeDiscover:
-		response.Options = append(response.Options, layers.DHCPOption{
+		response.Options = append(response.Options,
+			layers.DHCPOption{
 				Type:   layers.DHCPOptMessageType,
 				Data:   []byte{byte(layers.DHCPMsgTypeOffer)},
 				Length: 1,
-		})
+			},
+			layers.DHCPOption{
+				Type:   layers.DHCPOptLeaseTime,
+				Data:   binary.BigEndian.AppendUint32(nil, 3600),
+				Length: 4,
+			},
+			layers.DHCPOption{
+				Type:   layers.DHCPOptSubnetMask,
+				Data:   net.CIDRMask(srcNet.lanIP4.Bits(), 32),
+				Length: 4,
+			},
+			layers.DHCPOption{
+				Type:   layers.DHCPOptRouter,
+				Data:   gwIP.AsSlice(),
+				Length: 4,
+			},
+			layers.DHCPOption{
+				Type:   layers.DHCPOptDNS,
+				Data:   fakeDNS.v4.AsSlice(),
+				Length: 4,
+			},
+		)
 	case layers.DHCPMsgTypeRequest:
 		response.Options = append(response.Options,
 			layers.DHCPOption{
@@ -1591,7 +1794,7 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) {
 			},
 			layers.DHCPOption{
 				Type:   layers.DHCPOptSubnetMask,
-				Data:   net.CIDRMask(node.net.lanIP4.Bits(), 32),
+				Data:   net.CIDRMask(srcNet.lanIP4.Bits(), 32),
 				Length: 4,
 			},
 		)
@@ -1604,8 +1807,8 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) {
 	}
 	ip := &layers.IPv4{
 		Protocol: layers.IPProtocolUDP,
-		SrcIP:    ipLayer.DstIP,
+		SrcIP:    gwIP.AsSlice(),
-		DstIP:    ipLayer.SrcIP,
+		DstIP:    net.IPv4bcast, // DHCP responses are broadcast when client has no IP yet
 	}
 	udp := &layers.UDP{
 		SrcPort: udpLayer.DstPort,
@@ -1653,7 +1856,7 @@ func (s *Server) shouldInterceptTCP(pkt gopacket.Packet) bool {
 	}
 
 	if tcp.DstPort == 80 || tcp.DstPort == 443 {
-		for _, v := range []virtualIP{fakeControl, fakeDERP1, fakeDERP2, fakeLogCatcher} {
+		for _, v := range []virtualIP{fakeControl, fakeDERP1, fakeDERP2, fakeLogCatcher, fakeCloudInit, fakeFiles} {
 			if v.Match(flow.dst) {
 				return true
 			}

tstest/natlab/vnet/vnet_test.go

@@ -120,7 +120,7 @@ func TestPacketSideEffects(t *testing.T) {
 					check: all(
 						numPkts(2), // DHCP discover broadcast to node2 also, and the DHCP reply from router
 						pktSubstr("SrcMAC=52:cc:cc:cc:cc:01 DstMAC=ff:ff:ff:ff:ff:ff"),
-						pktSubstr("Options=[Option(ServerID:192.168.0.1), Option(MessageType:Offer)]}"),
+						pktSubstr("Option(ServerID:192.168.0.1), Option(MessageType:Offer), Option(LeaseTime:3600)"),
 					),
 				},
 				{