diff --git a/tstest/natlab/vnet/conf.go b/tstest/natlab/vnet/conf.go index ccccb8e21..64f28fbc9 100644 --- a/tstest/natlab/vnet/conf.go +++ b/tstest/natlab/vnet/conf.go @@ -72,6 +72,13 @@ func nodeMac(n int) MAC { return MAC{0x52, 0xcc, 0xcc, 0xcc, 0xcc, byte(n)} } +// nodeNICMac returns the MAC for the nicIdx-th secondary NIC (1-indexed) of node n. +// Primary NICs (index 0) use nodeMac. Secondary NICs use a different scheme: +// 52:cc:cc:cc:KK:NN where KK is the NIC index and NN is the node number. +func nodeNICMac(nodeNum, nicIdx int) MAC { + return MAC{0x52, 0xcc, 0xcc, 0xcc, byte(nicIdx), byte(nodeNum)} +} + func routerMac(n int) MAC { // 52=TS then 0xee for 'etwork return MAC{0x52, 0xee, 0xee, 0xee, 0xee, byte(n)} @@ -236,11 +243,31 @@ func (n *Node) String() string { return fmt.Sprintf("node%d", n.num) } -// MAC returns the MAC address of the node. +// MAC returns the MAC address of the node's primary NIC. func (n *Node) MAC() MAC { return n.mac } +// NumNICs returns the number of network interfaces on the node +// (one per network the node is on). +func (n *Node) NumNICs() int { + return len(n.nets) +} + +// NICMac returns the MAC address for the i-th NIC (0-indexed). +// NIC 0 is the primary NIC (same as MAC()). NIC 1+ are extra NICs. +func (n *Node) NICMac(i int) MAC { + if i == 0 { + return n.mac + } + return nodeNICMac(n.num, i) +} + +// Networks returns the list of networks this node is on. +func (n *Node) Networks() []*Network { + return n.nets +} + func (n *Node) Env() []TailscaledEnv { return n.env } @@ -306,6 +333,26 @@ func (n *Node) IsV6Only() bool { return false } +// LanIP returns the node's LAN IPv4 address on the given network. +// It requires the [Server] to have been initialized (i.e., [New] was called). +// Returns an invalid addr if the node has no IP on that network. +func (n *Node) LanIP(net *Network) netip.Addr { + if n.n == nil { + return netip.Addr{} + } + for i, nn := range n.nets { + if nn == net { + if i == 0 { + return n.n.lanIP + } + if i-1 < len(n.n.extraNICs) { + return n.n.extraNICs[i-1].lanIP + } + } + } + return netip.Addr{} +} + // Network returns the first network this node is connected to, // or nil if none. func (n *Node) Network() *Network { @@ -486,10 +533,11 @@ func (s *Server) initFromConfig(c *Config) error { if conf.err != nil { return conf.err } + primaryNet := netOfConf[conf.Network()] n := &node{ num: conf.num, mac: conf.mac, - net: netOfConf[conf.Network()], + net: primaryNet, verboseSyslog: conf.VerboseSyslog(), } n.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{ @@ -503,16 +551,50 @@ func (s *Server) initFromConfig(c *Config) error { s.nodes = append(s.nodes, n) s.nodeByMAC[n.mac] = n - if n.net.v4 { + if n.net != nil && n.net.v4 { // Allocate a lanIP for the node. Use the network's CIDR and use final // octet 101 (for first node), 102, etc. The node number comes from the - // last octent of the MAC address (0-based) + // last octet of the MAC address (0-based) ip4 := n.net.lanIP4.Addr().As4() ip4[3] = 100 + n.mac[5] n.lanIP = netip.AddrFrom4(ip4) n.net.nodesByIP4[n.lanIP] = n } - n.net.nodesByMAC[n.mac] = n + if n.net != nil { + n.net.nodesByMAC[n.mac] = n + } + + // Set up extra NICs for multi-homed nodes (nodes on more than one network). + for nicIdx, confNet := range conf.nets[1:] { + extraNet := netOfConf[confNet] + if extraNet == nil { + continue + } + mac := nodeNICMac(conf.num, nicIdx+1) + nic := nodeNIC{ + mac: mac, + net: extraNet, + } + nic.interfaceID = must.Get(s.pcapWriter.AddInterface(pcapgo.NgInterface{ + Name: fmt.Sprintf("%s-nic%d", n.String(), nicIdx+1), + LinkType: layers.LinkTypeEthernet, + })) + // Allocate a lanIP for the node. Use the network's CIDR and use final + // octet 101 (for first node), 102, etc. The node number comes from the + // last octet of the MAC address (0-based) + if extraNet.v4 { + ip4 := extraNet.lanIP4.Addr().As4() + ip4[3] = 100 + mac[5] + nic.lanIP = netip.AddrFrom4(ip4) + extraNet.nodesByIP4[nic.lanIP] = n + } + extraNet.nodesByMAC[mac] = n + if _, ok := s.nodeByMAC[mac]; ok { + return fmt.Errorf("two nodes have the same MAC %v", mac) + } + s.nodeByMAC[mac] = n + n.extraNICs = append(n.extraNICs, nic) + } } // Now that nodes are populated, set up NAT: diff --git a/tstest/natlab/vnet/vip.go b/tstest/natlab/vnet/vip.go index 9d7aa56a3..a6973ed50 100644 --- a/tstest/natlab/vnet/vip.go +++ b/tstest/natlab/vnet/vip.go @@ -19,6 +19,8 @@ var ( fakeDERP2 = newVIP("derp2.tailscale", "33.4.0.2") // 3340=DERP; 2=derp 2 fakeLogCatcher = newVIP("log.tailscale.com", 4) fakeSyslog = newVIP("syslog.tailscale", 9) + fakeCloudInit = newVIP("cloud-init.tailscale", 5) // serves cloud-init metadata/userdata per node + fakeFiles = newVIP("files.tailscale", 6) // serves binary files (tta, tailscale, tailscaled) to VMs ) type virtualIP struct { diff --git a/tstest/natlab/vnet/vnet.go b/tstest/natlab/vnet/vnet.go index 9eb81520c..2365d03eb 100644 --- a/tstest/natlab/vnet/vnet.go +++ b/tstest/natlab/vnet/vnet.go @@ -30,6 +30,7 @@ import ( "net/netip" "os/exec" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -267,10 +268,13 @@ func (n *network) handleIPPacketFromGvisor(ipRaw []byte) { n.logf("gvisor: serialize error: %v", err) return } - if nw, ok := n.writers.Load(node.mac); ok { + // Use the MAC address for this specific network (important for multi-NIC nodes + // where the primary MAC may be on a different network). + mac := node.macForNet(n) + if nw, ok := n.writers.Load(mac); ok { nw.write(resPkt) } else { - n.logf("gvisor write: no writeFunc for %v", node.mac) + n.logf("gvisor write: no writeFunc for %v (node %v on net %v)", mac, node, n.mac) } } @@ -371,6 +375,22 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) { return } + if destPort == 80 && fakeCloudInit.Match(destIP) { + r.Complete(false) + tc := gonet.NewTCPConn(&wq, ep) + hs := &http.Server{Handler: n.s.cloudInitHandler()} + go hs.Serve(netutil.NewOneConnListener(tc, nil)) + return + } + + if destPort == 80 && fakeFiles.Match(destIP) { + r.Complete(false) + tc := gonet.NewTCPConn(&wq, ep) + hs := &http.Server{Handler: n.s.fileServerHandler()} + go hs.Serve(netutil.NewOneConnListener(tc, nil)) + return + } + var targetDial string if n.s.derpIPs.Contains(destIP) { targetDial = destIP.String() + ":" + strconv.Itoa(int(destPort)) @@ -573,8 +593,10 @@ func (n *network) MACOfIP(ip netip.Addr) (_ MAC, ok bool) { if n.lanIP4.Addr() == ip { return n.mac, true } - if n, ok := n.nodesByIP4[ip]; ok { - return n.mac, true + if node, ok := n.nodesByIP4[ip]; ok { + // Use the MAC for this specific network (important for multi-NIC nodes + // where the primary MAC may be on a different network). + return node.macForNet(n), true } return MAC{}, false } @@ -585,6 +607,15 @@ func (n *network) SetControlBlackholed(v bool) { n.blackholeControl = v } +// nodeNIC represents a single network interface on a node. +// For multi-homed nodes, additional NICs beyond the primary are stored in node.extraNICs. +type nodeNIC struct { + mac MAC + net *network + lanIP netip.Addr + interfaceID int +} + type node struct { mac MAC num int // 1-based node number @@ -593,6 +624,8 @@ type node struct { lanIP netip.Addr // must be in net.lanIP prefix + unique in net verboseSyslog bool + extraNICs []nodeNIC // secondary NICs for multi-homed nodes + // logMu guards logBuf. // TODO(bradfitz): conditionally write these out to separate files at the end? // Currently they only hold logcatcher logs. @@ -601,6 +634,35 @@ type node struct { logCatcherWrites int } +// netForMAC returns the network associated with the given MAC address on this node. +// It checks the primary NIC first, then any extra NICs. +func (n *node) netForMAC(mac MAC) *network { + if mac == n.mac { + return n.net + } + for _, nic := range n.extraNICs { + if nic.mac == mac { + return nic.net + } + } + return nil +} + +// macForNet returns the MAC address that this node uses on the given network. +// For the primary network, this is node.mac. For secondary networks, it's the +// extra NIC's MAC. +func (n *node) macForNet(net *network) MAC { + if n.net == net { + return n.mac + } + for _, nic := range n.extraNICs { + if nic.net == net { + return nic.mac + } + } + return n.mac // fallback to primary +} + // String returns the string "nodeN" where N is the 1-based node number. func (n *node) String() string { return fmt.Sprintf("node%d", n.num) @@ -657,6 +719,9 @@ type Server struct { agentConnWaiter map[*node]chan<- struct{} // signaled after added to set agentConns set.Set[*agentConn] // not keyed by node; should be small/cheap enough to scan all agentDialer map[*node]netx.DialFunc + + cloudInitData map[int]*CloudInitData // node num → cloud-init config + fileContents map[string][]byte // filename → file bytes } func (s *Server) logf(format string, args ...any) { @@ -741,6 +806,96 @@ func New(c *Config) (*Server, error) { return s, nil } +// ControlServer returns the test control server used by this vnet. +func (s *Server) ControlServer() *testcontrol.Server { + return s.control +} + +// CloudInitData holds the cloud-init configuration for a node. +type CloudInitData struct { + MetaData string + UserData string + NetworkConfig string // optional; if set, served as network-config +} + +// SetCloudInitData registers cloud-init configuration for the given node number. +// This data is served via the cloud-init.tailscale VIP when the VM boots. +func (s *Server) SetCloudInitData(nodeNum int, data *CloudInitData) { + s.mu.Lock() + defer s.mu.Unlock() + mak.Set(&s.cloudInitData, nodeNum, data) +} + +// RegisterFile registers a file to be served by the files.tailscale VIP. +// The path is the URL path (e.g., "tta" is served at http://files.tailscale/tta). +func (s *Server) RegisterFile(path string, data []byte) { + s.mu.Lock() + defer s.mu.Unlock() + mak.Set(&s.fileContents, path, data) +} + +// cloudInitHandler returns an HTTP handler that serves cloud-init +// meta-data and user-data for VMs that boot with +// ds=nocloud;s=http://cloud-init.tailscale/node-N/. +func (s *Server) cloudInitHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Parse node number from URL path like "/node-2/meta-data" + path := strings.TrimPrefix(r.URL.Path, "/") + parts := strings.SplitN(path, "/", 2) + if len(parts) != 2 { + http.Error(w, "bad path", http.StatusNotFound) + return + } + nodeNum := 0 + if _, err := fmt.Sscanf(parts[0], "node-%d", &nodeNum); err != nil { + http.Error(w, "bad node number", http.StatusNotFound) + return + } + s.mu.Lock() + data := s.cloudInitData[nodeNum] + s.mu.Unlock() + if data == nil { + http.Error(w, "no cloud-init data for node", http.StatusNotFound) + return + } + switch parts[1] { + case "meta-data": + w.Header().Set("Content-Type", "text/yaml") + io.WriteString(w, data.MetaData) + case "user-data": + w.Header().Set("Content-Type", "text/yaml") + io.WriteString(w, data.UserData) + case "network-config": + if data.NetworkConfig == "" { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "text/yaml") + io.WriteString(w, data.NetworkConfig) + default: + http.Error(w, "not found", http.StatusNotFound) + } + }) +} + +// fileServerHandler returns an HTTP handler that serves files registered +// via RegisterFile. Files are served at http://files.tailscale/. +func (s *Server) fileServerHandler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/") + s.mu.Lock() + data, ok := s.fileContents[path] + s.mu.Unlock() + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + w.Write(data) + }) +} + func (s *Server) Close() { if shutdown := s.shuttingDown.Swap(true); !shutdown { s.shutdownCancel() @@ -905,9 +1060,14 @@ func (s *Server) ServeUnixConn(uc *net.UnixConn, proto Protocol) { } if !didReg[srcMAC] { didReg[srcMAC] = true + srcNet := srcNode.netForMAC(srcMAC) + if srcNet == nil { + s.logf("[conn %p] node %v has no network for MAC %v", c.uc, srcNode, srcMAC) + continue + } s.logf("[conn %p] Registering writer for MAC %v, node %v", c.uc, srcMAC, srcNode.lanIP) - srcNode.net.registerWriter(srcMAC, c) - defer srcNode.net.unregisterWriter(srcMAC) + srcNet.registerWriter(srcMAC, c) + defer srcNet.unregisterWriter(srcMAC) } if err := s.handleEthernetFrameFromVM(packetRaw); err != nil { @@ -930,13 +1090,18 @@ func (s *Server) handleEthernetFrameFromVM(packetRaw []byte) error { return fmt.Errorf("got frame from unknown MAC %v", srcMAC) } + srcNet := srcNode.netForMAC(srcMAC) + if srcNet == nil { + return fmt.Errorf("node %v has no network for MAC %v", srcNode, srcMAC) + } + must.Do(s.pcapWriter.WritePacket(gopacket.CaptureInfo{ Timestamp: time.Now(), CaptureLength: len(packetRaw), Length: len(packetRaw), InterfaceIndex: srcNode.interfaceID, }, packetRaw)) - srcNode.net.HandleEthernetPacket(ep) + srcNet.HandleEthernetPacket(ep) return nil } @@ -1191,8 +1356,8 @@ func (n *network) WriteUDPPacketNoNAT(p UDPPacket) { } eth := &layers.Ethernet{ - SrcMAC: n.mac.HWAddr(), // of gateway - DstMAC: node.mac.HWAddr(), + SrcMAC: n.mac.HWAddr(), // of gateway; on the specific network + DstMAC: node.macForNet(n).HWAddr(), // use the MAC for this network } ethRaw, err := n.serializedUDPPacket(src, dst, p.Payload, eth) if err != nil { @@ -1531,12 +1696,28 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { log.Printf("DHCP request from unknown node %v; ignoring", srcMAC) return nil, nil } - gwIP := node.net.lanIP4.Addr() + // Use the network associated with this MAC (important for multi-NIC nodes). + srcNet := node.netForMAC(srcMAC) + if srcNet == nil { + log.Printf("DHCP request from MAC %v with no associated network; ignoring", srcMAC) + return nil, nil + } + gwIP := srcNet.lanIP4.Addr() - ipLayer := request.Layer(layers.LayerTypeIPv4).(*layers.IPv4) udpLayer := request.Layer(layers.LayerTypeUDP).(*layers.UDP) dhcpLayer := request.Layer(layers.LayerTypeDHCPv4).(*layers.DHCPv4) + // Determine the client's LAN IP for this specific NIC. + clientIP := node.lanIP + if srcMAC != node.mac { + for _, nic := range node.extraNICs { + if nic.mac == srcMAC { + clientIP = nic.lanIP + break + } + } + } + response := &layers.DHCPv4{ Operation: layers.DHCPOpReply, HardwareType: layers.LinkTypeEthernet, @@ -1544,7 +1725,7 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { Xid: dhcpLayer.Xid, ClientHWAddr: dhcpLayer.ClientHWAddr, Flags: dhcpLayer.Flags, - YourClientIP: node.lanIP.AsSlice(), + YourClientIP: clientIP.AsSlice(), Options: []layers.DHCPOption{ { Type: layers.DHCPOptServerID, @@ -1562,11 +1743,33 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { } switch msgType { case layers.DHCPMsgTypeDiscover: - response.Options = append(response.Options, layers.DHCPOption{ - Type: layers.DHCPOptMessageType, - Data: []byte{byte(layers.DHCPMsgTypeOffer)}, - Length: 1, - }) + response.Options = append(response.Options, + layers.DHCPOption{ + Type: layers.DHCPOptMessageType, + Data: []byte{byte(layers.DHCPMsgTypeOffer)}, + Length: 1, + }, + layers.DHCPOption{ + Type: layers.DHCPOptLeaseTime, + Data: binary.BigEndian.AppendUint32(nil, 3600), + Length: 4, + }, + layers.DHCPOption{ + Type: layers.DHCPOptSubnetMask, + Data: net.CIDRMask(srcNet.lanIP4.Bits(), 32), + Length: 4, + }, + layers.DHCPOption{ + Type: layers.DHCPOptRouter, + Data: gwIP.AsSlice(), + Length: 4, + }, + layers.DHCPOption{ + Type: layers.DHCPOptDNS, + Data: fakeDNS.v4.AsSlice(), + Length: 4, + }, + ) case layers.DHCPMsgTypeRequest: response.Options = append(response.Options, layers.DHCPOption{ @@ -1591,7 +1794,7 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { }, layers.DHCPOption{ Type: layers.DHCPOptSubnetMask, - Data: net.CIDRMask(node.net.lanIP4.Bits(), 32), + Data: net.CIDRMask(srcNet.lanIP4.Bits(), 32), Length: 4, }, ) @@ -1604,8 +1807,8 @@ func (s *Server) createDHCPResponse(request gopacket.Packet) ([]byte, error) { } ip := &layers.IPv4{ Protocol: layers.IPProtocolUDP, - SrcIP: ipLayer.DstIP, - DstIP: ipLayer.SrcIP, + SrcIP: gwIP.AsSlice(), + DstIP: net.IPv4bcast, // DHCP responses are broadcast when client has no IP yet } udp := &layers.UDP{ SrcPort: udpLayer.DstPort, @@ -1653,7 +1856,7 @@ func (s *Server) shouldInterceptTCP(pkt gopacket.Packet) bool { } if tcp.DstPort == 80 || tcp.DstPort == 443 { - for _, v := range []virtualIP{fakeControl, fakeDERP1, fakeDERP2, fakeLogCatcher} { + for _, v := range []virtualIP{fakeControl, fakeDERP1, fakeDERP2, fakeLogCatcher, fakeCloudInit, fakeFiles} { if v.Match(flow.dst) { return true } diff --git a/tstest/natlab/vnet/vnet_test.go b/tstest/natlab/vnet/vnet_test.go index 93f208c29..9d7c78c45 100644 --- a/tstest/natlab/vnet/vnet_test.go +++ b/tstest/natlab/vnet/vnet_test.go @@ -120,7 +120,7 @@ func TestPacketSideEffects(t *testing.T) { check: all( numPkts(2), // DHCP discover broadcast to node2 also, and the DHCP reply from router pktSubstr("SrcMAC=52:cc:cc:cc:cc:01 DstMAC=ff:ff:ff:ff:ff:ff"), - pktSubstr("Options=[Option(ServerID:192.168.0.1), Option(MessageType:Offer)]}"), + pktSubstr("Option(ServerID:192.168.0.1), Option(MessageType:Offer), Option(LeaseTime:3600)"), ), }, {