tstest/natlab: add TestSubnetRouterFreeBSD with FreeBSD cloud image support

As a warm-up to making natlab support multiple operating systems,
start with an easy one (in that it's also Unixy and open source like
Linux) and add FreeBSD 15.0 as a VM OS option for the vmtest
integration test framework, and add TestSubnetRouterFreeBSD which
tests subnet routing through a FreeBSD VM (Gokrazy → FreeBSD →
Gokrazy).

Key changes:
- Add FreeBSD150 OSImage using the official FreeBSD 15.0
  BASIC-CLOUDINIT cloud image (xz-compressed qcow2)
- Add GOOS()/IsFreeBSD() methods to OSImage for cross-compilation
  and OS-specific behavior
- Handle xz-compressed image downloads in ensureImage
- Refactor compileBinaries into compileBinariesForOS to support
  multiple GOOS targets (linux, freebsd), with binaries registered
  at <goos>/<name> paths on the file server VIP
- Add FreeBSD-specific cloud-init (nuageinit) user-data generation:
  string-form runcmd (nuageinit doesn't support YAML arrays),
  fetch(1) instead of curl, FreeBSD sysctl names for IP forwarding,
  mkdir /usr/local/bin, PATH setup for tta
- Skip network-config in cidata ISO for FreeBSD (DHCP via rc.conf)

Updates tailscale/tailscale#13038

Change-Id: Ibeb4f7d02659d5cd8e3a7c3a66ee7b1a92a0110d
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

go.mod

@@ -471,7 +471,7 @@ require (
 	github.com/tomarrell/wrapcheck/v2 v2.8.3 // indirect
 	github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
 	github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect
-	github.com/ulikunitz/xz v0.5.15 // indirect
+	github.com/ulikunitz/xz v0.5.15
 	github.com/ultraware/funlen v0.1.0 // indirect
 	github.com/ultraware/whitespace v0.1.0 // indirect
 	github.com/uudashr/gocognit v1.1.2 // indirect

tstest/natlab/vmtest/cloudinit.go

@@ -13,18 +13,22 @@ import (
 )
 
 // createCloudInitISO creates a cidata seed ISO for the given cloud VM node.
-// The ISO contains meta-data, user-data, and network-config files.
+// For Linux VMs, the ISO contains meta-data, user-data, and network-config.
-// Cloud-init reads these during init-local (pre-network), which is critical
+// For FreeBSD VMs, the ISO contains meta-data and user-data only (nuageinit
-// for network-config to take effect before systemd-networkd-wait-online runs.
+// doesn't use netplan-style network-config; DHCP is enabled in rc.conf).
 func (e *Env) createCloudInitISO(n *Node) (string, error) {
 	metaData := fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", n.name, n.name)
 	userData := e.generateUserData(n)
 
-	// Network config: DHCP all ethernet interfaces.
+	files := map[string]string{
-	// The "optional: true" prevents systemd-networkd-wait-online from blocking.
+		"meta-data": metaData,
-	// The first vnet NIC gets the default route (metric 100).
+		"user-data": userData,
-	// Other interfaces get higher metrics to avoid routing conflicts.
+	}
-	networkConfig := `version: 2
+
+	// Linux cloud-init needs network-config to configure interfaces before
+	// systemd-networkd-wait-online blocks boot.
+	if n.os.GOOS() == "linux" {
+		files["network-config"] = `version: 2
 ethernets:
   primary:
     match:
@@ -41,6 +45,7 @@ ethernets:
       route-metric: 200
     optional: true
 `
+	}
 
 	iw, err := iso9660.NewWriter()
 	if err != nil {
@@ -48,11 +53,7 @@ ethernets:
 	}
 	defer iw.Cleanup()
 
-	for name, content := range map[string]string{
+	for name, content := range files {
-		"meta-data":      metaData,
-		"user-data":      userData,
-		"network-config": networkConfig,
-	} {
 		if err := iw.AddFile(strings.NewReader(content), name); err != nil {
 			return "", fmt.Errorf("adding %s to ISO: %w", name, err)
 		}
@@ -72,6 +73,18 @@ ethernets:
 
 // generateUserData creates the cloud-init user-data (#cloud-config) for a node.
 func (e *Env) generateUserData(n *Node) string {
+	switch n.os.GOOS() {
+	case "linux":
+		return e.generateLinuxUserData(n)
+	case "freebsd":
+		return e.generateFreeBSDUserData(n)
+	default:
+		panic(fmt.Sprintf("unsupported GOOS %q for cloud-init user-data", n.os.GOOS()))
+	}
+}
+
+// generateLinuxUserData creates Linux cloud-init user-data (#cloud-config) for a node.
+func (e *Env) generateLinuxUserData(n *Node) string {
 	var ud strings.Builder
 	ud.WriteString("#cloud-config\n")
 
@@ -95,8 +108,9 @@ func (e *Env) generateUserData(n *Node) string {
 
 	// Download binaries from the files.tailscale VIP (52.52.0.6).
 	// Use the IP directly to avoid DNS resolution issues during early boot.
+	binDir := n.os.GOOS() + "_" + n.os.GOARCH()
 	for _, bin := range []string{"tailscaled", "tailscale", "tta"} {
-		fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"curl -v --retry 10 --retry-delay 2 --retry-all-errors -o /usr/local/bin/%s http://52.52.0.6/%s 2>&1\"]\n", bin, bin)
+		fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"curl -v --retry 10 --retry-delay 2 --retry-all-errors -o /usr/local/bin/%s http://52.52.0.6/%s/%s 2>&1\"]\n", bin, binDir, bin)
 	}
 	ud.WriteString("  - [\"chmod\", \"+x\", \"/usr/local/bin/tailscaled\", \"/usr/local/bin/tailscale\", \"/usr/local/bin/tta\"]\n")
 
@@ -115,3 +129,55 @@ func (e *Env) generateUserData(n *Node) string {
 
 	return ud.String()
 }
+
+// generateFreeBSDUserData creates FreeBSD nuageinit user-data (#cloud-config)
+// for a node. FreeBSD's nuageinit supports a subset of cloud-init directives
+// including runcmd, which runs after networking is up.
+//
+// IMPORTANT: nuageinit's runcmd only supports string entries, not the YAML
+// array form that Linux cloud-init supports. Each entry must be a plain string
+// that gets passed to /bin/sh -c.
+func (e *Env) generateFreeBSDUserData(n *Node) string {
+	var ud strings.Builder
+	ud.WriteString("#cloud-config\n")
+	ud.WriteString("ssh_pwauth: true\n")
+
+	ud.WriteString("runcmd:\n")
+
+	// /usr/local/bin may not exist on a fresh FreeBSD cloud image (it's
+	// created when the first package is installed).
+	ud.WriteString("  - \"mkdir -p /usr/local/bin\"\n")
+
+	// Remove the default route via the debug NIC's SLIRP gateway so that
+	// traffic goes through the vnet NICs. The debug NIC is only for SSH.
+	ud.WriteString("  - \"route delete default 10.0.2.2 2>/dev/null || true\"\n")
+
+	// Download binaries from the files.tailscale VIP (52.52.0.6).
+	// FreeBSD's fetch(1) is part of the base system (no curl needed).
+	// Retry in a loop since the file server may not be ready immediately.
+	binDir := n.os.GOOS() + "_" + n.os.GOARCH()
+	for _, bin := range []string{"tailscaled", "tailscale", "tta"} {
+		fmt.Fprintf(&ud, "  - \"n=0; while [ $n -lt 10 ]; do fetch -o /usr/local/bin/%s http://52.52.0.6/%s/%s && break; n=$((n+1)); sleep 2; done\"\n", bin, binDir, bin)
+	}
+	ud.WriteString("  - \"chmod +x /usr/local/bin/tailscaled /usr/local/bin/tailscale /usr/local/bin/tta\"\n")
+
+	// Enable IP forwarding for subnet routers.
+	// This is currently a noop as of 2026-04-08 because FreeBSD uses
+	// gvisor netstack for subnet routing until
+	// https://github.com/tailscale/tailscale/issues/5573 etc are fixed.
+	if n.advertiseRoutes != "" {
+		ud.WriteString("  - \"sysctl net.inet.ip.forwarding=1\"\n")
+		ud.WriteString("  - \"sysctl net.inet6.ip6.forwarding=1\"\n")
+	}
+
+	// Start tailscaled and tta in the background.
+	// Set PATH to include /usr/local/bin so that tta can find "tailscale"
+	// (TTA uses exec.Command("tailscale", ...) without a full path).
+	ud.WriteString("  - \"export PATH=/usr/local/bin:$PATH && /usr/local/bin/tailscaled --state=mem: &\"\n")
+	ud.WriteString("  - \"sleep 2\"\n")
+
+	// Start tta (Tailscale Test Agent).
+	ud.WriteString("  - \"export PATH=/usr/local/bin:$PATH && /usr/local/bin/tta &\"\n")
+
+	return ud.String()
+}

tstest/natlab/vmtest/images.go

@@ -14,17 +14,36 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
+
+	"github.com/ulikunitz/xz"
 )
 
 // OSImage describes a VM operating system image.
 type OSImage struct {
 	Name      string
 	URL       string // download URL for the cloud image
-	SHA256    string // expected SHA256 hash of the image
+	SHA256    string // expected SHA256 hash of the image (of the final qcow2, after any decompression)
 	MemoryMB  int    // RAM for the VM
 	IsGokrazy bool   // true for gokrazy images (different QEMU setup)
 }
 
+// GOOS returns the Go OS name for this image.
+func (img OSImage) GOOS() string {
+	if img.IsGokrazy {
+		return "linux"
+	}
+	if strings.HasPrefix(img.Name, "freebsd") {
+		return "freebsd"
+	}
+	return "linux"
+}
+
+// GOARCH returns the Go architecture name for this image.
+func (img OSImage) GOARCH() string {
+	return "amd64"
+}
+
 var (
 	// Gokrazy is a minimal Tailscale appliance image built from the gokrazy/natlabapp directory.
 	Gokrazy = OSImage{
@@ -46,6 +65,14 @@ var (
 		URL:      "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
 		MemoryMB: 1024,
 	}
+
+	// FreeBSD150 is FreeBSD 15.0-RELEASE with BASIC-CLOUDINIT (nuageinit) support.
+	// The image is distributed as xz-compressed qcow2.
+	FreeBSD150 = OSImage{
+		Name:     "freebsd-15.0",
+		URL:      "https://download.freebsd.org/releases/VM-IMAGES/15.0-RELEASE/amd64/Latest/FreeBSD-15.0-RELEASE-amd64-BASIC-CLOUDINIT-ufs.qcow2.xz",
+		MemoryMB: 1024,
+	}
 )
 
 // imageCacheDir returns the directory for cached VM images.
@@ -84,6 +111,7 @@ func ensureImage(ctx context.Context, img OSImage) error {
 		}
 	}
 
+	isXZ := strings.HasSuffix(img.URL, ".xz")
 	log.Printf("downloading %s from %s...", img.Name, img.URL)
 
 	req, err := http.NewRequestWithContext(ctx, "GET", img.URL, nil)
@@ -99,6 +127,16 @@ func ensureImage(ctx context.Context, img OSImage) error {
 		return fmt.Errorf("downloading %s: HTTP %s", img.Name, resp.Status)
 	}
 
+	// Set up the reader pipeline: HTTP body → (optional xz decompress) → file.
+	var src io.Reader = resp.Body
+	if isXZ {
+		xzr, err := xz.NewReader(resp.Body)
+		if err != nil {
+			return fmt.Errorf("creating xz reader for %s: %w", img.Name, err)
+		}
+		src = xzr
+	}
+
 	tmpFile := cachedPath + ".tmp"
 	f, err := os.Create(tmpFile)
 	if err != nil {
@@ -111,8 +149,7 @@ func ensureImage(ctx context.Context, img OSImage) error {
 
 	h := sha256.New()
 	w := io.MultiWriter(f, h)
-
+	if _, err := io.Copy(w, src); err != nil {
-	if _, err := io.Copy(w, resp.Body); err != nil {
 		return fmt.Errorf("downloading %s: %w", img.Name, err)
 	}
 	if err := f.Close(); err != nil {

tstest/natlab/vmtest/qemu.go

@@ -72,7 +72,7 @@ func (e *Env) startGokrazyQEMU(n *Node) error {
 	return e.launchQEMU(n.name, logPath, args)
 }
 
-// startCloudQEMU launches a QEMU process for a cloud image (Ubuntu, Debian, etc).
+// startCloudQEMU launches a QEMU process for a cloud image (Ubuntu, Debian, FreeBSD, etc).
 func (e *Env) startCloudQEMU(n *Node) error {
 	basePath := cachedImagePath(n.os)
 	disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name))

tstest/natlab/vmtest/vmtest.go

@@ -26,7 +26,6 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
-	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -185,17 +184,22 @@ func (e *Env) Start() {
 		t.Fatal(err)
 	}
 
-	// Determine if we have any non-gokrazy "cloud" images (e.g. Ubuntu, Debian)
+	// Determine which GOOS/GOARCH pairs need compiled binaries (non-gokrazy
-	// that require compiled binaries pushed into their image later. (Gokrazy
+	// images). Gokrazy has binaries built-in, so doesn't need compilation.
-	// has them built-in, so doesn't need the compileBinaries step.)
+	type platform struct{ goos, goarch string }
-	needBuildBinaries := slices.ContainsFunc(e.nodes, func(n *Node) bool { return !n.os.IsGokrazy })
+	needPlatform := set.Set[platform]{}
+	for _, n := range e.nodes {
+		if !n.os.IsGokrazy {
+			needPlatform.Add(platform{n.os.GOOS(), n.os.GOARCH()})
+		}
+	}
 
 	// Compile binaries and download/build images in parallel.
 	// Any failure cancels the others via the errgroup context.
 	eg, egCtx := errgroup.WithContext(ctx)
-	if needBuildBinaries {
+	for _, p := range needPlatform.Slice() {
 		eg.Go(func() error {
-			return e.compileBinaries(egCtx)
+			return e.compileBinariesForOS(egCtx, p.goos, p.goarch)
 		})
 	}
 	didOS := set.Set[string]{} // dedup by image name
@@ -227,13 +231,15 @@ func (e *Env) Start() {
 	t.Cleanup(func() { e.server.Close() })
 
 	// Register compiled binaries with the file server VIP.
-	if needBuildBinaries {
+	// Binaries are registered at <goos>_<goarch>/<name> (e.g. "linux_amd64/tta").
+	for _, p := range needPlatform.Slice() {
+		dir := p.goos + "_" + p.goarch
 		for _, name := range []string{"tta", "tailscale", "tailscaled"} {
-			data, err := os.ReadFile(filepath.Join(e.binDir, name))
+			data, err := os.ReadFile(filepath.Join(e.binDir, dir, name))
 			if err != nil {
-				t.Fatalf("reading compiled %s: %v", name, err)
+				t.Fatalf("reading compiled %s/%s: %v", dir, name, err)
 			}
-			e.server.RegisterFile(name, data)
+			e.server.RegisterFile(dir+"/"+name, data)
 		}
 	}
 
@@ -603,14 +609,20 @@ func (e *Env) ensureGokrazy(ctx context.Context) error {
 	return nil
 }
 
-// compileBinaries cross-compiles tta, tailscale, and tailscaled for linux/amd64
+// compileBinariesForOS cross-compiles tta, tailscale, and tailscaled for the
-// and places them in e.binDir.
+// given GOOS/GOARCH and places them in e.binDir/<goos>_<goarch>/.
-func (e *Env) compileBinaries(ctx context.Context) error {
+func (e *Env) compileBinariesForOS(ctx context.Context, goos, goarch string) error {
 	modRoot, err := findModRoot()
 	if err != nil {
 		return err
 	}
 
+	dir := goos + "_" + goarch
+	outDir := filepath.Join(e.binDir, dir)
+	if err := os.MkdirAll(outDir, 0755); err != nil {
+		return err
+	}
+
 	binaries := []struct{ name, pkg string }{
 		{"tta", "./cmd/tta"},
 		{"tailscale", "./cmd/tailscale"},
@@ -620,15 +632,15 @@ func (e *Env) compileBinaries(ctx context.Context) error {
 	var eg errgroup.Group
 	for _, bin := range binaries {
 		eg.Go(func() error {
-			outPath := filepath.Join(e.binDir, bin.name)
+			outPath := filepath.Join(outDir, bin.name)
-			e.t.Logf("compiling %s...", bin.name)
+			e.t.Logf("compiling %s/%s...", dir, bin.name)
 			cmd := exec.CommandContext(ctx, "go", "build", "-o", outPath, bin.pkg)
 			cmd.Dir = modRoot
-			cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0")
+			cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch, "CGO_ENABLED=0")
 			if out, err := cmd.CombinedOutput(); err != nil {
-				return fmt.Errorf("building %s: %v\n%s", bin.name, err, out)
+				return fmt.Errorf("building %s/%s: %v\n%s", dir, bin.name, err, out)
 			}
-			e.t.Logf("compiled %s", bin.name)
+			e.t.Logf("compiled %s/%s", dir, bin.name)
 			return nil
 		})
 	}

tstest/natlab/vmtest/vmtest_test.go

@@ -13,6 +13,15 @@ import (
 )
 
 func TestSubnetRouter(t *testing.T) {
+	testSubnetRouterForOS(t, vmtest.Ubuntu2404)
+}
+
+func TestSubnetRouterFreeBSD(t *testing.T) {
+	testSubnetRouterForOS(t, vmtest.FreeBSD150)
+}
+
+func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) {
+	t.Helper()
 	env := vmtest.New(t)
 
 	clientNet := env.AddNetwork("2.1.1.1", "192.168.1.1/24", "2000:1::1/64", vnet.EasyNAT)
@@ -21,7 +30,7 @@ func TestSubnetRouter(t *testing.T) {
 	client := env.AddNode("client", clientNet,
 		vmtest.OS(vmtest.Gokrazy))
 	sr := env.AddNode("subnet-router", clientNet, internalNet,
-		vmtest.OS(vmtest.Ubuntu2404),
+		vmtest.OS(srOS),
 		vmtest.AdvertiseRoutes("10.0.0.0/24"))
 	backend := env.AddNode("backend", internalNet,
 		vmtest.OS(vmtest.Gokrazy),