diff --git a/go.mod b/go.mod index fdaab6db2..5c421e167 100644 --- a/go.mod +++ b/go.mod @@ -471,7 +471,7 @@ require ( github.com/tomarrell/wrapcheck/v2 v2.8.3 // indirect github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 // indirect - github.com/ulikunitz/xz v0.5.15 // indirect + github.com/ulikunitz/xz v0.5.15 github.com/ultraware/funlen v0.1.0 // indirect github.com/ultraware/whitespace v0.1.0 // indirect github.com/uudashr/gocognit v1.1.2 // indirect diff --git a/tstest/natlab/vmtest/cloudinit.go b/tstest/natlab/vmtest/cloudinit.go index 41b9ae10e..334863f9c 100644 --- a/tstest/natlab/vmtest/cloudinit.go +++ b/tstest/natlab/vmtest/cloudinit.go @@ -13,18 +13,22 @@ import ( ) // createCloudInitISO creates a cidata seed ISO for the given cloud VM node. -// The ISO contains meta-data, user-data, and network-config files. -// Cloud-init reads these during init-local (pre-network), which is critical -// for network-config to take effect before systemd-networkd-wait-online runs. +// For Linux VMs, the ISO contains meta-data, user-data, and network-config. +// For FreeBSD VMs, the ISO contains meta-data and user-data only (nuageinit +// doesn't use netplan-style network-config; DHCP is enabled in rc.conf). func (e *Env) createCloudInitISO(n *Node) (string, error) { metaData := fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", n.name, n.name) userData := e.generateUserData(n) - // Network config: DHCP all ethernet interfaces. - // The "optional: true" prevents systemd-networkd-wait-online from blocking. - // The first vnet NIC gets the default route (metric 100). - // Other interfaces get higher metrics to avoid routing conflicts. - networkConfig := `version: 2 + files := map[string]string{ + "meta-data": metaData, + "user-data": userData, + } + + // Linux cloud-init needs network-config to configure interfaces before + // systemd-networkd-wait-online blocks boot. + if n.os.GOOS() == "linux" { + files["network-config"] = `version: 2 ethernets: primary: match: @@ -41,6 +45,7 @@ ethernets: route-metric: 200 optional: true ` + } iw, err := iso9660.NewWriter() if err != nil { @@ -48,11 +53,7 @@ ethernets: } defer iw.Cleanup() - for name, content := range map[string]string{ - "meta-data": metaData, - "user-data": userData, - "network-config": networkConfig, - } { + for name, content := range files { if err := iw.AddFile(strings.NewReader(content), name); err != nil { return "", fmt.Errorf("adding %s to ISO: %w", name, err) } @@ -72,6 +73,18 @@ ethernets: // generateUserData creates the cloud-init user-data (#cloud-config) for a node. func (e *Env) generateUserData(n *Node) string { + switch n.os.GOOS() { + case "linux": + return e.generateLinuxUserData(n) + case "freebsd": + return e.generateFreeBSDUserData(n) + default: + panic(fmt.Sprintf("unsupported GOOS %q for cloud-init user-data", n.os.GOOS())) + } +} + +// generateLinuxUserData creates Linux cloud-init user-data (#cloud-config) for a node. +func (e *Env) generateLinuxUserData(n *Node) string { var ud strings.Builder ud.WriteString("#cloud-config\n") @@ -95,8 +108,9 @@ func (e *Env) generateUserData(n *Node) string { // Download binaries from the files.tailscale VIP (52.52.0.6). // Use the IP directly to avoid DNS resolution issues during early boot. + binDir := n.os.GOOS() + "_" + n.os.GOARCH() for _, bin := range []string{"tailscaled", "tailscale", "tta"} { - fmt.Fprintf(&ud, " - [\"/bin/sh\", \"-c\", \"curl -v --retry 10 --retry-delay 2 --retry-all-errors -o /usr/local/bin/%s http://52.52.0.6/%s 2>&1\"]\n", bin, bin) + fmt.Fprintf(&ud, " - [\"/bin/sh\", \"-c\", \"curl -v --retry 10 --retry-delay 2 --retry-all-errors -o /usr/local/bin/%s http://52.52.0.6/%s/%s 2>&1\"]\n", bin, binDir, bin) } ud.WriteString(" - [\"chmod\", \"+x\", \"/usr/local/bin/tailscaled\", \"/usr/local/bin/tailscale\", \"/usr/local/bin/tta\"]\n") @@ -115,3 +129,55 @@ func (e *Env) generateUserData(n *Node) string { return ud.String() } + +// generateFreeBSDUserData creates FreeBSD nuageinit user-data (#cloud-config) +// for a node. FreeBSD's nuageinit supports a subset of cloud-init directives +// including runcmd, which runs after networking is up. +// +// IMPORTANT: nuageinit's runcmd only supports string entries, not the YAML +// array form that Linux cloud-init supports. Each entry must be a plain string +// that gets passed to /bin/sh -c. +func (e *Env) generateFreeBSDUserData(n *Node) string { + var ud strings.Builder + ud.WriteString("#cloud-config\n") + ud.WriteString("ssh_pwauth: true\n") + + ud.WriteString("runcmd:\n") + + // /usr/local/bin may not exist on a fresh FreeBSD cloud image (it's + // created when the first package is installed). + ud.WriteString(" - \"mkdir -p /usr/local/bin\"\n") + + // Remove the default route via the debug NIC's SLIRP gateway so that + // traffic goes through the vnet NICs. The debug NIC is only for SSH. + ud.WriteString(" - \"route delete default 10.0.2.2 2>/dev/null || true\"\n") + + // Download binaries from the files.tailscale VIP (52.52.0.6). + // FreeBSD's fetch(1) is part of the base system (no curl needed). + // Retry in a loop since the file server may not be ready immediately. + binDir := n.os.GOOS() + "_" + n.os.GOARCH() + for _, bin := range []string{"tailscaled", "tailscale", "tta"} { + fmt.Fprintf(&ud, " - \"n=0; while [ $n -lt 10 ]; do fetch -o /usr/local/bin/%s http://52.52.0.6/%s/%s && break; n=$((n+1)); sleep 2; done\"\n", bin, binDir, bin) + } + ud.WriteString(" - \"chmod +x /usr/local/bin/tailscaled /usr/local/bin/tailscale /usr/local/bin/tta\"\n") + + // Enable IP forwarding for subnet routers. + // This is currently a noop as of 2026-04-08 because FreeBSD uses + // gvisor netstack for subnet routing until + // https://github.com/tailscale/tailscale/issues/5573 etc are fixed. + if n.advertiseRoutes != "" { + ud.WriteString(" - \"sysctl net.inet.ip.forwarding=1\"\n") + ud.WriteString(" - \"sysctl net.inet6.ip6.forwarding=1\"\n") + } + + // Start tailscaled and tta in the background. + // Set PATH to include /usr/local/bin so that tta can find "tailscale" + // (TTA uses exec.Command("tailscale", ...) without a full path). + ud.WriteString(" - \"export PATH=/usr/local/bin:$PATH && /usr/local/bin/tailscaled --state=mem: &\"\n") + ud.WriteString(" - \"sleep 2\"\n") + + // Start tta (Tailscale Test Agent). + ud.WriteString(" - \"export PATH=/usr/local/bin:$PATH && /usr/local/bin/tta &\"\n") + + return ud.String() +} diff --git a/tstest/natlab/vmtest/images.go b/tstest/natlab/vmtest/images.go index 54e7eb577..49eba443f 100644 --- a/tstest/natlab/vmtest/images.go +++ b/tstest/natlab/vmtest/images.go @@ -14,17 +14,36 @@ import ( "os" "os/exec" "path/filepath" + "strings" + + "github.com/ulikunitz/xz" ) // OSImage describes a VM operating system image. type OSImage struct { Name string URL string // download URL for the cloud image - SHA256 string // expected SHA256 hash of the image + SHA256 string // expected SHA256 hash of the image (of the final qcow2, after any decompression) MemoryMB int // RAM for the VM IsGokrazy bool // true for gokrazy images (different QEMU setup) } +// GOOS returns the Go OS name for this image. +func (img OSImage) GOOS() string { + if img.IsGokrazy { + return "linux" + } + if strings.HasPrefix(img.Name, "freebsd") { + return "freebsd" + } + return "linux" +} + +// GOARCH returns the Go architecture name for this image. +func (img OSImage) GOARCH() string { + return "amd64" +} + var ( // Gokrazy is a minimal Tailscale appliance image built from the gokrazy/natlabapp directory. Gokrazy = OSImage{ @@ -46,6 +65,14 @@ var ( URL: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2", MemoryMB: 1024, } + + // FreeBSD150 is FreeBSD 15.0-RELEASE with BASIC-CLOUDINIT (nuageinit) support. + // The image is distributed as xz-compressed qcow2. + FreeBSD150 = OSImage{ + Name: "freebsd-15.0", + URL: "https://download.freebsd.org/releases/VM-IMAGES/15.0-RELEASE/amd64/Latest/FreeBSD-15.0-RELEASE-amd64-BASIC-CLOUDINIT-ufs.qcow2.xz", + MemoryMB: 1024, + } ) // imageCacheDir returns the directory for cached VM images. @@ -84,6 +111,7 @@ func ensureImage(ctx context.Context, img OSImage) error { } } + isXZ := strings.HasSuffix(img.URL, ".xz") log.Printf("downloading %s from %s...", img.Name, img.URL) req, err := http.NewRequestWithContext(ctx, "GET", img.URL, nil) @@ -99,6 +127,16 @@ func ensureImage(ctx context.Context, img OSImage) error { return fmt.Errorf("downloading %s: HTTP %s", img.Name, resp.Status) } + // Set up the reader pipeline: HTTP body → (optional xz decompress) → file. + var src io.Reader = resp.Body + if isXZ { + xzr, err := xz.NewReader(resp.Body) + if err != nil { + return fmt.Errorf("creating xz reader for %s: %w", img.Name, err) + } + src = xzr + } + tmpFile := cachedPath + ".tmp" f, err := os.Create(tmpFile) if err != nil { @@ -111,8 +149,7 @@ func ensureImage(ctx context.Context, img OSImage) error { h := sha256.New() w := io.MultiWriter(f, h) - - if _, err := io.Copy(w, resp.Body); err != nil { + if _, err := io.Copy(w, src); err != nil { return fmt.Errorf("downloading %s: %w", img.Name, err) } if err := f.Close(); err != nil { diff --git a/tstest/natlab/vmtest/qemu.go b/tstest/natlab/vmtest/qemu.go index 03d424010..df56322fa 100644 --- a/tstest/natlab/vmtest/qemu.go +++ b/tstest/natlab/vmtest/qemu.go @@ -72,7 +72,7 @@ func (e *Env) startGokrazyQEMU(n *Node) error { return e.launchQEMU(n.name, logPath, args) } -// startCloudQEMU launches a QEMU process for a cloud image (Ubuntu, Debian, etc). +// startCloudQEMU launches a QEMU process for a cloud image (Ubuntu, Debian, FreeBSD, etc). func (e *Env) startCloudQEMU(n *Node) error { basePath := cachedImagePath(n.os) disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name)) diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index f028a7c6b..e6c89467f 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -26,7 +26,6 @@ import ( "os" "os/exec" "path/filepath" - "slices" "strings" "testing" "time" @@ -185,17 +184,22 @@ func (e *Env) Start() { t.Fatal(err) } - // Determine if we have any non-gokrazy "cloud" images (e.g. Ubuntu, Debian) - // that require compiled binaries pushed into their image later. (Gokrazy - // has them built-in, so doesn't need the compileBinaries step.) - needBuildBinaries := slices.ContainsFunc(e.nodes, func(n *Node) bool { return !n.os.IsGokrazy }) + // Determine which GOOS/GOARCH pairs need compiled binaries (non-gokrazy + // images). Gokrazy has binaries built-in, so doesn't need compilation. + type platform struct{ goos, goarch string } + needPlatform := set.Set[platform]{} + for _, n := range e.nodes { + if !n.os.IsGokrazy { + needPlatform.Add(platform{n.os.GOOS(), n.os.GOARCH()}) + } + } // Compile binaries and download/build images in parallel. // Any failure cancels the others via the errgroup context. eg, egCtx := errgroup.WithContext(ctx) - if needBuildBinaries { + for _, p := range needPlatform.Slice() { eg.Go(func() error { - return e.compileBinaries(egCtx) + return e.compileBinariesForOS(egCtx, p.goos, p.goarch) }) } didOS := set.Set[string]{} // dedup by image name @@ -227,13 +231,15 @@ func (e *Env) Start() { t.Cleanup(func() { e.server.Close() }) // Register compiled binaries with the file server VIP. - if needBuildBinaries { + // Binaries are registered at _/ (e.g. "linux_amd64/tta"). + for _, p := range needPlatform.Slice() { + dir := p.goos + "_" + p.goarch for _, name := range []string{"tta", "tailscale", "tailscaled"} { - data, err := os.ReadFile(filepath.Join(e.binDir, name)) + data, err := os.ReadFile(filepath.Join(e.binDir, dir, name)) if err != nil { - t.Fatalf("reading compiled %s: %v", name, err) + t.Fatalf("reading compiled %s/%s: %v", dir, name, err) } - e.server.RegisterFile(name, data) + e.server.RegisterFile(dir+"/"+name, data) } } @@ -603,14 +609,20 @@ func (e *Env) ensureGokrazy(ctx context.Context) error { return nil } -// compileBinaries cross-compiles tta, tailscale, and tailscaled for linux/amd64 -// and places them in e.binDir. -func (e *Env) compileBinaries(ctx context.Context) error { +// compileBinariesForOS cross-compiles tta, tailscale, and tailscaled for the +// given GOOS/GOARCH and places them in e.binDir/_/. +func (e *Env) compileBinariesForOS(ctx context.Context, goos, goarch string) error { modRoot, err := findModRoot() if err != nil { return err } + dir := goos + "_" + goarch + outDir := filepath.Join(e.binDir, dir) + if err := os.MkdirAll(outDir, 0755); err != nil { + return err + } + binaries := []struct{ name, pkg string }{ {"tta", "./cmd/tta"}, {"tailscale", "./cmd/tailscale"}, @@ -620,15 +632,15 @@ func (e *Env) compileBinaries(ctx context.Context) error { var eg errgroup.Group for _, bin := range binaries { eg.Go(func() error { - outPath := filepath.Join(e.binDir, bin.name) - e.t.Logf("compiling %s...", bin.name) + outPath := filepath.Join(outDir, bin.name) + e.t.Logf("compiling %s/%s...", dir, bin.name) cmd := exec.CommandContext(ctx, "go", "build", "-o", outPath, bin.pkg) cmd.Dir = modRoot - cmd.Env = append(os.Environ(), "GOOS=linux", "GOARCH=amd64", "CGO_ENABLED=0") + cmd.Env = append(os.Environ(), "GOOS="+goos, "GOARCH="+goarch, "CGO_ENABLED=0") if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("building %s: %v\n%s", bin.name, err, out) + return fmt.Errorf("building %s/%s: %v\n%s", dir, bin.name, err, out) } - e.t.Logf("compiled %s", bin.name) + e.t.Logf("compiled %s/%s", dir, bin.name) return nil }) } diff --git a/tstest/natlab/vmtest/vmtest_test.go b/tstest/natlab/vmtest/vmtest_test.go index de8199139..91c8359f1 100644 --- a/tstest/natlab/vmtest/vmtest_test.go +++ b/tstest/natlab/vmtest/vmtest_test.go @@ -13,6 +13,15 @@ import ( ) func TestSubnetRouter(t *testing.T) { + testSubnetRouterForOS(t, vmtest.Ubuntu2404) +} + +func TestSubnetRouterFreeBSD(t *testing.T) { + testSubnetRouterForOS(t, vmtest.FreeBSD150) +} + +func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { + t.Helper() env := vmtest.New(t) clientNet := env.AddNetwork("2.1.1.1", "192.168.1.1/24", "2000:1::1/64", vnet.EasyNAT) @@ -21,7 +30,7 @@ func TestSubnetRouter(t *testing.T) { client := env.AddNode("client", clientNet, vmtest.OS(vmtest.Gokrazy)) sr := env.AddNode("subnet-router", clientNet, internalNet, - vmtest.OS(vmtest.Ubuntu2404), + vmtest.OS(srOS), vmtest.AdvertiseRoutes("10.0.0.0/24")) backend := env.AddNode("backend", internalNet, vmtest.OS(vmtest.Gokrazy),