vmtest: add VM-based integration test framework

Add tstest/natlab/vmtest, a high-level framework for running multi-VM integration tests with mixed OS types (gokrazy + Ubuntu/Debian cloud images) connected via natlab's vnet virtual network. The vmtest package provides: - Env type that orchestrates vnet, QEMU processes, and agent connections - OS image support (Gokrazy, Ubuntu2404, Debian12) with download/cache - QEMU launch per OS type (microvm for gokrazy, q35+KVM for cloud) - Cloud-init seed ISO generation with network-config for multi-NIC - Cross-compilation of test binaries for cloud VMs - Debug SSH NIC on cloud VMs for interactive debugging - Test helpers: ApproveRoutes, HTTPGet, TailscalePing, DumpStatus, WaitForPeerRoute, SSHExec TTA enhancements (cmd/tta): - Parameterize /up (accept-routes, advertise-routes, snat-subnet-routes) - Add /set, /start-webserver, /http-get endpoints - /http-get uses local.Client.UserDial for Tailscale-routed requests - Fix /ping for non-gokrazy systems TestSubnetRouter exercises a 3-VM subnet router scenario: client (gokrazy) → subnet-router (Ubuntu, dual-NIC) → backend (gokrazy) Verifies HTTP access to the backend webserver through the Tailscale subnet route. Passes in ~30 seconds. Updates tailscale/tailscale#13038 Change-Id: I165b64af241d37f5f5870e796a52502fc56146fa Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
2026-04-08 18:09:05 +00:00
parent d948b78b23
commit ec0b23a21f
12 changed files with 1382 additions and 11 deletions
@@ -0,0 +1,239 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"time"
+
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+// startQEMU launches a QEMU process for the given node.
+func (e *Env) startQEMU(n *Node) error {
+	if n.os.IsGokrazy {
+		return e.startGokrazyQEMU(n)
+	}
+	return e.startCloudQEMU(n)
+}
+
+// startGokrazyQEMU launches a QEMU process for a gokrazy node.
+// This follows the same pattern as tstest/integration/nat/nat_test.go.
+func (e *Env) startGokrazyQEMU(n *Node) error {
+	disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name))
+	if err := createOverlay(e.gokrazyBase, disk); err != nil {
+		return err
+	}
+
+	var envBuf bytes.Buffer
+	for _, env := range n.vnetNode.Env() {
+		fmt.Fprintf(&envBuf, " tailscaled.env=%s=%s", env.Key, env.Value)
+	}
+	sysLogAddr := net.JoinHostPort(vnet.FakeSyslogIPv4().String(), "995")
+	if n.vnetNode.IsV6Only() {
+		sysLogAddr = net.JoinHostPort(vnet.FakeSyslogIPv6().String(), "995")
+	}
+
+	logPath := filepath.Join(e.tempDir, n.name+".log")
+
+	args := []string{
+		"-M", "microvm,isa-serial=off",
+		"-m", fmt.Sprintf("%dM", n.os.MemoryMB),
+		"-nodefaults", "-no-user-config", "-nographic",
+		"-kernel", e.gokrazyKernel,
+		"-append", "console=hvc0 root=PARTUUID=60c24cc1-f3f9-427a-8199-76baa2d60001/PARTNROFF=1 ro init=/gokrazy/init panic=10 oops=panic pci=off nousb tsc=unstable clocksource=hpet gokrazy.remote_syslog.target=" + sysLogAddr + " tailscale-tta=1" + envBuf.String(),
+		"-drive", "id=blk0,file=" + disk + ",format=qcow2",
+		"-device", "virtio-blk-device,drive=blk0",
+		"-device", "virtio-serial-device",
+		"-device", "virtio-rng-device",
+		"-chardev", "file,id=virtiocon0,path=" + logPath,
+		"-device", "virtconsole,chardev=virtiocon0",
+	}
+
+	// Add network devices — one per NIC.
+	for i := range n.vnetNode.NumNICs() {
+		mac := n.vnetNode.NICMac(i)
+		netdevID := fmt.Sprintf("net%d", i)
+		args = append(args,
+			"-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr),
+			"-device", fmt.Sprintf("virtio-net-device,netdev=%s,mac=%s", netdevID, mac),
+		)
+	}
+
+	return e.launchQEMU(n.name, logPath, args)
+}
+
+// startCloudQEMU launches a QEMU process for a cloud image (Ubuntu, Debian, etc).
+func (e *Env) startCloudQEMU(n *Node) error {
+	basePath := cachedImagePath(n.os)
+	disk := filepath.Join(e.tempDir, fmt.Sprintf("%s.qcow2", n.name))
+	if err := createOverlay(basePath, disk); err != nil {
+		return err
+	}
+
+	// Create a seed ISO with cloud-init config (meta-data, user-data, network-config).
+	// This MUST be a local ISO (not HTTP) so cloud-init reads network-config during
+	// init-local, before systemd-networkd-wait-online blocks boot.
+	seedISO, err := e.createCloudInitISO(n)
+	if err != nil {
+		return fmt.Errorf("creating cloud-init ISO: %w", err)
+	}
+
+	logPath := filepath.Join(e.tempDir, n.name+".log")
+	qmpSock := filepath.Join(e.tempDir, n.name+"-qmp.sock")
+
+	args := []string{
+		"-machine", "q35,accel=kvm",
+		"-m", fmt.Sprintf("%dM", n.os.MemoryMB),
+		"-cpu", "host",
+		"-smp", "2",
+		"-display", "none",
+		"-drive", fmt.Sprintf("file=%s,if=virtio", disk),
+		"-drive", fmt.Sprintf("file=%s,if=virtio,media=cdrom,readonly=on", seedISO),
+		"-smbios", "type=1,serial=ds=nocloud",
+		"-serial", "file:" + logPath,
+		"-qmp", "unix:" + qmpSock + ",server,nowait",
+	}
+
+	// Add network devices — one per NIC.
+	// romfile="" disables the iPXE option ROM entirely, saving ~5s per NIC at boot
+	// and avoiding "duplicate fw_cfg file name" errors with multiple NICs.
+	for i := range n.vnetNode.NumNICs() {
+		mac := n.vnetNode.NICMac(i)
+		netdevID := fmt.Sprintf("net%d", i)
+		args = append(args,
+			"-netdev", fmt.Sprintf("stream,id=%s,addr.type=unix,addr.path=%s", netdevID, e.sockAddr),
+			"-device", fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s,romfile=", netdevID, mac),
+		)
+	}
+
+	// Add a debug NIC with user-mode networking for SSH access from the host.
+	// Use port 0 so the OS picks a free port; we query the actual port via QMP after launch.
+	args = append(args,
+		"-netdev", "user,id=debug0,hostfwd=tcp:127.0.0.1:0-:22",
+		"-device", "virtio-net-pci,netdev=debug0,romfile=",
+	)
+
+	if err := e.launchQEMU(n.name, logPath, args); err != nil {
+		return err
+	}
+
+	// Query QMP to find the actual SSH port that QEMU allocated.
+	port, err := qmpQueryHostFwd(qmpSock)
+	if err != nil {
+		return fmt.Errorf("querying SSH port via QMP: %w", err)
+	}
+	n.sshPort = port
+	e.t.Logf("[%s] SSH debug: ssh -p %d root@127.0.0.1 (password: root)", n.name, port)
+	return nil
+}
+
+// launchQEMU starts a qemu-system-x86_64 process with the given args.
+// VM console output goes to logPath (via QEMU's -serial or -chardev).
+// QEMU's own stdout/stderr go to logPath.qemu for diagnostics.
+func (e *Env) launchQEMU(name, logPath string, args []string) error {
+	cmd := exec.Command("qemu-system-x86_64", args...)
+	// Send stdout/stderr to the log file for any QEMU diagnostic messages.
+	// Stdin must be /dev/null to prevent QEMU from trying to read.
+	devNull, err := os.Open(os.DevNull)
+	if err != nil {
+		return fmt.Errorf("open /dev/null: %w", err)
+	}
+	cmd.Stdin = devNull
+	qemuLog, err := os.Create(logPath + ".qemu")
+	if err != nil {
+		devNull.Close()
+		return err
+	}
+	cmd.Stdout = qemuLog
+	cmd.Stderr = qemuLog
+	if err := cmd.Start(); err != nil {
+		devNull.Close()
+		qemuLog.Close()
+		return fmt.Errorf("qemu for %s: %w", name, err)
+	}
+	e.t.Logf("launched QEMU for %s (pid %d), log: %s", name, cmd.Process.Pid, logPath)
+	e.qemuProcs = append(e.qemuProcs, cmd)
+	e.t.Cleanup(func() {
+		cmd.Process.Kill()
+		cmd.Wait()
+		devNull.Close()
+		qemuLog.Close()
+		// Dump tail of VM log on failure for debugging.
+		if e.t.Failed() {
+			if data, err := os.ReadFile(logPath); err == nil {
+				lines := bytes.Split(data, []byte("\n"))
+				start := 0
+				if len(lines) > 50 {
+					start = len(lines) - 50
+				}
+				e.t.Logf("=== last 50 lines of %s log ===", name)
+				for _, line := range lines[start:] {
+					e.t.Logf("[%s] %s", name, line)
+				}
+			}
+		}
+	})
+	return nil
+}
+
+// qmpQueryHostFwd connects to a QEMU QMP socket and queries the host port
+// assigned to the first TCP host forward rule (the SSH debug port).
+func qmpQueryHostFwd(sockPath string) (int, error) {
+	// Wait for the QMP socket to appear.
+	var conn net.Conn
+	for range 50 {
+		var err error
+		conn, err = net.Dial("unix", sockPath)
+		if err == nil {
+			break
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+	if conn == nil {
+		return 0, fmt.Errorf("QMP socket %s not available", sockPath)
+	}
+	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(5 * time.Second))
+
+	// Read the QMP greeting.
+	var greeting json.RawMessage
+	dec := json.NewDecoder(conn)
+	if err := dec.Decode(&greeting); err != nil {
+		return 0, fmt.Errorf("reading QMP greeting: %w", err)
+	}
+
+	// Send qmp_capabilities to initialize.
+	fmt.Fprintf(conn, `{"execute":"qmp_capabilities"}`+"\n")
+	var capsResp json.RawMessage
+	if err := dec.Decode(&capsResp); err != nil {
+		return 0, fmt.Errorf("reading qmp_capabilities response: %w", err)
+	}
+
+	// Query "info usernet" via human-monitor-command.
+	fmt.Fprintf(conn, `{"execute":"human-monitor-command","arguments":{"command-line":"info usernet"}}`+"\n")
+	var hmpResp struct {
+		Return string `json:"return"`
+	}
+	if err := dec.Decode(&hmpResp); err != nil {
+		return 0, fmt.Errorf("reading info usernet response: %w", err)
+	}
+
+	// Parse the port from output like:
+	//   TCP[HOST_FORWARD]  12       127.0.0.1 35323       10.0.2.15    22
+	re := regexp.MustCompile(`TCP\[HOST_FORWARD\]\s+\d+\s+127\.0\.0\.1\s+(\d+)\s+`)
+	m := re.FindStringSubmatch(hmpResp.Return)
+	if m == nil {
+		return 0, fmt.Errorf("no hostfwd port found in: %s", hmpResp.Return)
+	}
+	return strconv.Atoi(m[1])
+}