tstest/natlab/vmtest: add Fedora + DNS-backend coverage, harden non-KVM boot (#20409)

* tstest/natlab/vmtest: make cloud VM boot robust without KVM

Adding heavier distro images (Fedora) surfaced several ways the cloud VM
boot path breaks under TCG software emulation (no /dev/kvm), especially
with multiple concurrent VMs on few cores.

- Add a virtio-rng device to the cloud path so early boot doesn't block in
  getrandom() waiting for the CRNG to seed.
- When no hardware acceleration is available, relax the stuck-console
  watchdog (tuned for KVM's ~1-2s first output) and serialize VM boots so a
  heavy guest doesn't starve its siblings' emulation threads.
- Bound the bring-up context to the test deadline and dump each VM's console
  on failure, so a hang surfaces as a diagnosable Fatalf instead of an
  opaque `go test -timeout` panic (which skips cleanups).

Fixes tailscale/corp#44794
Updates tailscale/corp#44793

Signed-off-by: Brendan Creane <bcreane@gmail.com>

* tstest/natlab/vmtest: add Fedora and DNS-backend test coverage

Add the first RHEL-family distro and the machinery to assert and provision
distinct DNS backends, so adding a distro isn't "basically equivalent" to
the others.

- Add a Fedora 43 image (NetworkManager + systemd-resolved, SELinux
  enforcing). restorecon-relabel the curl'd binaries so they exec under
  enforcing mode.
- Add DNSBackend/AssertDNSBackend, reading the dns_manager_linux_mode_*
  clientmetric to assert which backend tailscaled selected.
- Add a WithDNSMode node option. WithDNSMode(DNSDirect) masks
  systemd-resolved and writes a plain resolv.conf pointing at natlab's fake
  DNS, forcing the direct backend -- so one image covers multiple backends.

Fixes tailscale/corp#44796
Updates tailscale/corp#44793

Signed-off-by: Brendan Creane <bcreane@gmail.com>

---------

Signed-off-by: Brendan Creane <bcreane@gmail.com>

tstest/natlab/vmtest/cloudinit.go

@@ -18,6 +18,7 @@ import (
 	"github.com/creachadair/mds/shell"
 	"github.com/kdomanski/iso9660"
 	"golang.org/x/crypto/ssh"
+	"tailscale.com/tstest/natlab/vnet"
 )
 
 // createCloudInitISO creates a cidata seed ISO for the given cloud VM node.
@@ -129,12 +130,23 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 	}
 	ud.WriteString("  - [\"chmod\", \"+x\", \"/usr/local/bin/tailscaled\", \"/usr/local/bin/tailscale\", \"/usr/local/bin/tta\"]\n")
 
+	// Apply the bin_t label for SELinux enforcement: we curl the binaries in
+	// rather than installing a package, so nothing else labels them, and
+	// enforcing mode would deny exec. No-op on non-SELinux systems, but only
+	// RHEL-family images ship restorecon.
+	if n.os.Family == LinuxRHEL {
+		ud.WriteString("  - [\"/bin/sh\", \"-c\", \"restorecon -v /usr/local/bin/tailscaled /usr/local/bin/tailscale /usr/local/bin/tta 2>&1 || true\"]\n")
+	}
+
 	// Enable IP forwarding for subnet routers.
 	if n.advertiseRoutes != "" {
 		ud.WriteString("  - [\"sysctl\", \"-w\", \"net.ipv4.ip_forward=1\"]\n")
 		ud.WriteString("  - [\"sysctl\", \"-w\", \"net.ipv6.conf.all.forwarding=1\"]\n")
 	}
 
+	// Provision the requested DNS backend before tailscaled starts.
+	writeLinuxDNSModeSetup(&ud, n.dnsMode)
+
 	// Start tailscaled, either via the stock systemd unit or directly in
 	// the background. --statedir provides a VarRoot so features like
 	// Taildrop (which needs a place to stash incoming files) have a
@@ -157,6 +169,30 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 	return ud.String()
 }
 
+// writeLinuxDNSModeSetup appends cloud-init runcmd entries that provision the
+// guest so tailscaled selects the requested DNS backend. Must run before the
+// tailscaled launch entries. The zero value (DNSDefault) is a no-op. mode is
+// validated in AddNode, so any other unknown value is a bug and panics.
+func writeLinuxDNSModeSetup(ud *strings.Builder, mode DNSMode) {
+	switch mode {
+	default:
+		// AddNode validates the mode, so an unknown value here is a bug.
+		panic(fmt.Sprintf("unhandled DNSMode %q", mode))
+	case DNSDefault:
+		// The empty/zero value: leave the image's DNS config alone, so
+		// systemd-resolved stays enabled and tailscaled selects it (or
+		// whatever the image runs by default). No-op.
+	case DNSDirect:
+		// Mask systemd-resolved and drop a plain resolv.conf so dnsMode() in
+		// net/dns/manager_linux.go falls through to "direct". Point it at
+		// natlab's fake DNS VIP (not a public resolver): it's the only resolver
+		// reachable in vnet and it serves the internal *.tailscale names.
+		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"systemctl disable --now systemd-resolved 2>/dev/null || true\"]\n")
+		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"systemctl mask systemd-resolved 2>/dev/null || true\"]\n")
+		fmt.Fprintf(ud, "  - [\"/bin/sh\", \"-c\", \"rm -f /etc/resolv.conf && printf 'nameserver %s\\\\n' >/etc/resolv.conf\"]\n", vnet.FakeDNSIPv4())
+	}
+}
+
 // generateFreeBSDUserData creates FreeBSD nuageinit user-data (#cloud-config)
 // for a node. FreeBSD's nuageinit supports a subset of cloud-init directives
 // including runcmd, which runs after networking is up.

tstest/natlab/vmtest/images.go

@@ -19,14 +19,32 @@ import (
 	"github.com/ulikunitz/xz"
 )
 
+// LinuxFamily classifies a Linux distro by the conventions that affect how we
+// provision it via cloud-init (default network manager, MAC/LSM, etc). It is
+// only meaningful for Linux cloud images, which must declare one; the zero
+// value means "undeclared".
+type LinuxFamily string
+
+const (
+	// LinuxDebian covers Debian and Ubuntu cloud images: systemd-networkd for
+	// networking and AppArmor as the LSM.
+	LinuxDebian LinuxFamily = "debian"
+
+	// LinuxRHEL covers Fedora, CentOS Stream, Rocky, and AlmaLinux cloud
+	// images: NetworkManager + systemd-resolved for networking and SELinux
+	// enforcing.
+	LinuxRHEL LinuxFamily = "rhel"
+)
+
 // OSImage describes a VM operating system image.
 type OSImage struct {
 	Name      string
-	URL       string // download URL for the cloud image
+	URL       string      // download URL for the cloud image
-	SHA256    string // expected SHA256 hash of the image (of the final qcow2, after any decompression)
+	SHA256    string      // expected SHA256 hash of the image (of the final qcow2, after any decompression)
-	MemoryMB  int    // RAM for the VM
+	MemoryMB  int         // RAM for the VM
-	IsGokrazy bool   // true for gokrazy images (different QEMU setup)
+	Family    LinuxFamily // Linux distro family (affects cloud-init user-data); empty means Debian-like
-	IsMacOS   bool   // true for macOS images (launched via tailmac, not QEMU)
+	IsGokrazy bool        // true for gokrazy images (different QEMU setup)
+	IsMacOS   bool        // true for macOS images (launched via tailmac, not QEMU)
 }
 
 // GOOS returns the Go OS name for this image.
@@ -51,6 +69,14 @@ func (img OSImage) GOARCH() string {
 	return "amd64"
 }
 
+// isLinuxCloudImage reports whether the image is a Linux distro cloud image
+// (Ubuntu, Debian, Fedora, ...), as opposed to gokrazy or a non-Linux OS.
+// These are the images provisioned via generateLinuxUserData and are the ones
+// that must declare a LinuxFamily.
+func (img OSImage) isLinuxCloudImage() bool {
+	return img.GOOS() == "linux" && !img.IsGokrazy
+}
+
 var (
 	// Gokrazy is a minimal Tailscale appliance image built from the gokrazy/natlabapp directory.
 	Gokrazy = OSImage{
@@ -64,6 +90,7 @@ var (
 		Name:     "ubuntu-24.04",
 		URL:      "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img",
 		MemoryMB: 1024,
+		Family:   LinuxDebian,
 	}
 
 	// Debian12 is Debian 12 (Bookworm) generic cloud image.
@@ -71,6 +98,7 @@ var (
 		Name:     "debian-12",
 		URL:      "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2",
 		MemoryMB: 1024,
+		Family:   LinuxDebian,
 	}
 
 	// FreeBSD150 is FreeBSD 15.0-RELEASE with BASIC-CLOUDINIT (nuageinit) support.
@@ -81,6 +109,15 @@ var (
 		MemoryMB: 1024,
 	}
 
+	// Fedora43 is the Fedora 43 Cloud Base image: NetworkManager +
+	// systemd-resolved, SELinux enforcing, hence LinuxRHEL.
+	Fedora43 = OSImage{
+		Name:     "fedora-43",
+		URL:      "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2",
+		MemoryMB: 1024,
+		Family:   LinuxRHEL,
+	}
+
 	// MacOS is a macOS VM launched via tailmac (Apple Virtualization.framework).
 	// Uses a Tart pre-built base image (ghcr.io/cirruslabs/macos-tahoe-base)
 	// which is automatically pulled on first use. Only runs on macOS arm64 hosts.
@@ -96,7 +133,7 @@ var (
 // uses a separate snapshot pipeline). It is intended for tooling such as
 // a CI prep step that wants to warm the image cache.
 func CloudImages() []OSImage {
-	return []OSImage{Ubuntu2404, Debian12, FreeBSD150}
+	return []OSImage{Ubuntu2404, Debian12, FreeBSD150, Fedora43}
 }
 
 // EnsureImage downloads img to the local cache if not already present.

tstest/natlab/vmtest/qemu.go

@@ -28,16 +28,30 @@ import (
 // platforms (macOS, etc.) TCG is used, which allows the tests to run
 // without a same-architecture hypervisor at the cost of speed.
 func qemuAccelArgs() []string {
+	if hardwareAccelAvailable() {
+		return []string{"-enable-kvm", "-cpu", "host"}
+	}
+	return nil
+}
+
+// hardwareAccelAvailable reports whether hardware-accelerated virtualisation
+// (KVM) is usable. When false, VMs run under TCG software emulation, which is
+// dramatically slower and, when several VMs boot concurrently, prone to CPU
+// starvation — a heavy guest (e.g. Fedora) can monopolize host cores and stall
+// its lighter siblings' emulation threads. Callers use this to relax timeouts
+// tuned for KVM's near-native boot speed. VMTEST_NO_KVM=1 forces TCG, for
+// reproducing slow-host behavior.
+func hardwareAccelAvailable() bool {
 	if os.Getenv("VMTEST_NO_KVM") == "1" {
-		return nil
+		return false
 	}
 	if runtime.GOOS == "linux" {
 		if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0); err == nil {
 			f.Close()
-			return []string{"-enable-kvm", "-cpu", "host"}
+			return true
 		}
 	}
-	return nil
+	return false
 }
 
 // gokrazyPlatform boots gokrazy (Linux) VMs via QEMU.
@@ -179,6 +193,10 @@ func (e *Env) startCloudQEMU(n *Node) error {
 		"-smbios", "type=1,serial=ds=nocloud",
 		"-serial", "file:" + logPath,
 		"-qmp", "unix:" + qmpSock + ",server,nowait",
+		// Feed host entropy to the guest so early boot doesn't block in
+		// getrandom() waiting for the CRNG to seed. Cheap and worth it on any
+		// backend; the stall is especially likely under TCG.
+		"-device", "virtio-rng-pci",
 	}
 
 	// Add network devices — one per NIC.
@@ -250,13 +268,29 @@ func (r *qemuRun) kill() {
 // VM console output goes to logPath (via QEMU's -serial or -chardev).
 // QEMU's own stdout/stderr go to logPath.qemu for diagnostics.
 func (e *Env) launchQEMU(name, logPath string, args []string) error {
-	// stuckTimeout is generous: a healthy VM prints SeaBIOS/kernel
+	// stuckTimeout is generous: a healthy VM prints SeaBIOS/kernel output
-	// output within ~1-2s on KVM, but slow shared CI hardware can lag.
+	// within ~1-2s on KVM, but slow CI hardware can lag. Under TCG a heavy
-	// Setting it too low risks killing a healthy-but-slow VM; setting it
+	// concurrent guest can starve its siblings, so give them much longer to
-	// too high masks the wedge case we want to recover from.
+	// emit a first console byte before we kill and retry.
-	const stuckTimeout = 45 * time.Second
+	stuckTimeout := 45 * time.Second
+	if !hardwareAccelAvailable() {
+		stuckTimeout = 4 * time.Minute
+	}
 	const maxAttempts = 3
 
+	// Dump the VM's console tail and QEMU's own stderr on test failure.
+	// Registered before the boot loop so it fires even when the node never
+	// boots (all attempts fail below), not just after a successful launch.
+	// The console log is empty when the guest never produced output (e.g. QEMU
+	// exited before the kernel ran); in that case the .qemu file holds the only
+	// diagnostic — KVM errors, "kvm not available", CPU model mismatch, etc.
+	e.t.Cleanup(func() {
+		if e.t.Failed() {
+			dumpLogTail(e.t, name, "console", logPath)
+			dumpLogTail(e.t, name, "qemu stderr", logPath+".qemu")
+		}
+	})
+
 	var lastErr error
 	for attempt := 1; attempt <= maxAttempts; attempt++ {
 		if attempt > 1 {
@@ -276,18 +310,7 @@ func (e *Env) launchQEMU(name, logPath string, args []string) error {
 			if e.ctx != nil {
 				go e.tailLogFile(e.ctx, name, logPath)
 			}
-			e.t.Cleanup(func() {
+			e.t.Cleanup(run.kill)
-				run.kill()
-				// Dump tail of VM log and QEMU's own stderr on failure.
-				// The console log (logPath) is empty when the guest never
-				// produced output (e.g. QEMU exited before the kernel ran);
-				// in that case the .qemu file holds the only diagnostic —
-				// KVM errors, "kvm not available", CPU model mismatch, etc.
-				if e.t.Failed() {
-					dumpLogTail(e.t, name, "console", logPath)
-					dumpLogTail(e.t, name, "qemu stderr", logPath+".qemu")
-				}
-			})
 			return nil
 		}
 		lastErr = fmt.Errorf("QEMU for %s produced no console output in %v", name, stuckTimeout)

tstest/natlab/vmtest/vmtest.go

@@ -447,7 +447,8 @@ type Node struct {
 	advertiseRoutes  string
 	snatSubnetRoutes *bool // nil means default (true)
 	webServerPort    int
-	sshPort          int // host port for SSH debug access (cloud VMs only)
+	sshPort          int     // host port for SSH debug access (cloud VMs only)
+	dnsMode          DNSMode // desired Linux DNS backend to provision; "" means the image default
 }
 
 // AddNode creates a new VM node. The name is used for identification and as the
@@ -486,6 +487,13 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 			n.snatSubnetRoutes = &v
 		case nodeOptWebServer:
 			n.webServerPort = int(o)
+		case nodeOptDNSMode:
+			switch DNSMode(o) {
+			case DNSDefault, DNSDirect:
+			default:
+				e.t.Fatalf("AddNode(%q): unsupported DNSMode %q", name, DNSMode(o))
+			}
+			n.dnsMode = DNSMode(o)
 		default:
 			// Pass through to vnet (TailscaledEnv, NodeOption, MAC, etc.)
 			vnetOpts = append(vnetOpts, o)
@@ -509,6 +517,12 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 		e.t.Skipf("macOS VM tests require a macOS arm64 host (got %s/%s)", runtime.GOOS, runtime.GOARCH)
 	}
 
+	// Linux cloud images must declare a family; it drives distro-specific
+	// cloud-init provisioning. gokrazy and non-Linux images don't use that path.
+	if n.os.isLinuxCloudImage() && n.os.Family == "" {
+		e.t.Fatalf("AddNode(%q): Linux cloud image %q has no LinuxFamily set", name, n.os.Name)
+	}
+
 	n.vnetNode = e.cfg.AddNode(vnetOpts...)
 	n.num = n.vnetNode.Num()
 	return n
@@ -544,6 +558,28 @@ type nodeOptSystemdUnit struct{}
 type nodeOptAdvertiseRoutes string
 type nodeOptSNATSubnetRoutes bool
 type nodeOptWebServer int
+type nodeOptDNSMode DNSMode
+
+// DNSMode is a provisioning directive, not a DNS-backend name: it says what, if
+// anything, to do to the guest's DNS before tailscaled starts, letting one
+// distro image cover multiple backends. DNSDefault leaves DNS untouched (the
+// resulting backend is image-dependent); the other modes provision the guest to
+// force a specific backend, and are named to match the mode strings in
+// net/dns/manager_linux.go so the result can be checked with
+// [Env.AssertDNSBackend].
+type DNSMode string
+
+const (
+	// DNSDefault leaves the image's DNS configuration untouched, so the backend
+	// is whatever the image runs by default (typically systemd-resolved on
+	// modern distros). It has no [Env.AssertDNSBackend] counterpart because the
+	// result isn't forced.
+	DNSDefault DNSMode = ""
+
+	// DNSDirect masks systemd-resolved and installs a plain /etc/resolv.conf
+	// so tailscaled selects the "direct" manager (rewrites resolv.conf itself).
+	DNSDirect DNSMode = "direct"
+)
 
 // OS returns a NodeOption that sets the node's operating system image.
 func OS(img OSImage) nodeOptOS { return nodeOptOS(img) }
@@ -587,10 +623,22 @@ func SNATSubnetRoutes(v bool) nodeOptSNATSubnetRoutes { return nodeOptSNATSubnet
 // The webserver responds with "Hello world I am <nodename> from <sourceIP>" on all requests.
 func WebServer(port int) nodeOptWebServer { return nodeOptWebServer(port) }
 
+// WithDNSMode returns a NodeOption that provisions the (Linux) node so
+// tailscaled selects the given DNS backend. Only meaningful for Linux cloud
+// images; ignored for gokrazy/macOS. See [DNSMode].
+func WithDNSMode(m DNSMode) nodeOptDNSMode { return nodeOptDNSMode(m) }
+
 // Start initializes the virtual network, boots all VMs in parallel, and waits
 // for all TTA agents to connect. It should be called after all AddNetwork/AddNode calls.
 func (e *Env) Start() {
 	t := e.t
+
+	// Give bring-up (image build, boot, agent wait) a generous budget measured
+	// from now. We deliberately do NOT derive this from the test deadline: on
+	// CI, per-test timeouts can be tight (a few minutes), and reserving headroom
+	// under the deadline was observed to steal time bring-up legitimately needs,
+	// failing slow-but-healthy runs. If bring-up genuinely exceeds this, the
+	// `go test -timeout` panic is an acceptable (if blunt) backstop.
 	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
 	t.Cleanup(cancel)
 	e.ctx = ctx
@@ -633,7 +681,14 @@ func (e *Env) Start() {
 	// Boot all nodes in parallel. Each platform handles its own
 	// dependencies (image prep, binary compilation, socket setup)
 	// via sync.Once, so independent work overlaps naturally.
+	//
+	// Under TCG, concurrent boots oversubscribe the host CPUs and a heavy
+	// guest can starve its siblings past the stuck-detector; serialize boots
+	// there so each clears that gate before the next starts. KVM stays parallel.
 	var bootEg errgroup.Group
+	if !hardwareAccelAvailable() {
+		bootEg.SetLimit(1)
+	}
 	for _, n := range e.nodes {
 		bootEg.Go(func() error {
 			return n.platform().boot(ctx, e, n)
@@ -999,6 +1054,41 @@ func (e *Env) ClientMetrics(n *Node) ClientMetrics {
 	return out
 }
 
+// dnsBackendMetricPrefix is the prefix of the clientmetric gauge that
+// tailscaled sets to 1 for the Linux DNS mode it selected. See
+// net/dns/manager_linux.go.
+const dnsBackendMetricPrefix = "dns_manager_linux_mode_"
+
+// DNSBackend returns the Linux DNS backend ("mode") the node's tailscaled
+// selected (e.g. "systemd-resolved", "direct"). tailscaled sets a single gauge
+// named dns_manager_linux_mode_<mode> to 1 for its selected mode (see
+// net/dns/manager_linux.go); this finds that gauge and returns <mode>. It fails
+// the test if none is set (non-Linux node, or DNS not yet configured).
+func (e *Env) DNSBackend(n *Node) string {
+	e.t.Helper()
+	for name, m := range e.ClientMetrics(n) {
+		mode, ok := strings.CutPrefix(name, dnsBackendMetricPrefix)
+		if !ok || m.Value != 1 {
+			continue
+		}
+		// tailscaled sanitizes "-" to "_" when forming the metric name;
+		// reverse it so we return net/dns's spelling ("systemd-resolved").
+		return strings.ReplaceAll(mode, "_", "-")
+	}
+	e.t.Fatalf("Node %q: no %s* gauge set (non-Linux node, or DNS not yet configured)", n.Name(), dnsBackendMetricPrefix)
+	return ""
+}
+
+// AssertDNSBackend fails the test unless the node's selected Linux DNS backend
+// matches want (see [Env.DNSBackend] for the mode strings). Use it in distro
+// tests to prove the node exercises the intended DNS manager.
+func (e *Env) AssertDNSBackend(n *Node, want string) {
+	e.t.Helper()
+	if got := e.DNSBackend(n); got != want {
+		e.t.Fatalf("Node %q: DNS backend = %q, want %q", n.Name(), got, want)
+	}
+}
+
 // ClientMetrics is a view of the client metrics exported by a node.
 // The keys of the map are the metric names.
 type ClientMetrics map[string]ClientMetric

tstest/natlab/vmtest/vmtest_test.go

@@ -84,6 +84,28 @@ func TestSubnetRouterFreeBSD(t *testing.T) {
 	testSubnetRouterForOS(t, vmtest.FreeBSD150)
 }
 
+func TestSubnetRouterFedora(t *testing.T) {
+	testSubnetRouterForOS(t, vmtest.Fedora43)
+}
+
+// TestFedoraDNSDirect verifies that provisioning a Fedora node with
+// WithDNSMode(DNSDirect) — which masks systemd-resolved — makes tailscaled
+// select the "direct" DNS backend instead of the image default
+// ("systemd-resolved"). This is what lets one distro image cover multiple DNS
+// backends, so adding a distro isn't "basically equivalent" to the others.
+func TestFedoraDNSDirect(t *testing.T) {
+	env := vmtest.New(t)
+
+	net := env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT)
+	node := env.AddNode("fedora", net,
+		vmtest.OS(vmtest.Fedora43),
+		vmtest.WithDNSMode(vmtest.DNSDirect))
+
+	env.Start()
+
+	env.AssertDNSBackend(node, "direct")
+}
+
 func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) {
 	t.Helper()
 	env := vmtest.New(t)
@@ -107,6 +129,14 @@ func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) {
 
 	env.Start()
 
+	// Log which DNS backend the (Linux) subnet router selected. This is the
+	// whole point of testing multiple distros: they should exercise different
+	// DNS managers. Once we've confirmed the expected value per distro, this
+	// can become an AssertDNSBackend. FreeBSD has no Linux DNS gauge, so skip.
+	if srOS.GOOS() == "linux" {
+		t.Logf("subnet-router (%s) DNS backend: %s", srOS.Name, env.DNSBackend(sr))
+	}
+
 	approveStep.Begin()
 	env.ApproveRoutes(sr, "10.0.0.0/24")
 	approveStep.End(nil)