From a7cb5745a2ee92e9edd4c6b79457e03749347d19 Mon Sep 17 00:00:00 2001 From: Brendan Creane Date: Mon, 20 Jul 2026 12:41:25 -0700 Subject: [PATCH] tstest/natlab/vmtest: add Fedora + DNS-backend coverage, harden non-KVM boot (#20409) * tstest/natlab/vmtest: make cloud VM boot robust without KVM Adding heavier distro images (Fedora) surfaced several ways the cloud VM boot path breaks under TCG software emulation (no /dev/kvm), especially with multiple concurrent VMs on few cores. - Add a virtio-rng device to the cloud path so early boot doesn't block in getrandom() waiting for the CRNG to seed. - When no hardware acceleration is available, relax the stuck-console watchdog (tuned for KVM's ~1-2s first output) and serialize VM boots so a heavy guest doesn't starve its siblings' emulation threads. - Bound the bring-up context to the test deadline and dump each VM's console on failure, so a hang surfaces as a diagnosable Fatalf instead of an opaque `go test -timeout` panic (which skips cleanups). Fixes tailscale/corp#44794 Updates tailscale/corp#44793 Signed-off-by: Brendan Creane * tstest/natlab/vmtest: add Fedora and DNS-backend test coverage Add the first RHEL-family distro and the machinery to assert and provision distinct DNS backends, so adding a distro isn't "basically equivalent" to the others. - Add a Fedora 43 image (NetworkManager + systemd-resolved, SELinux enforcing). restorecon-relabel the curl'd binaries so they exec under enforcing mode. - Add DNSBackend/AssertDNSBackend, reading the dns_manager_linux_mode_* clientmetric to assert which backend tailscaled selected. - Add a WithDNSMode node option. WithDNSMode(DNSDirect) masks systemd-resolved and writes a plain resolv.conf pointing at natlab's fake DNS, forcing the direct backend -- so one image covers multiple backends. Fixes tailscale/corp#44796 Updates tailscale/corp#44793 Signed-off-by: Brendan Creane --------- Signed-off-by: Brendan Creane --- tstest/natlab/vmtest/cloudinit.go | 36 +++++++++++ tstest/natlab/vmtest/images.go | 49 +++++++++++++-- tstest/natlab/vmtest/qemu.go | 63 +++++++++++++------- tstest/natlab/vmtest/vmtest.go | 92 ++++++++++++++++++++++++++++- tstest/natlab/vmtest/vmtest_test.go | 30 ++++++++++ 5 files changed, 243 insertions(+), 27 deletions(-) diff --git a/tstest/natlab/vmtest/cloudinit.go b/tstest/natlab/vmtest/cloudinit.go index 7b7bac572..eff228bba 100644 --- a/tstest/natlab/vmtest/cloudinit.go +++ b/tstest/natlab/vmtest/cloudinit.go @@ -18,6 +18,7 @@ import ( "github.com/creachadair/mds/shell" "github.com/kdomanski/iso9660" "golang.org/x/crypto/ssh" + "tailscale.com/tstest/natlab/vnet" ) // createCloudInitISO creates a cidata seed ISO for the given cloud VM node. @@ -129,12 +130,23 @@ func (e *Env) generateLinuxUserData(n *Node) string { } ud.WriteString(" - [\"chmod\", \"+x\", \"/usr/local/bin/tailscaled\", \"/usr/local/bin/tailscale\", \"/usr/local/bin/tta\"]\n") + // Apply the bin_t label for SELinux enforcement: we curl the binaries in + // rather than installing a package, so nothing else labels them, and + // enforcing mode would deny exec. No-op on non-SELinux systems, but only + // RHEL-family images ship restorecon. + if n.os.Family == LinuxRHEL { + ud.WriteString(" - [\"/bin/sh\", \"-c\", \"restorecon -v /usr/local/bin/tailscaled /usr/local/bin/tailscale /usr/local/bin/tta 2>&1 || true\"]\n") + } + // Enable IP forwarding for subnet routers. if n.advertiseRoutes != "" { ud.WriteString(" - [\"sysctl\", \"-w\", \"net.ipv4.ip_forward=1\"]\n") ud.WriteString(" - [\"sysctl\", \"-w\", \"net.ipv6.conf.all.forwarding=1\"]\n") } + // Provision the requested DNS backend before tailscaled starts. + writeLinuxDNSModeSetup(&ud, n.dnsMode) + // Start tailscaled, either via the stock systemd unit or directly in // the background. --statedir provides a VarRoot so features like // Taildrop (which needs a place to stash incoming files) have a @@ -157,6 +169,30 @@ func (e *Env) generateLinuxUserData(n *Node) string { return ud.String() } +// writeLinuxDNSModeSetup appends cloud-init runcmd entries that provision the +// guest so tailscaled selects the requested DNS backend. Must run before the +// tailscaled launch entries. The zero value (DNSDefault) is a no-op. mode is +// validated in AddNode, so any other unknown value is a bug and panics. +func writeLinuxDNSModeSetup(ud *strings.Builder, mode DNSMode) { + switch mode { + default: + // AddNode validates the mode, so an unknown value here is a bug. + panic(fmt.Sprintf("unhandled DNSMode %q", mode)) + case DNSDefault: + // The empty/zero value: leave the image's DNS config alone, so + // systemd-resolved stays enabled and tailscaled selects it (or + // whatever the image runs by default). No-op. + case DNSDirect: + // Mask systemd-resolved and drop a plain resolv.conf so dnsMode() in + // net/dns/manager_linux.go falls through to "direct". Point it at + // natlab's fake DNS VIP (not a public resolver): it's the only resolver + // reachable in vnet and it serves the internal *.tailscale names. + fmt.Fprintf(ud, " - [\"/bin/sh\", \"-c\", \"systemctl disable --now systemd-resolved 2>/dev/null || true\"]\n") + fmt.Fprintf(ud, " - [\"/bin/sh\", \"-c\", \"systemctl mask systemd-resolved 2>/dev/null || true\"]\n") + fmt.Fprintf(ud, " - [\"/bin/sh\", \"-c\", \"rm -f /etc/resolv.conf && printf 'nameserver %s\\\\n' >/etc/resolv.conf\"]\n", vnet.FakeDNSIPv4()) + } +} + // generateFreeBSDUserData creates FreeBSD nuageinit user-data (#cloud-config) // for a node. FreeBSD's nuageinit supports a subset of cloud-init directives // including runcmd, which runs after networking is up. diff --git a/tstest/natlab/vmtest/images.go b/tstest/natlab/vmtest/images.go index e01bf1d71..93bb99dca 100644 --- a/tstest/natlab/vmtest/images.go +++ b/tstest/natlab/vmtest/images.go @@ -19,14 +19,32 @@ import ( "github.com/ulikunitz/xz" ) +// LinuxFamily classifies a Linux distro by the conventions that affect how we +// provision it via cloud-init (default network manager, MAC/LSM, etc). It is +// only meaningful for Linux cloud images, which must declare one; the zero +// value means "undeclared". +type LinuxFamily string + +const ( + // LinuxDebian covers Debian and Ubuntu cloud images: systemd-networkd for + // networking and AppArmor as the LSM. + LinuxDebian LinuxFamily = "debian" + + // LinuxRHEL covers Fedora, CentOS Stream, Rocky, and AlmaLinux cloud + // images: NetworkManager + systemd-resolved for networking and SELinux + // enforcing. + LinuxRHEL LinuxFamily = "rhel" +) + // OSImage describes a VM operating system image. type OSImage struct { Name string - URL string // download URL for the cloud image - SHA256 string // expected SHA256 hash of the image (of the final qcow2, after any decompression) - MemoryMB int // RAM for the VM - IsGokrazy bool // true for gokrazy images (different QEMU setup) - IsMacOS bool // true for macOS images (launched via tailmac, not QEMU) + URL string // download URL for the cloud image + SHA256 string // expected SHA256 hash of the image (of the final qcow2, after any decompression) + MemoryMB int // RAM for the VM + Family LinuxFamily // Linux distro family (affects cloud-init user-data); empty means Debian-like + IsGokrazy bool // true for gokrazy images (different QEMU setup) + IsMacOS bool // true for macOS images (launched via tailmac, not QEMU) } // GOOS returns the Go OS name for this image. @@ -51,6 +69,14 @@ func (img OSImage) GOARCH() string { return "amd64" } +// isLinuxCloudImage reports whether the image is a Linux distro cloud image +// (Ubuntu, Debian, Fedora, ...), as opposed to gokrazy or a non-Linux OS. +// These are the images provisioned via generateLinuxUserData and are the ones +// that must declare a LinuxFamily. +func (img OSImage) isLinuxCloudImage() bool { + return img.GOOS() == "linux" && !img.IsGokrazy +} + var ( // Gokrazy is a minimal Tailscale appliance image built from the gokrazy/natlabapp directory. Gokrazy = OSImage{ @@ -64,6 +90,7 @@ var ( Name: "ubuntu-24.04", URL: "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img", MemoryMB: 1024, + Family: LinuxDebian, } // Debian12 is Debian 12 (Bookworm) generic cloud image. @@ -71,6 +98,7 @@ var ( Name: "debian-12", URL: "https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2", MemoryMB: 1024, + Family: LinuxDebian, } // FreeBSD150 is FreeBSD 15.0-RELEASE with BASIC-CLOUDINIT (nuageinit) support. @@ -81,6 +109,15 @@ var ( MemoryMB: 1024, } + // Fedora43 is the Fedora 43 Cloud Base image: NetworkManager + + // systemd-resolved, SELinux enforcing, hence LinuxRHEL. + Fedora43 = OSImage{ + Name: "fedora-43", + URL: "https://download.fedoraproject.org/pub/fedora/linux/releases/43/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-43-1.6.x86_64.qcow2", + MemoryMB: 1024, + Family: LinuxRHEL, + } + // MacOS is a macOS VM launched via tailmac (Apple Virtualization.framework). // Uses a Tart pre-built base image (ghcr.io/cirruslabs/macos-tahoe-base) // which is automatically pulled on first use. Only runs on macOS arm64 hosts. @@ -96,7 +133,7 @@ var ( // uses a separate snapshot pipeline). It is intended for tooling such as // a CI prep step that wants to warm the image cache. func CloudImages() []OSImage { - return []OSImage{Ubuntu2404, Debian12, FreeBSD150} + return []OSImage{Ubuntu2404, Debian12, FreeBSD150, Fedora43} } // EnsureImage downloads img to the local cache if not already present. diff --git a/tstest/natlab/vmtest/qemu.go b/tstest/natlab/vmtest/qemu.go index 47f4191aa..93a69e137 100644 --- a/tstest/natlab/vmtest/qemu.go +++ b/tstest/natlab/vmtest/qemu.go @@ -28,16 +28,30 @@ import ( // platforms (macOS, etc.) TCG is used, which allows the tests to run // without a same-architecture hypervisor at the cost of speed. func qemuAccelArgs() []string { + if hardwareAccelAvailable() { + return []string{"-enable-kvm", "-cpu", "host"} + } + return nil +} + +// hardwareAccelAvailable reports whether hardware-accelerated virtualisation +// (KVM) is usable. When false, VMs run under TCG software emulation, which is +// dramatically slower and, when several VMs boot concurrently, prone to CPU +// starvation — a heavy guest (e.g. Fedora) can monopolize host cores and stall +// its lighter siblings' emulation threads. Callers use this to relax timeouts +// tuned for KVM's near-native boot speed. VMTEST_NO_KVM=1 forces TCG, for +// reproducing slow-host behavior. +func hardwareAccelAvailable() bool { if os.Getenv("VMTEST_NO_KVM") == "1" { - return nil + return false } if runtime.GOOS == "linux" { if f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0); err == nil { f.Close() - return []string{"-enable-kvm", "-cpu", "host"} + return true } } - return nil + return false } // gokrazyPlatform boots gokrazy (Linux) VMs via QEMU. @@ -179,6 +193,10 @@ func (e *Env) startCloudQEMU(n *Node) error { "-smbios", "type=1,serial=ds=nocloud", "-serial", "file:" + logPath, "-qmp", "unix:" + qmpSock + ",server,nowait", + // Feed host entropy to the guest so early boot doesn't block in + // getrandom() waiting for the CRNG to seed. Cheap and worth it on any + // backend; the stall is especially likely under TCG. + "-device", "virtio-rng-pci", } // Add network devices — one per NIC. @@ -250,13 +268,29 @@ func (r *qemuRun) kill() { // VM console output goes to logPath (via QEMU's -serial or -chardev). // QEMU's own stdout/stderr go to logPath.qemu for diagnostics. func (e *Env) launchQEMU(name, logPath string, args []string) error { - // stuckTimeout is generous: a healthy VM prints SeaBIOS/kernel - // output within ~1-2s on KVM, but slow shared CI hardware can lag. - // Setting it too low risks killing a healthy-but-slow VM; setting it - // too high masks the wedge case we want to recover from. - const stuckTimeout = 45 * time.Second + // stuckTimeout is generous: a healthy VM prints SeaBIOS/kernel output + // within ~1-2s on KVM, but slow CI hardware can lag. Under TCG a heavy + // concurrent guest can starve its siblings, so give them much longer to + // emit a first console byte before we kill and retry. + stuckTimeout := 45 * time.Second + if !hardwareAccelAvailable() { + stuckTimeout = 4 * time.Minute + } const maxAttempts = 3 + // Dump the VM's console tail and QEMU's own stderr on test failure. + // Registered before the boot loop so it fires even when the node never + // boots (all attempts fail below), not just after a successful launch. + // The console log is empty when the guest never produced output (e.g. QEMU + // exited before the kernel ran); in that case the .qemu file holds the only + // diagnostic — KVM errors, "kvm not available", CPU model mismatch, etc. + e.t.Cleanup(func() { + if e.t.Failed() { + dumpLogTail(e.t, name, "console", logPath) + dumpLogTail(e.t, name, "qemu stderr", logPath+".qemu") + } + }) + var lastErr error for attempt := 1; attempt <= maxAttempts; attempt++ { if attempt > 1 { @@ -276,18 +310,7 @@ func (e *Env) launchQEMU(name, logPath string, args []string) error { if e.ctx != nil { go e.tailLogFile(e.ctx, name, logPath) } - e.t.Cleanup(func() { - run.kill() - // Dump tail of VM log and QEMU's own stderr on failure. - // The console log (logPath) is empty when the guest never - // produced output (e.g. QEMU exited before the kernel ran); - // in that case the .qemu file holds the only diagnostic — - // KVM errors, "kvm not available", CPU model mismatch, etc. - if e.t.Failed() { - dumpLogTail(e.t, name, "console", logPath) - dumpLogTail(e.t, name, "qemu stderr", logPath+".qemu") - } - }) + e.t.Cleanup(run.kill) return nil } lastErr = fmt.Errorf("QEMU for %s produced no console output in %v", name, stuckTimeout) diff --git a/tstest/natlab/vmtest/vmtest.go b/tstest/natlab/vmtest/vmtest.go index 6e75e90a5..95cf15311 100644 --- a/tstest/natlab/vmtest/vmtest.go +++ b/tstest/natlab/vmtest/vmtest.go @@ -447,7 +447,8 @@ type Node struct { advertiseRoutes string snatSubnetRoutes *bool // nil means default (true) webServerPort int - sshPort int // host port for SSH debug access (cloud VMs only) + sshPort int // host port for SSH debug access (cloud VMs only) + dnsMode DNSMode // desired Linux DNS backend to provision; "" means the image default } // AddNode creates a new VM node. The name is used for identification and as the @@ -486,6 +487,13 @@ func (e *Env) AddNode(name string, opts ...any) *Node { n.snatSubnetRoutes = &v case nodeOptWebServer: n.webServerPort = int(o) + case nodeOptDNSMode: + switch DNSMode(o) { + case DNSDefault, DNSDirect: + default: + e.t.Fatalf("AddNode(%q): unsupported DNSMode %q", name, DNSMode(o)) + } + n.dnsMode = DNSMode(o) default: // Pass through to vnet (TailscaledEnv, NodeOption, MAC, etc.) vnetOpts = append(vnetOpts, o) @@ -509,6 +517,12 @@ func (e *Env) AddNode(name string, opts ...any) *Node { e.t.Skipf("macOS VM tests require a macOS arm64 host (got %s/%s)", runtime.GOOS, runtime.GOARCH) } + // Linux cloud images must declare a family; it drives distro-specific + // cloud-init provisioning. gokrazy and non-Linux images don't use that path. + if n.os.isLinuxCloudImage() && n.os.Family == "" { + e.t.Fatalf("AddNode(%q): Linux cloud image %q has no LinuxFamily set", name, n.os.Name) + } + n.vnetNode = e.cfg.AddNode(vnetOpts...) n.num = n.vnetNode.Num() return n @@ -544,6 +558,28 @@ type nodeOptSystemdUnit struct{} type nodeOptAdvertiseRoutes string type nodeOptSNATSubnetRoutes bool type nodeOptWebServer int +type nodeOptDNSMode DNSMode + +// DNSMode is a provisioning directive, not a DNS-backend name: it says what, if +// anything, to do to the guest's DNS before tailscaled starts, letting one +// distro image cover multiple backends. DNSDefault leaves DNS untouched (the +// resulting backend is image-dependent); the other modes provision the guest to +// force a specific backend, and are named to match the mode strings in +// net/dns/manager_linux.go so the result can be checked with +// [Env.AssertDNSBackend]. +type DNSMode string + +const ( + // DNSDefault leaves the image's DNS configuration untouched, so the backend + // is whatever the image runs by default (typically systemd-resolved on + // modern distros). It has no [Env.AssertDNSBackend] counterpart because the + // result isn't forced. + DNSDefault DNSMode = "" + + // DNSDirect masks systemd-resolved and installs a plain /etc/resolv.conf + // so tailscaled selects the "direct" manager (rewrites resolv.conf itself). + DNSDirect DNSMode = "direct" +) // OS returns a NodeOption that sets the node's operating system image. func OS(img OSImage) nodeOptOS { return nodeOptOS(img) } @@ -587,10 +623,22 @@ func SNATSubnetRoutes(v bool) nodeOptSNATSubnetRoutes { return nodeOptSNATSubnet // The webserver responds with "Hello world I am from " on all requests. func WebServer(port int) nodeOptWebServer { return nodeOptWebServer(port) } +// WithDNSMode returns a NodeOption that provisions the (Linux) node so +// tailscaled selects the given DNS backend. Only meaningful for Linux cloud +// images; ignored for gokrazy/macOS. See [DNSMode]. +func WithDNSMode(m DNSMode) nodeOptDNSMode { return nodeOptDNSMode(m) } + // Start initializes the virtual network, boots all VMs in parallel, and waits // for all TTA agents to connect. It should be called after all AddNetwork/AddNode calls. func (e *Env) Start() { t := e.t + + // Give bring-up (image build, boot, agent wait) a generous budget measured + // from now. We deliberately do NOT derive this from the test deadline: on + // CI, per-test timeouts can be tight (a few minutes), and reserving headroom + // under the deadline was observed to steal time bring-up legitimately needs, + // failing slow-but-healthy runs. If bring-up genuinely exceeds this, the + // `go test -timeout` panic is an acceptable (if blunt) backstop. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) t.Cleanup(cancel) e.ctx = ctx @@ -633,7 +681,14 @@ func (e *Env) Start() { // Boot all nodes in parallel. Each platform handles its own // dependencies (image prep, binary compilation, socket setup) // via sync.Once, so independent work overlaps naturally. + // + // Under TCG, concurrent boots oversubscribe the host CPUs and a heavy + // guest can starve its siblings past the stuck-detector; serialize boots + // there so each clears that gate before the next starts. KVM stays parallel. var bootEg errgroup.Group + if !hardwareAccelAvailable() { + bootEg.SetLimit(1) + } for _, n := range e.nodes { bootEg.Go(func() error { return n.platform().boot(ctx, e, n) @@ -999,6 +1054,41 @@ func (e *Env) ClientMetrics(n *Node) ClientMetrics { return out } +// dnsBackendMetricPrefix is the prefix of the clientmetric gauge that +// tailscaled sets to 1 for the Linux DNS mode it selected. See +// net/dns/manager_linux.go. +const dnsBackendMetricPrefix = "dns_manager_linux_mode_" + +// DNSBackend returns the Linux DNS backend ("mode") the node's tailscaled +// selected (e.g. "systemd-resolved", "direct"). tailscaled sets a single gauge +// named dns_manager_linux_mode_ to 1 for its selected mode (see +// net/dns/manager_linux.go); this finds that gauge and returns . It fails +// the test if none is set (non-Linux node, or DNS not yet configured). +func (e *Env) DNSBackend(n *Node) string { + e.t.Helper() + for name, m := range e.ClientMetrics(n) { + mode, ok := strings.CutPrefix(name, dnsBackendMetricPrefix) + if !ok || m.Value != 1 { + continue + } + // tailscaled sanitizes "-" to "_" when forming the metric name; + // reverse it so we return net/dns's spelling ("systemd-resolved"). + return strings.ReplaceAll(mode, "_", "-") + } + e.t.Fatalf("Node %q: no %s* gauge set (non-Linux node, or DNS not yet configured)", n.Name(), dnsBackendMetricPrefix) + return "" +} + +// AssertDNSBackend fails the test unless the node's selected Linux DNS backend +// matches want (see [Env.DNSBackend] for the mode strings). Use it in distro +// tests to prove the node exercises the intended DNS manager. +func (e *Env) AssertDNSBackend(n *Node, want string) { + e.t.Helper() + if got := e.DNSBackend(n); got != want { + e.t.Fatalf("Node %q: DNS backend = %q, want %q", n.Name(), got, want) + } +} + // ClientMetrics is a view of the client metrics exported by a node. // The keys of the map are the metric names. type ClientMetrics map[string]ClientMetric diff --git a/tstest/natlab/vmtest/vmtest_test.go b/tstest/natlab/vmtest/vmtest_test.go index f69fcf7f4..0daae92fa 100644 --- a/tstest/natlab/vmtest/vmtest_test.go +++ b/tstest/natlab/vmtest/vmtest_test.go @@ -84,6 +84,28 @@ func TestSubnetRouterFreeBSD(t *testing.T) { testSubnetRouterForOS(t, vmtest.FreeBSD150) } +func TestSubnetRouterFedora(t *testing.T) { + testSubnetRouterForOS(t, vmtest.Fedora43) +} + +// TestFedoraDNSDirect verifies that provisioning a Fedora node with +// WithDNSMode(DNSDirect) — which masks systemd-resolved — makes tailscaled +// select the "direct" DNS backend instead of the image default +// ("systemd-resolved"). This is what lets one distro image cover multiple DNS +// backends, so adding a distro isn't "basically equivalent" to the others. +func TestFedoraDNSDirect(t *testing.T) { + env := vmtest.New(t) + + net := env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT) + node := env.AddNode("fedora", net, + vmtest.OS(vmtest.Fedora43), + vmtest.WithDNSMode(vmtest.DNSDirect)) + + env.Start() + + env.AssertDNSBackend(node, "direct") +} + func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { t.Helper() env := vmtest.New(t) @@ -107,6 +129,14 @@ func testSubnetRouterForOS(t testing.TB, srOS vmtest.OSImage) { env.Start() + // Log which DNS backend the (Linux) subnet router selected. This is the + // whole point of testing multiple distros: they should exercise different + // DNS managers. Once we've confirmed the expected value per distro, this + // can become an AssertDNSBackend. FreeBSD has no Linux DNS gauge, so skip. + if srOS.GOOS() == "linux" { + t.Logf("subnet-router (%s) DNS backend: %s", srOS.Name, env.DNSBackend(sr)) + } + approveStep.Begin() env.ApproveRoutes(sr, "10.0.0.0/24") approveStep.End(nil)