.github/workflows/test.yml

@@ -365,30 +365,6 @@ jobs:
       working-directory: src
       run: ./tool/go test $(./tool/go run ./tool/listpkgs --has-root-tests)
 
-  vm:
-    needs: gomod-cache
-    runs-on: ["self-hosted", "linux", "vm"]
-    # VM tests run with some privileges, don't let them run on 3p PRs.
-    if: github.repository == 'tailscale/tailscale'
-    steps:
-    - name: checkout
-      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-      with:
-        path: src
-    - name: Restore Go module cache
-      uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
-      with:
-        path: gomodcache
-        key: ${{ needs.gomod-cache.outputs.cache-key }}
-        enableCrossOsArchive: true
-    - name: Run VM tests
-      working-directory: src
-      run: ./tool/go test ./tstest/integration/vms -v -no-s3 -run-vm-tests -run=TestRunUbuntu2404
-      env:
-        HOME: "/var/lib/ghrunner/home"
-        TMPDIR: "/tmp"
-        XDG_CACHE_HOME: "/var/lib/ghrunner/cache"
-
   cross: # cross-compile checks, build only.
     needs: gomod-cache
     strategy:
@@ -912,7 +888,6 @@ jobs:
       - test
       - windows
       - macos
-      - vm
       - cross
       - ios
       - wasm
@@ -958,7 +933,6 @@ jobs:
       - test
       - windows
       - macos
-      - vm
       - cross
       - ios
       - wasm
@@ -1008,7 +982,6 @@ jobs:
       - test
       - windows
       - macos
-      - vm
       - wasm
       - fuzz
       - race-root-integration

tstest/natlab/vmtest/cloudinit.go

@@ -6,6 +6,7 @@ package vmtest
 import (
 	"crypto/ed25519"
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/pem"
 	"fmt"
 	"os"
@@ -110,6 +111,10 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		ud.WriteString(fmt.Sprintf("    ssh_authorized_keys:\n      - %s\n", strings.TrimSpace(string(pubkey))))
 	}
 
+	if n.systemdUnit {
+		e.writeSystemdUnitFiles(&ud, n)
+	}
+
 	ud.WriteString("runcmd:\n")
 
 	// Remove the default route from the debug NIC (enp0s4) so traffic goes through vnet.
@@ -130,12 +135,21 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 		ud.WriteString("  - [\"sysctl\", \"-w\", \"net.ipv6.conf.all.forwarding=1\"]\n")
 	}
 
-	// Start tailscaled in the background. --statedir provides a VarRoot so
-	// features like Taildrop (which needs a place to stash incoming files)
-	// have a directory to work with.
-	ud.WriteString("  - [\"mkdir\", \"-p\", \"/var/lib/tailscale\"]\n")
-	fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"%s/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale &\"]\n", tailscaledEnvPrefix(n))
-	ud.WriteString("  - [\"sleep\", \"2\"]\n")
+	// Start tailscaled, either via the stock systemd unit or directly in
+	// the background. --statedir provides a VarRoot so features like
+	// Taildrop (which needs a place to stash incoming files) have a
+	// directory to work with.
+	if n.systemdUnit {
+		// The unit's ExecStart runs /usr/sbin/tailscaled.
+		ud.WriteString("  - [\"cp\", \"/usr/local/bin/tailscaled\", \"/usr/sbin/tailscaled\"]\n")
+		ud.WriteString("  - [\"systemctl\", \"daemon-reload\"]\n")
+		// Type=notify makes this block until tailscaled reports readiness.
+		ud.WriteString("  - [\"systemctl\", \"start\", \"tailscaled.service\"]\n")
+	} else {
+		ud.WriteString("  - [\"mkdir\", \"-p\", \"/var/lib/tailscale\"]\n")
+		fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"%s/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale &\"]\n", tailscaledEnvPrefix(n))
+		ud.WriteString("  - [\"sleep\", \"2\"]\n")
+	}
 
 	// Start tta (Tailscale Test Agent).
 	ud.WriteString("  - [\"/bin/sh\", \"-c\", \"/usr/local/bin/tta &\"]\n")
@@ -225,6 +239,40 @@ func (e *Env) generateFreeBSDUserData(n *Node) string {
 	return ud.String()
 }
 
+// writeSystemdUnitFiles appends a cloud-init write_files section that
+// installs the stock tailscaled systemd unit from the source tree, along
+// with the packaging's /etc/default/tailscaled EnvironmentFile (plus any
+// per-node TailscaledEnv variables). File contents are base64-encoded to
+// sidestep YAML quoting.
+func (e *Env) writeSystemdUnitFiles(ud *strings.Builder, n *Node) {
+	modRoot, err := findModRoot()
+	if err != nil {
+		e.t.Fatalf("finding module root for tailscaled.service: %v", err)
+	}
+	unit, err := os.ReadFile(filepath.Join(modRoot, "cmd/tailscaled/tailscaled.service"))
+	if err != nil {
+		e.t.Fatalf("reading tailscaled.service: %v", err)
+	}
+	defaults, err := os.ReadFile(filepath.Join(modRoot, "cmd/tailscaled/tailscaled.defaults"))
+	if err != nil {
+		e.t.Fatalf("reading tailscaled.defaults: %v", err)
+	}
+	var envFile strings.Builder
+	envFile.Write(defaults)
+	for _, env := range n.vnetNode.Env() {
+		fmt.Fprintf(&envFile, "%s=%q\n", env.Key, env.Value)
+	}
+
+	ud.WriteString("write_files:\n")
+	writeFile := func(path string, content []byte) {
+		fmt.Fprintf(ud, "  - path: %s\n", path)
+		fmt.Fprintf(ud, "    encoding: b64\n")
+		fmt.Fprintf(ud, "    content: %s\n", base64.StdEncoding.EncodeToString(content))
+	}
+	writeFile("/etc/systemd/system/tailscaled.service", unit)
+	writeFile("/etc/default/tailscaled", []byte(envFile.String()))
+}
+
 func tailscaledEnvPrefix(n *Node) string {
 	env := n.vnetNode.Env()
 	if len(env) == 0 {

tstest/natlab/vmtest/dns_test.go

@@ -0,0 +1,56 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"tailscale.com/tailcfg"
+	"tailscale.com/tstest"
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+	"tailscale.com/types/dnstype"
+)
+
+// TestDNSExtraRecordsSearchDomains verifies that control-plane DNS config
+// makes it all the way into an Ubuntu guest's OS resolver: an ExtraRecords
+// entry ("extratest.record" → 1.2.3.4) resolves via libc (getent), and the
+// "record" search domain lets the bare name "extratest" resolve too. That
+// requires tailscaled to have plumbed MagicDNS routes and search domains
+// into systemd-resolved.
+func TestDNSExtraRecordsSearchDomains(t *testing.T) {
+	env := vmtest.New(t, vmtest.ControlDNS("tailnet.test", &tailcfg.DNSConfig{
+		Proxied: true,
+		Domains: []string{"record"},
+		Routes:  map[string][]*dnstype.Resolver{"record": nil},
+		ExtraRecords: []tailcfg.DNSRecord{
+			{Name: "extratest.record", Type: "A", Value: "1.2.3.4"},
+		},
+	}))
+	node := env.AddNode("node",
+		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
+		vmtest.OS(vmtest.Ubuntu2404))
+	env.Start()
+
+	for _, name := range []string{"extratest.record", "extratest"} {
+		// Retry for a bit: tailscaled applies the DNS config to
+		// systemd-resolved asynchronously after coming up.
+		if err := tstest.WaitFor(30*time.Second, func() error {
+			out, err := env.SSHExec(node, "getent hosts "+name)
+			if err != nil {
+				return fmt.Errorf("getent hosts %s: %v (%s)", name, err, strings.TrimSpace(out))
+			}
+			if !strings.Contains(out, "1.2.3.4") {
+				return fmt.Errorf("getent hosts %s = %q, want it to contain 1.2.3.4", name, strings.TrimSpace(out))
+			}
+			return nil
+		}); err != nil {
+			out, _ := env.SSHExec(node, "resolvectl status; cat /etc/resolv.conf")
+			t.Fatalf("%v\nresolver state:\n%s", err, out)
+		}
+	}
+}

tstest/natlab/vmtest/systemd_test.go

@@ -0,0 +1,48 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"strings"
+	"testing"
+
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+// TestUbuntuSystemdUnit runs tailscaled on an Ubuntu node via the stock
+// systemd unit that Linux packages ship (cmd/tailscaled/tailscaled.service)
+// rather than launching the binary directly, exercising the unit's
+// directives (EnvironmentFile, RuntimeDirectory, StateDirectory) and its
+// Type=notify readiness handshake. Env.Start already asserts that the node
+// reaches the Running backend state under that unit.
+func TestUbuntuSystemdUnit(t *testing.T) {
+	env := vmtest.New(t)
+	node := env.AddNode("node",
+		env.AddNetwork("2.1.1.1", "192.168.1.1/24", vnet.EasyNAT),
+		vmtest.OS(vmtest.Ubuntu2404),
+		vmtest.SystemdUnit())
+	env.Start()
+
+	// With Type=notify, systemd only reports the unit active after
+	// tailscaled sends READY=1, so this also verifies the sd_notify
+	// handshake worked.
+	out, err := env.SSHExec(node, "systemctl is-active tailscaled.service")
+	if err != nil {
+		t.Fatalf("systemctl is-active: %v\n%s", err, out)
+	}
+	if got := strings.TrimSpace(out); got != "active" {
+		t.Fatalf("tailscaled.service state = %q, want %q", got, "active")
+	}
+
+	// Verify the tailscaled that TTA talked to is the one systemd runs,
+	// not a stray process started some other way.
+	out, err = env.SSHExec(node, `test "$(systemctl show -p MainPID --value tailscaled.service)" = "$(pidof tailscaled)" && echo match`)
+	if err != nil {
+		t.Fatalf("MainPID check: %v\n%s", err, out)
+	}
+	if got := strings.TrimSpace(out); got != "match" {
+		t.Fatalf("MainPID check output = %q, want %q", got, "match")
+	}
+}

tstest/natlab/vmtest/vmtest.go

@@ -97,6 +97,9 @@ type Env struct {
 	selfSignedDERPCertPinning bool // serve test DERP map with sha256-raw cert pins
 	fakeACME                  bool // point tailscaled at vnet's fake ACME server
 
+	controlDNSDomain string             // MagicDNS domain for the test control server (see ControlDNS)
+	controlDNS       *tailcfg.DNSConfig // DNS config for the test control server (see ControlDNS)
+
 	// Shared resource initialization (sync.Once for things multiple nodes share).
 	vnetOnce      sync.Once
 	gokrazyOnce   sync.Once
@@ -403,6 +406,16 @@ func FakeACME() EnvOption {
 	return envOptFunc(func(e *Env) { e.fakeACME = true })
 }
 
+// ControlDNS returns an [EnvOption] that makes the test control server send
+// the given DNS configuration in MapResponses, with node names placed under
+// the given MagicDNS domain (e.g. "tailnet.test").
+func ControlDNS(magicDNSDomain string, cfg *tailcfg.DNSConfig) EnvOption {
+	return envOptFunc(func(e *Env) {
+		e.controlDNSDomain = magicDNSDomain
+		e.controlDNS = cfg
+	})
+}
+
 // AddNetwork creates a new virtual network. Arguments follow the same pattern as
 // vnet.Config.AddNetwork (string IPs, NAT types, NetworkService values).
 func (e *Env) AddNetwork(opts ...any) *vnet.Network {
@@ -430,6 +443,7 @@ type Node struct {
 	joinTailnet      bool
 	runSSH           bool // true to enable the node's Tailscale SSH server
 	noAgent          bool // true to skip TTA agent setup (e.g. macOS VMs without TTA)
+	systemdUnit      bool // true to run tailscaled via the stock systemd unit (Linux cloud VMs only)
 	advertiseRoutes  string
 	snatSubnetRoutes *bool // nil means default (true)
 	webServerPort    int
@@ -463,6 +477,8 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 			n.runSSH = true
 		case nodeOptNoAgent:
 			n.noAgent = true
+		case nodeOptSystemdUnit:
+			n.systemdUnit = true
 		case nodeOptAdvertiseRoutes:
 			n.advertiseRoutes = string(o)
 		case nodeOptSNATSubnetRoutes:
@@ -482,6 +498,10 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 		})
 	}
 
+	if n.systemdUnit && (n.os.IsGokrazy || n.os.GOOS() != "linux") {
+		e.t.Fatalf("SystemdUnit is only supported on Linux cloud VMs; node %s is %s", name, n.os.Name)
+	}
+
 	// macOS VMs require a macOS arm64 host (Apple Virtualization.framework via
 	// tailmac). Skip the test now rather than letting it proceed through the
 	// rest of the setup only to fail later.
@@ -520,6 +540,7 @@ type nodeOptOS OSImage
 type nodeOptNoTailscale struct{}
 type nodeOptTailscaleSSH struct{}
 type nodeOptNoAgent struct{}
+type nodeOptSystemdUnit struct{}
 type nodeOptAdvertiseRoutes string
 type nodeOptSNATSubnetRoutes bool
 type nodeOptWebServer int
@@ -542,6 +563,14 @@ func TailscaleSSH() nodeOptTailscaleSSH { return nodeOptTailscaleSSH{} }
 // won't work. Useful for VMs that just need to boot and respond to ICMP.
 func NoAgent() nodeOptNoAgent { return nodeOptNoAgent{} }
 
+// SystemdUnit returns a NodeOption that makes the node run tailscaled via the
+// stock systemd unit that Linux packages ship (cmd/tailscaled/tailscaled.service
+// with cmd/tailscaled/tailscaled.defaults as its EnvironmentFile), instead of
+// launching tailscaled directly as a background process. This exercises the
+// unit's sandboxing directives and its Type=notify readiness handshake.
+// It is only supported on Linux cloud VMs (e.g. Ubuntu, Debian).
+func SystemdUnit() nodeOptSystemdUnit { return nodeOptSystemdUnit{} }
+
 // AdvertiseRoutes returns a NodeOption that configures the node to advertise
 // the given routes (comma-separated CIDRs) when joining the tailnet.
 func AdvertiseRoutes(routes string) nodeOptAdvertiseRoutes {
@@ -1642,6 +1671,11 @@ func (e *Env) initVnet() {
 		if e.selfSignedDERPCertPinning {
 			e.server.ControlServer().DERPMap = e.buildSelfSignedDERPMap()
 		}
+		if e.controlDNS != nil {
+			cs := e.server.ControlServer()
+			cs.MagicDNSDomain = e.controlDNSDomain
+			cs.DNSConfig = e.controlDNS.Clone()
+		}
 		if e.fakeACME {
 			cs := e.server.ControlServer()
 			cs.MagicDNSDomain = "tailnet.test"