tstest/integration: run a smoke test against a Windows tailscaled service (#20382)

* tstest/integration: run a test against a real Windows tailscaled service

Updates #20381

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

* tstest/integration: serialize Windows service tests and clean up state

Updates #20381

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

* tstest/integration: address review feedback on Windows service tests

Updates #20381

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

* tstest/integration: use background context for service teardown

Updates #20381

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

* tstest/integration: run Windows service test via the normal windows CI run

Updates #20381

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

* tstest/integration: address review feedback on Windows service tests

Updates #20381

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

---------

Signed-off-by: Yaruk Asghar <yaruk@tailscale.com>

tsconst/wintun/wintun.go

@@ -0,0 +1,24 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package wintun is the single source of truth for the pinned wintun.dll release
+// the Tailscale Windows client is built and tested against.
+package wintun
+
+const (
+	// Version is the pinned wintun release.
+	Version = "0.14.1"
+	// URL is the wintun.net download for Version.
+	URL = "https://www.wintun.net/builds/wintun-" + Version + ".zip"
+	// SHA256 is the hex-encoded sha256 of the zip at URL.
+	SHA256 = "07c256185d6ee3652e09fa55c0b673e2624b565e02c4b9091c79ca7d2f24ef51"
+)
+
+// DLLZipPath returns the path of the wintun.dll for goArch within the release zip.
+func DLLZipPath(goArch string) string {
+	arch := goArch
+	if goArch == "386" {
+		arch = "x86"
+	}
+	return "wintun/bin/" + arch + "/wintun.dll"
+}

tstest/integration/integration.go

@@ -40,6 +40,7 @@ import (
 	"tailscale.com/ipn/ipnstate"
 	"tailscale.com/ipn/store"
 	"tailscale.com/net/stun/stuntest"
+	"tailscale.com/paths"
 	"tailscale.com/safesocket"
 	"tailscale.com/syncs"
 	"tailscale.com/tailcfg"
@@ -49,6 +50,7 @@ import (
 	"tailscale.com/types/logger"
 	"tailscale.com/types/logid"
 	"tailscale.com/types/nettype"
+	"tailscale.com/util/cibuild"
 	"tailscale.com/util/rands"
 	"tailscale.com/util/zstdframe"
 	"tailscale.com/version"
@@ -57,6 +59,10 @@ import (
 var (
 	verboseTailscaled = flag.Bool("verbose-tailscaled", false, "verbose tailscaled logging")
 	verboseTailscale  = flag.Bool("verbose-tailscale", false, "verbose tailscale CLI logging")
+
+	// runWindowsServiceTests enables the Windows service-mode integration tests.
+	// On by default in CI; tests opt in via NewTestEnv(t, canRunAsServiceOnWindows()).
+	runWindowsServiceTests = flag.Bool("run-windows-service-tests", cibuild.On(), "run Windows service-mode integration tests")
 )
 
 // MainError is an error that's set if an error conditions happens outside of a
@@ -98,7 +104,7 @@ type BinaryInfo struct {
 // was cleaned up.
 func (b BinaryInfo) CopyTo(dir string) (BinaryInfo, error) {
 	ret := b
-	ret.Path = filepath.Join(dir, path.Base(b.Path))
+	ret.Path = filepath.Join(dir, filepath.Base(b.Path))
 
 	switch runtime.GOOS {
 	case "linux":
@@ -496,6 +502,7 @@ func (lc *LogCatcher) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 type TestEnv struct {
 	t                      testing.TB
 	tunMode                bool
+	windowsService         bool // run tailscaled as a Windows service
 	cli                    string
 	daemon                 string
 	loopbackPort           *int
@@ -534,11 +541,40 @@ func (f ConfigureControl) ModifyTestEnv(te *TestEnv) {
 	f(te.Control)
 }
 
+// canRunAsServiceOnWindowsOpt is the TestEnvOpt returned by canRunAsServiceOnWindows.
+type canRunAsServiceOnWindowsOpt struct{}
+
+func (canRunAsServiceOnWindowsOpt) ModifyTestEnv(te *TestEnv) {
+	// Only run as a service on Windows; on other platforms the test runs
+	// the normal userspace daemon with a faked Windows GOOS, as it always has.
+	if runtime.GOOS == "windows" {
+		te.windowsService = true
+	}
+}
+
+// canRunAsServiceOnWindows enables the test to run on Windows.
+// TODO(#20464): remove this and explicitly skip tests that need more work
+// before they can run on Windows, instead of requiring tests to opt in with this option.
+func canRunAsServiceOnWindows() TestEnvOpt { return canRunAsServiceOnWindowsOpt{} }
+
 // NewTestEnv starts a bunch of services and returns a new test environment.
 // NewTestEnv arranges for the environment's resources to be cleaned up on exit.
 func NewTestEnv(t testing.TB, opts ...TestEnvOpt) *TestEnv {
+	// Integration tests skip on Windows unless a test opts in via canRunAsServiceOnWindows.
+	// Pre-scan the opts before starting any servers so a skip leaks nothing.
+	canRunAsService := false
+	for _, o := range opts {
+		if _, ok := o.(canRunAsServiceOnWindowsOpt); ok {
+			canRunAsService = true
+		}
+	}
 	if runtime.GOOS == "windows" {
-		t.Skip("not tested/working on Windows yet")
+		if !canRunAsService {
+			t.Skip("integration tests skip on Windows unless the test calls canRunAsServiceOnWindows")
+		}
+		if !*runWindowsServiceTests {
+			t.Skip("Windows service tests disabled (--run-windows-service-tests=false)")
+		}
 	}
 	derpMap := RunDERPAndSTUN(t, logger.Discard, "127.0.0.1")
 	logc := new(LogCatcher)
@@ -608,11 +644,18 @@ func NewTestNode(t *testing.T, env *TestEnv) *TestNode {
 		sockFile = filepath.Join(os.TempDir(), rands.HexString(8)+".sock")
 		t.Cleanup(func() { os.Remove(sockFile) })
 	}
+	stateFile := filepath.Join(dir, "tailscaled.state") // matches what cmd/tailscaled uses
+	if env.windowsService {
+		// A LocalSystem service ignores --socket/--statedir and uses the
+		// default pipe and state path; point the harness at those.
+		sockFile = paths.DefaultTailscaledSocket()
+		stateFile = paths.DefaultTailscaledStateFile()
+	}
 	n := &TestNode{
 		env:       env,
 		dir:       dir,
 		sockFile:  sockFile,
-		stateFile: filepath.Join(dir, "tailscaled.state"), // matches what cmd/tailscaled uses
+		stateFile: stateFile,
 	}
 
 	// Look for a data race or panic.
@@ -775,9 +818,17 @@ func (op *nodeOutputParser) parseLinesLocked() {
 
 type Daemon struct {
 	Process *os.Process
+
+	// svc is set when the daemon is a Windows service (no owned Process);
+	// MustCleanShutdown then stops it via the SCM.
+	svc *TestNode
 }
 
 func (d *Daemon) MustCleanShutdown(t testing.TB) {
+	if d.svc != nil {
+		d.svc.stopService()
+		return
+	}
 	d.Process.Signal(os.Interrupt)
 	ps, err := d.Process.Wait()
 	if err != nil {
@@ -809,6 +860,40 @@ func (n *TestNode) awaitTailscaledRunnable() error {
 	return nil
 }
 
+// daemonEnv returns the extra environment variables to use when starting tailscaled.
+// The ipnGOOS argument overrides [envknob.GOOS].
+func (n *TestNode) daemonEnv(ipnGOOS string) []string {
+	env := []string{
+		"TS_DEBUG_PERMIT_HTTP_C2N=1",
+		"TS_LOG_TARGET=" + n.env.LogCatcherServer.URL,
+		"HTTP_PROXY=" + n.env.TrafficTrapServer.URL,
+		"HTTPS_PROXY=" + n.env.TrafficTrapServer.URL,
+		"TS_DEBUG_FAKE_GOOS=" + ipnGOOS,
+		"TS_LOGS_DIR=" + n.dir,
+		"TS_NETCHECK_GENERATE_204_URL=" + n.env.ControlServer.URL + "/generate_204",
+		"TS_ASSUME_NETWORK_UP_FOR_TEST=1", // don't pause control client in airplane mode (no wifi, etc)
+		"TS_PANIC_IF_HIT_MAIN_CONTROL=1",
+		"TS_DISABLE_PORTMAPPER=1", // shouldn't be needed; test is all localhost
+		"TS_DEBUG_LOG_RATE=all",
+	}
+	if n.allowUpdates {
+		env = append(env, "TS_TEST_ALLOW_AUTO_UPDATE=1")
+	}
+	if n.env.loopbackPort != nil {
+		env = append(env, "TS_DEBUG_NETSTACK_LOOPBACK_PORT="+strconv.Itoa(*n.env.loopbackPort))
+	}
+	if n.env.neverDirectUDP {
+		env = append(env, "TS_DEBUG_NEVER_DIRECT_UDP=1")
+	}
+	if n.env.relayServerUseLoopback {
+		env = append(env, "TS_DEBUG_RELAY_SERVER_ADDRS=::1,127.0.0.1")
+	}
+	if version.IsRace() {
+		env = append(env, "GORACE=halt_on_error=1")
+	}
+	return env
+}
+
 // StartDaemon starts the node's tailscaled, failing if it fails to start.
 // StartDaemon ensures that the process will exit when the test completes.
 func (n *TestNode) StartDaemon() *Daemon {
@@ -822,6 +907,12 @@ func (n *TestNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon {
 		t.Fatalf("awaitTailscaledRunnable: %v", err)
 	}
 
+	if n.env.windowsService {
+		// TODO(#20443): plumb service logs here so races/panics/DEBUG-ADDR are seen in service mode.
+		n.tailscaledParser = &nodeOutputParser{n: n}
+		return n.startWindowsServiceDaemon()
+	}
+
 	cmd := exec.Command(n.env.daemon)
 	cmd.Args = append(cmd.Args,
 		"--statedir="+n.dir,
@@ -843,34 +934,7 @@ func (n *TestNode) StartDaemonAsIPNGOOS(ipnGOOS string) *Daemon {
 	if n.encryptState {
 		cmd.Args = append(cmd.Args, "--encrypt-state")
 	}
-	cmd.Env = append(os.Environ(),
+	cmd.Env = append(os.Environ(), n.daemonEnv(ipnGOOS)...)
-		"TS_DEBUG_PERMIT_HTTP_C2N=1",
-		"TS_LOG_TARGET="+n.env.LogCatcherServer.URL,
-		"HTTP_PROXY="+n.env.TrafficTrapServer.URL,
-		"HTTPS_PROXY="+n.env.TrafficTrapServer.URL,
-		"TS_DEBUG_FAKE_GOOS="+ipnGOOS,
-		"TS_LOGS_DIR="+t.TempDir(),
-		"TS_NETCHECK_GENERATE_204_URL="+n.env.ControlServer.URL+"/generate_204",
-		"TS_ASSUME_NETWORK_UP_FOR_TEST=1", // don't pause control client in airplane mode (no wifi, etc)
-		"TS_PANIC_IF_HIT_MAIN_CONTROL=1",
-		"TS_DISABLE_PORTMAPPER=1", // shouldn't be needed; test is all localhost
-		"TS_DEBUG_LOG_RATE=all",
-	)
-	if n.allowUpdates {
-		cmd.Env = append(cmd.Env, "TS_TEST_ALLOW_AUTO_UPDATE=1")
-	}
-	if n.env.loopbackPort != nil {
-		cmd.Env = append(cmd.Env, "TS_DEBUG_NETSTACK_LOOPBACK_PORT="+strconv.Itoa(*n.env.loopbackPort))
-	}
-	if n.env.neverDirectUDP {
-		cmd.Env = append(cmd.Env, "TS_DEBUG_NEVER_DIRECT_UDP=1")
-	}
-	if n.env.relayServerUseLoopback {
-		cmd.Env = append(cmd.Env, "TS_DEBUG_RELAY_SERVER_ADDRS=::1,127.0.0.1")
-	}
-	if version.IsRace() {
-		cmd.Env = append(cmd.Env, "GORACE=halt_on_error=1")
-	}
 	n.tailscaledParser = &nodeOutputParser{n: n}
 	cmd.Stderr = n.tailscaledParser
 	if *verboseTailscaled {

tstest/integration/integration_test.go

@@ -35,6 +35,7 @@ import (
 	"go4.org/mem"
 	"tailscale.com/client/local"
 	"tailscale.com/cmd/testwrapper/flakytest"
+	"tailscale.com/envknob"
 	"tailscale.com/feature"
 	_ "tailscale.com/feature/clientupdate"
 	"tailscale.com/health"
@@ -57,6 +58,11 @@ func TestMain(m *testing.M) {
 	// Have to disable UPnP which hits the network, otherwise it fails due to HTTP proxy.
 	os.Setenv("TS_DISABLE_UPNP", "true")
 	flag.Parse()
+	if *runWindowsServiceTests && runtime.GOOS == "windows" {
+		// On Windows the service is a singleton, so its tests run serially.
+		// envknob.Setenv refreshes the TS_SERIAL_TESTS that tstest.Parallel reads.
+		envknob.Setenv("TS_SERIAL_TESTS", "true")
+	}
 	v := m.Run()
 	if v != 0 {
 		os.Exit(v)
@@ -1310,7 +1316,7 @@ func TestNoControlConnWhenDown(t *testing.T) {
 // without the GUI to kick off a Start.
 func TestOneNodeUpWindowsStyle(t *testing.T) {
 	tstest.Parallel(t)
-	env := NewTestEnv(t)
+	env := NewTestEnv(t, canRunAsServiceOnWindows())
 	n1 := NewTestNode(t, env)
 	n1.upFlagGOOS = "windows"
 

tstest/integration/service_notwindows.go

@@ -0,0 +1,18 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !windows
+
+package integration
+
+// Non-Windows stubs for the Windows service backend; never called, since
+// windowsService is only set on Windows.
+
+func (n *TestNode) startWindowsServiceDaemon() *Daemon {
+	n.env.t.Fatal("Windows service daemon is only supported on Windows")
+	return nil
+}
+
+func (n *TestNode) stopService() {
+	n.env.t.Fatal("Windows service daemon is only supported on Windows")
+}

tstest/integration/service_windows.go

@@ -0,0 +1,270 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build windows
+
+package integration
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/windows/svc"
+	"golang.org/x/sys/windows/svc/mgr"
+	"tailscale.com/tsconst/wintun"
+)
+
+// serviceName is the Windows service tailscaled installs itself as.
+const serviceName = "Tailscale"
+
+// startWindowsServiceDaemon installs and starts tailscaled as a Windows service.
+func (n *TestNode) startWindowsServiceDaemon() *Daemon {
+	t := n.env.t
+	t.Helper()
+
+	// A pre-existing Tailscale service means this isn't a disposable/CI machine;
+	// fail rather than uninstall it. Stale state is wiped below, not fatal.
+	if serviceExists(t) {
+		t.Fatal("existing Tailscale service found; run only on a disposable/CI machine")
+	}
+
+	n.cleanupServiceState()
+	stageWintun(t, filepath.Dir(n.env.daemon))
+	n.writeServiceEnvFile()
+
+	if out, err := exec.CommandContext(t.Context(), n.env.daemon, "install-system-daemon").CombinedOutput(); err != nil {
+		t.Fatalf("install-system-daemon: %v\n%s", err, out)
+	}
+	// Teardown (LIFO): stop, uninstall, then wipe state for the next test.
+	t.Cleanup(func() {
+		n.stopService()
+		n.uninstallService()
+		n.cleanupServiceState()
+	})
+
+	n.startService()
+	n.waitServiceReady(90 * time.Second)
+	return &Daemon{svc: n}
+}
+
+// startService starts the service and waits for the SCM to report it Running.
+func (n *TestNode) startService() {
+	t := n.env.t
+	t.Helper()
+	m := connectSCM(t)
+	defer m.Disconnect()
+	s, err := m.OpenService(serviceName)
+	if err != nil {
+		t.Fatalf("open service %q: %v", serviceName, err)
+	}
+	defer s.Close()
+	if err := s.Start(); err != nil {
+		t.Fatalf("start service %q: %v", serviceName, err)
+	}
+	n.waitServiceState(s, svc.Running, 60*time.Second)
+}
+
+// stopService requests a stop and waits until the service is Stopped, failing
+// the test if it doesn't stop in time.
+func (n *TestNode) stopService() {
+	t := n.env.t
+	t.Helper()
+	m := connectSCM(t)
+	defer m.Disconnect()
+	s, err := m.OpenService(serviceName)
+	if err != nil {
+		return // not installed; nothing to stop
+	}
+	defer s.Close()
+	st, err := s.Query()
+	if err != nil {
+		t.Fatalf("query service %q: %v", serviceName, err)
+	}
+	if st.State == svc.Stopped {
+		return
+	}
+	if _, err := s.Control(svc.Stop); err != nil {
+		t.Fatalf("stop service %q: %v", serviceName, err)
+	}
+	n.waitServiceState(s, svc.Stopped, 60*time.Second)
+}
+
+// uninstallService removes the service via tailscaled's uninstall-system-daemon
+// and waits until it's gone; a missing service is fine.
+func (n *TestNode) uninstallService() {
+	t := n.env.t
+	t.Helper()
+	if !serviceExists(t) {
+		return
+	}
+	// Not t.Context(): this also runs from t.Cleanup, where it's canceled.
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+	if out, err := exec.CommandContext(ctx, n.env.daemon, "uninstall-system-daemon").CombinedOutput(); err != nil {
+		t.Fatalf("uninstall-system-daemon: %v\n%s", err, out)
+	}
+	n.waitServiceGone(30 * time.Second)
+}
+
+// writeServiceEnvFile writes the harness env to the file tailscaled reads at
+// startup, since a service doesn't inherit the test process's environment.
+func (n *TestNode) writeServiceEnvFile() {
+	t := n.env.t
+	t.Helper()
+	dir := serviceStateDir()
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		t.Fatalf("creating %s: %v", dir, err)
+	}
+	dst := filepath.Join(dir, "tailscaled-env.txt")
+	body := strings.Join(n.daemonEnv("windows"), "\n") + "\n"
+	if err := os.WriteFile(dst, []byte(body), 0o644); err != nil {
+		t.Fatalf("writing %s: %v", dst, err)
+	}
+}
+
+// cleanupServiceState removes the service's global state dir so the next test
+// doesn't inherit a prior node's identity; call only after the service stops.
+func (n *TestNode) cleanupServiceState() {
+	t := n.env.t
+	t.Helper()
+	dir := serviceStateDir()
+	if err := os.RemoveAll(dir); err != nil {
+		t.Logf("removing %s: %v", dir, err)
+	}
+}
+
+// serviceStateDir is the global state dir a Tailscale service uses.
+func serviceStateDir() string {
+	return filepath.Join(os.Getenv("ProgramData"), "Tailscale")
+}
+
+// waitServiceReady polls the LocalAPI until BackendState leaves NoState,
+// guarding the post-start race (tailscale/tailscale#8695).
+func (n *TestNode) waitServiceReady(timeout time.Duration) {
+	t := n.env.t
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	var last error
+	for time.Now().Before(deadline) {
+		st, err := n.Status()
+		if err == nil && st.BackendState != "" && st.BackendState != "NoState" {
+			return
+		}
+		last = err
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatalf("service LocalAPI not ready within %v (last err: %v)", timeout, last)
+}
+
+// waitServiceState polls s until it reaches want, failing the test on timeout.
+func (n *TestNode) waitServiceState(s *mgr.Service, want svc.State, timeout time.Duration) {
+	t := n.env.t
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		st, err := s.Query()
+		if err != nil {
+			t.Fatalf("query service %q: %v", serviceName, err)
+		}
+		if st.State == want {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+	t.Fatalf("service %q did not reach state %d within %v", serviceName, want, timeout)
+}
+
+// waitServiceGone polls until the service no longer exists.
+func (n *TestNode) waitServiceGone(timeout time.Duration) {
+	t := n.env.t
+	t.Helper()
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		if !serviceExists(t) {
+			return
+		}
+		time.Sleep(time.Second)
+	}
+	t.Fatalf("service %q still present after %v", serviceName, timeout)
+}
+
+// serviceExists reports whether the Tailscale service is currently installed.
+func serviceExists(t testing.TB) bool {
+	t.Helper()
+	m := connectSCM(t)
+	defer m.Disconnect()
+	s, err := m.OpenService(serviceName)
+	if err != nil {
+		return false
+	}
+	s.Close()
+	return true
+}
+
+// connectSCM connects to the Windows service manager, failing the test on error.
+func connectSCM(t testing.TB) *mgr.Mgr {
+	t.Helper()
+	m, err := mgr.Connect()
+	if err != nil {
+		t.Fatalf("connect to service manager: %v", err)
+	}
+	return m
+}
+
+// stageWintun downloads and verifies wintun.dll into dir; tailscaled loads it
+// from its executable's dir (cmd/tailscaled.fullyQualifiedWintunPath).
+func stageWintun(t testing.TB, dir string) {
+	t.Helper()
+	req, err := http.NewRequestWithContext(t.Context(), "GET", wintun.URL, nil)
+	if err != nil {
+		t.Fatalf("wintun request: %v", err)
+	}
+	res, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("downloading wintun: %v", err)
+	}
+	defer res.Body.Close()
+	if res.StatusCode != http.StatusOK {
+		t.Fatalf("downloading %s: HTTP %s", wintun.URL, res.Status)
+	}
+	zipBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		t.Fatalf("reading wintun zip: %v", err)
+	}
+	if sum := sha256.Sum256(zipBytes); hex.EncodeToString(sum[:]) != wintun.SHA256 {
+		t.Fatalf("wintun zip sha256 = %s, want %s", hex.EncodeToString(sum[:]), wintun.SHA256)
+	}
+	zr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+	if err != nil {
+		t.Fatalf("opening wintun zip: %v", err)
+	}
+	member := wintun.DLLZipPath(runtime.GOARCH)
+	f, err := zr.Open(member)
+	if err != nil {
+		t.Fatalf("wintun zip missing %q: %v", member, err)
+	}
+	defer f.Close()
+	out, err := os.Create(filepath.Join(dir, "wintun.dll"))
+	if err != nil {
+		t.Fatalf("creating wintun.dll: %v", err)
+	}
+	if _, err := io.Copy(out, f); err != nil {
+		out.Close()
+		t.Fatalf("extracting wintun.dll: %v", err)
+	}
+	if err := out.Close(); err != nil {
+		t.Fatalf("closing wintun.dll: %v", err)
+	}
+}