tstest, cmd/tta: add Tailscale SSH end-to-end VM test

Add TestTailscaleSSH to tstest/natlab/vmtest, exercising the Tailscale
SSH server (tailscale up --ssh, not a system sshd) end to end: an
Ubuntu client node SSHes over the tailnet into an Ubuntu server node
as both root and a non-root user, and into a gokrazy node.

The gokrazy sessions exercise the gokrazy special cases in the SSH
code: util/osuser hard-codes the login shell to serial-busybox ash and
synthesizes a root user when lookup fails (so any username works and
becomes root, unlike Ubuntu where nonexistent users are rejected), and
the incubator's findSU refuses su on gokrazy, handling sessions
in-process.

To support this, testcontrol gains an SSHPolicy field that's sent in
MapResponses along with the CapabilitySSH node capability, tta's /up
handler accepts an ssh=true parameter, and vmtest gains a
TailscaleSSH node option that wires the two together with a
permissive any-principal policy.

Updates tailscale/corp#44813
Updates #13038

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
Change-Id: I3f6b9c41a72e05d8c94dd7f6ab1937cf24b81c92

cmd/tta/tta.go

@@ -198,6 +198,9 @@ func main() {
 		if r.URL.Query().Get("accept-routes") == "true" {
 			args = append(args, "--accept-routes")
 		}
+		if r.URL.Query().Get("ssh") == "true" {
+			args = append(args, "--ssh")
+		}
 		serveCmd(w, "tailscale", args...)
 	})
 	ttaMux.HandleFunc("/set", func(w http.ResponseWriter, r *http.Request) {

tstest/integration/testcontrol/testcontrol.go

@@ -66,6 +66,11 @@ type Server struct {
 	// grants rules.
 	PeerRelayGrants bool
 
+	// SSHPolicy, if non-nil, is sent to every node in MapResponses.
+	// Each node also gets [tailcfg.CapabilitySSH] added to its capability
+	// map, permitting "tailscale up --ssh".
+	SSHPolicy *tailcfg.SSHPolicy
+
 	// AllNodesSameUser, if true, makes all created nodes
 	// belong to the same user.
 	AllNodesSameUser bool
@@ -1612,10 +1617,14 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		dns = s.DNSConfig.Clone()
 	}
 	magicDNSDomain := s.MagicDNSDomain
+	sshPolicy := s.SSHPolicy.Clone()
 	s.mu.Unlock()
 
 	node.CapMap = nodeCapMap
 	node.Capabilities = append(node.Capabilities, tailcfg.NodeAttrDisableUPnP)
+	if sshPolicy != nil {
+		mak.Set(&node.CapMap, tailcfg.CapabilitySSH, nil)
+	}
 
 	t := time.Date(2020, 8, 3, 0, 0, 0, 1, time.UTC)
 	if dns != nil && magicDNSDomain != "" {
@@ -1629,6 +1638,7 @@ func (s *Server) MapResponse(req *tailcfg.MapRequest) (res *tailcfg.MapResponse,
 		CollectServices: cmp.Or(s.CollectServices, opt.True),
 		PacketFilter:    packetFilterWithIngress(s.PeerRelayGrants),
 		DNSConfig:       dns,
+		SSHPolicy:       sshPolicy,
 		ControlTime:     &t,
 	}
 

tstest/natlab/vmtest/ssh_test.go

@@ -0,0 +1,151 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"fmt"
+	"net/netip"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/creachadair/mds/shell"
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+// TestTailscaleSSH exercises the Tailscale SSH server ("tailscale up --ssh",
+// not a system sshd) on both Ubuntu and gokrazy nodes, with an Ubuntu node
+// as the SSH client.
+//
+// On Ubuntu it tests logging in as both root and a non-root user, which
+// exercises the incubator's su-based login path.
+//
+// On gokrazy it exercises the gokrazy special cases in the SSH code:
+// util/osuser hard-codes the login shell to /tmp/serial-busybox/ash and
+// falls back to a synthesized root user (uid 0, home dir "/") when user
+// lookup fails, since gokrazy has no user database; and the incubator's
+// findSU refuses to use su on gokrazy, so sessions are handled in-process.
+func TestTailscaleSSH(t *testing.T) {
+	env := vmtest.New(t)
+	client := sshTestNode(env, "client", vmtest.Ubuntu2404)
+	ubuntu := sshTestNode(env, "ubuntu", vmtest.Ubuntu2404, vmtest.TailscaleSSH())
+	gokrazy := sshTestNode(env, "gokrazy", vmtest.Gokrazy, vmtest.TailscaleSSH())
+	env.Start()
+
+	if out, err := env.SSHExec(ubuntu, "useradd -m -s /bin/bash tsuser"); err != nil {
+		t.Fatalf("useradd: %v\n%s", err, out)
+	}
+
+	ubuntuIP := tailscaleIP4(t, env, ubuntu)
+	gokrazyIP := tailscaleIP4(t, env, gokrazy)
+
+	// ssh runs cmd as user on the node at ip over Tailscale SSH, from the
+	// client node, returning the combined output and the ssh client's exit
+	// code. The command run via Env.SSHExec always exits 0 so that its
+	// transport-error (exit 255) retry loop doesn't kick in when a
+	// Tailscale SSH connection is expected to fail.
+	const exitMarker = "tailscale-ssh-exit="
+	ssh := func(user string, ip netip.Addr, cmd string) (string, int) {
+		t.Helper()
+		inner := "ssh" +
+			" -o StrictHostKeyChecking=no" +
+			" -o UserKnownHostsFile=/dev/null" +
+			" -o BatchMode=yes" +
+			" -o ConnectTimeout=10" +
+			" -o LogLevel=ERROR" +
+			" " + user + "@" + ip.String() +
+			" " + shell.Quote(cmd)
+		out, err := env.SSHExec(client, inner+" 2>&1; echo "+exitMarker+"$?")
+		if err != nil {
+			t.Fatalf("ssh %s@%s: %v\n%s", user, ip, err, out)
+		}
+		i := strings.LastIndex(out, exitMarker)
+		if i == -1 {
+			t.Fatalf("ssh %s@%s: no exit marker in output:\n%s", user, ip, out)
+		}
+		code, err := strconv.Atoi(strings.TrimSpace(out[i+len(exitMarker):]))
+		if err != nil {
+			t.Fatalf("ssh %s@%s: bad exit marker in output:\n%s", user, ip, out)
+		}
+		return strings.TrimSpace(out[:i]), code
+	}
+
+	// waitSSH waits for the node's Tailscale SSH server to accept a
+	// connection; the first one can race tailscaled's SSH server startup
+	// and WireGuard path setup.
+	waitSSH := func(name, user string, ip netip.Addr) {
+		t.Helper()
+		deadline := time.Now().Add(2 * time.Minute)
+		for {
+			out, code := ssh(user, ip, "echo ok")
+			if code == 0 && strings.Contains(out, "ok") {
+				t.Logf("[%s] Tailscale SSH up as %s@%s", name, user, ip)
+				return
+			}
+			if time.Now().After(deadline) {
+				t.Fatalf("[%s] Tailscale SSH as %s@%s never came up; last output (exit %d):\n%s", name, user, ip, code, out)
+			}
+			time.Sleep(2 * time.Second)
+		}
+	}
+	waitSSH("ubuntu", "root", ubuntuIP)
+	waitSSH("gokrazy", "root", gokrazyIP)
+
+	check := func(desc, user string, ip netip.Addr, cmd, want string) {
+		t.Helper()
+		out, code := ssh(user, ip, cmd)
+		if code != 0 {
+			t.Errorf("%s: ssh %s@%s %q exited %d:\n%s", desc, user, ip, cmd, code, out)
+			return
+		}
+		if out != want {
+			t.Errorf("%s: ssh %s@%s %q = %q, want %q", desc, user, ip, cmd, out, want)
+		}
+	}
+
+	check("ubuntu root login", "root", ubuntuIP, "id -un && pwd", "root\n/root")
+	check("ubuntu non-root login", "tsuser", ubuntuIP, "id -un && pwd", "tsuser\n/home/tsuser")
+
+	// A nonexistent user is rejected on Ubuntu...
+	if out, code := ssh("nosuchuser", ubuntuIP, "true"); code == 0 {
+		t.Errorf("ubuntu nonexistent user: ssh succeeded, want failure:\n%s", out)
+	}
+
+	// ... but on gokrazy any username works and becomes root, via the
+	// synthesized-user fallback in util/osuser (uid 0, home dir "/").
+	// $0 is the login shell the session ran the command with, which on
+	// gokrazy is hard-coded rather than looked up with getent. These
+	// sessions also implicitly exercise the incubator's su-less
+	// in-process path, since gokrazy has no su.
+	check("gokrazy root login shell", "root", gokrazyIP, "echo $0", "/tmp/serial-busybox/ash")
+	check("gokrazy nonexistent user maps to root", "nosuchuser", gokrazyIP, "pwd", "/")
+}
+
+// sshTestNode adds a node named name running img behind its own
+// easy-NAT network.
+func sshTestNode(env *vmtest.Env, name string, img vmtest.OSImage, opts ...any) *vmtest.Node {
+	n := env.NumNodes()
+	opts = append([]any{
+		env.AddNetwork(
+			fmt.Sprintf("2.%d.%d.%d", n, n, n), // public IP
+			fmt.Sprintf("192.168.%d.1/24", n), vnet.EasyNAT),
+		vmtest.OS(img),
+	}, opts...)
+	return env.AddNode(name, opts...)
+}
+
+// tailscaleIP4 returns the node's IPv4 Tailscale address.
+func tailscaleIP4(t *testing.T, env *vmtest.Env, n *vmtest.Node) netip.Addr {
+	t.Helper()
+	st := env.Status(n)
+	for _, ip := range st.Self.TailscaleIPs {
+		if ip.Is4() {
+			return ip
+		}
+	}
+	t.Fatalf("no IPv4 Tailscale address for %s; have %v", n.Name(), st.Self.TailscaleIPs)
+	panic("unreachable")
+}

tstest/natlab/vmtest/vmtest.go

@@ -428,6 +428,7 @@ type Node struct {
 	vnetNode         *vnet.Node // primary vnet node (set during Start)
 	agent            *vnet.NodeAgentClient
 	joinTailnet      bool
+	runSSH           bool // true to enable the node's Tailscale SSH server
 	noAgent          bool // true to skip TTA agent setup (e.g. macOS VMs without TTA)
 	advertiseRoutes  string
 	snatSubnetRoutes *bool // nil means default (true)
@@ -458,6 +459,8 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 		case nodeOptNoTailscale:
 			n.joinTailnet = false
 			vnetOpts = append(vnetOpts, vnet.DontJoinTailnet)
+		case nodeOptTailscaleSSH:
+			n.runSSH = true
 		case nodeOptNoAgent:
 			n.noAgent = true
 		case nodeOptAdvertiseRoutes:
@@ -515,6 +518,7 @@ func (n *Node) DropControlTraffic() {
 
 type nodeOptOS OSImage
 type nodeOptNoTailscale struct{}
+type nodeOptTailscaleSSH struct{}
 type nodeOptNoAgent struct{}
 type nodeOptAdvertiseRoutes string
 type nodeOptSNATSubnetRoutes bool
@@ -526,6 +530,13 @@ func OS(img OSImage) nodeOptOS { return nodeOptOS(img) }
 // DontJoinTailnet returns a NodeOption that prevents the node from running tailscale up.
 func DontJoinTailnet() nodeOptNoTailscale { return nodeOptNoTailscale{} }
 
+// TailscaleSSH returns a NodeOption that enables the node's Tailscale SSH
+// server by passing --ssh to tailscale up. If any node has this option, the
+// test control server is configured with a permissive SSH policy that lets
+// any tailnet node connect as any SSH user, mapped to the same-named local
+// user.
+func TailscaleSSH() nodeOptTailscaleSSH { return nodeOptTailscaleSSH{} }
+
 // NoAgent returns a NodeOption that skips TTA agent setup. The node will not
 // have a test agent, so agent-dependent operations (Status, ExecOnNode, etc.)
 // won't work. Useful for VMs that just need to boot and respond to ICMP.
@@ -709,6 +720,9 @@ func (e *Env) tailscaleUp(ctx context.Context, n *Node) error {
 	if n.advertiseRoutes != "" {
 		url += "&advertise-routes=" + n.advertiseRoutes
 	}
+	if n.runSSH {
+		url += "&ssh=true"
+	}
 	if n.snatSubnetRoutes != nil {
 		if *n.snatSubnetRoutes {
 			url += "&snat-subnet-routes=true"
@@ -1636,6 +1650,18 @@ func (e *Env) initVnet() {
 			}
 			cs.DNSConfig.Proxied = true
 		}
+		for _, n := range e.nodes {
+			if n.runSSH {
+				e.server.ControlServer().SSHPolicy = &tailcfg.SSHPolicy{
+					Rules: []*tailcfg.SSHRule{{
+						Principals: []*tailcfg.SSHPrincipal{{Any: true}},
+						SSHUsers:   map[string]string{"*": "="},
+						Action:     &tailcfg.SSHAction{Accept: true},
+					}},
+				}
+				break
+			}
+		}
 	})
 }