tstest/natlab: add ACME cert vmtest

This adds a fake vnet ACME service, TXT-backed SetDNS support, and a
VM test that fetches a certificate with tailscale cert, serves it with
tailscale serve, and verifies HTTPS from a second node.

This adds coverage motivated by #19915.

Updates #13038

Change-Id: Ie1e53409509337d81c8fbceb63f59f3dfbd48207
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

tstest/integration/testcontrol/testcontrol.go

@@ -60,6 +60,7 @@ type Server struct {
 	DNSConfig          *tailcfg.DNSConfig // nil means no DNS config
 	MagicDNSDomain     string
 	C2NResponses       syncs.Map[string, func(*http.Response)] // token => onResponse func
+	OnSetDNS           func(*tailcfg.SetDNSRequest) error
 
 	// PeerRelayGrants, if true, inserts relay capabilities into the wildcard
 	// grants rules.
@@ -508,6 +509,8 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
 		s.serveMap(w, r, mkey)
 	case "/machine/register":
 		s.serveRegister(w, r, mkey)
+	case "/machine/set-dns":
+		s.serveSetDNS(w, r, mkey)
 	case "/machine/update-health":
 		io.Copy(io.Discard, r.Body)
 		w.WriteHeader(http.StatusNoContent)
@@ -516,6 +519,66 @@ func (s *Server) serveMachine(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func (s *Server) serveSetDNS(w http.ResponseWriter, r *http.Request, mkey key.MachinePublic) {
+	var req tailcfg.SetDNSRequest
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if req.Type != "TXT" {
+		http.Error(w, "only TXT records are supported", http.StatusBadRequest)
+		return
+	}
+	if req.Name == "" || req.Value == "" {
+		http.Error(w, "missing name or value", http.StatusBadRequest)
+		return
+	}
+	if req.NodeKey.IsZero() {
+		http.Error(w, "missing node key", http.StatusBadRequest)
+		return
+	}
+
+	s.mu.Lock()
+	node := s.nodes[req.NodeKey]
+	certDomains := s.certDomainsLocked(node)
+	s.mu.Unlock()
+	if node == nil {
+		http.Error(w, "unknown node key", http.StatusForbidden)
+		return
+	}
+	if node.Machine != mkey {
+		http.Error(w, "node key does not belong to machine", http.StatusForbidden)
+		return
+	}
+	baseName, ok := strings.CutPrefix(req.Name, "_acme-challenge.")
+	if !ok || !slices.Contains(certDomains, baseName) {
+		http.Error(w, "name is not an ACME challenge for a cert domain", http.StatusForbidden)
+		return
+	}
+	if s.OnSetDNS != nil {
+		if err := s.OnSetDNS(&req); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			return
+		}
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(tailcfg.SetDNSResponse{})
+}
+
+func (s *Server) certDomainsLocked(node *tailcfg.Node) []string {
+	if node == nil {
+		return nil
+	}
+	var ret []string
+	if s.DNSConfig != nil {
+		ret = append(ret, s.DNSConfig.CertDomains...)
+	}
+	if s.MagicDNSDomain != "" {
+		ret = append(ret, node.Hostinfo.Hostname()+"."+s.MagicDNSDomain)
+	}
+	return ret
+}
+
 // SetSubnetRoutes sets the list of subnet routes which a node is routing.
 func (s *Server) SetSubnetRoutes(nodeKey key.NodePublic, routes []netip.Prefix) {
 	s.mu.Lock()

tstest/natlab/vmtest/acme_test.go

@@ -0,0 +1,86 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/creachadair/mds/shell"
+	"tailscale.com/tstest/natlab/vmtest"
+	"tailscale.com/tstest/natlab/vnet"
+)
+
+func TestACMECertServeHTTPS(t *testing.T) {
+	env := vmtest.New(t, vmtest.FakeACME())
+	issuer := ubuntuHard(env, "issuer")
+	client := ubuntuHard(env, "client")
+	env.Start()
+
+	st := env.Status(issuer)
+	if len(st.CertDomains) == 0 {
+		t.Fatalf("issuer has no CertDomains in status")
+	}
+	domain := st.CertDomains[0]
+
+	out, err := env.SSHExec(issuer, certAndWatchHealthCommand(domain))
+	if err != nil {
+		t.Fatalf("tailscale cert: %v\n%s", err, out)
+	}
+
+	out, err = env.SSHExec(issuer, "tailscale serve --bg --https=443 text:natlab-acme-ok")
+	if err != nil {
+		t.Fatalf("tailscale serve: %v\n%s", err, out)
+	}
+
+	rootB64 := base64.StdEncoding.EncodeToString(env.FakeACMERootPEM())
+	out, err = env.SSHExec(client, "printf %s "+shell.Quote(rootB64)+" | base64 -d >/tmp/fake-acme-root.pem")
+	if err != nil {
+		t.Fatalf("install fake ACME root: %v\n%s", err, out)
+	}
+	out, err = env.SSHExec(client, "curl --fail --silent --show-error --cacert /tmp/fake-acme-root.pem https://"+shell.Quote(domain)+"/")
+	if err != nil {
+		t.Fatalf("curl served HTTPS page: %v\n%s", err, out)
+	}
+	if strings.TrimSpace(out) != "natlab-acme-ok" {
+		t.Fatalf("curl body = %q, want %q", out, "natlab-acme-ok")
+	}
+}
+
+func ubuntuHard(env *vmtest.Env, name string) *vmtest.Node {
+	n := env.NumNodes()
+	return env.AddNode(name,
+		env.AddNetwork(
+			fmt.Sprintf("2.%d.%d.%d", n, n, n),
+			fmt.Sprintf("10.0.%d.1/24", n), vnet.HardNAT),
+		vmtest.OS(vmtest.Ubuntu2404))
+}
+
+func certAndWatchHealthCommand(domain string) string {
+	qdomain := shell.Quote(domain)
+	return fmt.Sprintf(`
+set -eu
+cd /tmp
+rm -f cert.out cert.status cert.done cert-health.out
+(set +e; tailscale cert %[1]s >cert.out 2>&1; echo $? >cert.status; touch cert.done) &
+certpid=$!
+for i in $(seq 1 60); do
+	if tailscale status --json | grep -F "Fetching TLS certificate" >cert-health.out; then
+		break
+	fi
+	if [ -e cert.done ]; then
+		break
+	fi
+	sleep 0.1
+done
+wait "$certpid" || true
+cat cert.out
+if [ "$(cat cert.status)" != "0" ]; then
+	exit "$(cat cert.status)"
+fi
+test -s cert-health.out
+`, qdomain)
+}

tstest/natlab/vmtest/cloudinit.go

@@ -4,12 +4,18 @@
 package vmtest
 
 import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"encoding/pem"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strings"
 
+	"github.com/creachadair/mds/shell"
 	"github.com/kdomanski/iso9660"
+	"golang.org/x/crypto/ssh"
 )
 
 // createCloudInitISO creates a cidata seed ISO for the given cloud VM node.
@@ -18,6 +24,9 @@ import (
 // doesn't use netplan-style network-config; DHCP is enabled in rc.conf).
 func (e *Env) createCloudInitISO(n *Node) (string, error) {
 	metaData := fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", n.name, n.name)
+	if err := ensureDebugSSHKey(); err != nil {
+		return "", err
+	}
 	userData := e.generateUserData(n)
 
 	files := map[string]string{
@@ -124,7 +133,7 @@ func (e *Env) generateLinuxUserData(n *Node) string {
 	// features like Taildrop (which needs a place to stash incoming files)
 	// have a directory to work with.
 	ud.WriteString("  - [\"mkdir\", \"-p\", \"/var/lib/tailscale\"]\n")
-	ud.WriteString("  - [\"/bin/sh\", \"-c\", \"/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale &\"]\n")
+	fmt.Fprintf(&ud, "  - [\"/bin/sh\", \"-c\", \"%s/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale &\"]\n", tailscaledEnvPrefix(n))
 	ud.WriteString("  - [\"sleep\", \"2\"]\n")
 
 	// Start tta (Tailscale Test Agent).
@@ -183,7 +192,7 @@ func (e *Env) generateFreeBSDUserData(n *Node) string {
 	// path). --statedir provides a VarRoot so features like Taildrop have a
 	// directory.
 	ud.WriteString("  - \"mkdir -p /var/lib/tailscale\"\n")
-	ud.WriteString("  - \"export PATH=/usr/local/bin:$PATH && /usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale </dev/null >/var/log/tailscaled.log 2>&1 &\"\n")
+	fmt.Fprintf(&ud, "  - \"export PATH=/usr/local/bin:$PATH && %s/usr/local/bin/tailscaled --state=mem: --statedir=/var/lib/tailscale </dev/null >/var/log/tailscaled.log 2>&1 &\"\n", tailscaledEnvPrefix(n))
 	ud.WriteString("  - \"sleep 2\"\n")
 
 	// Start tta (Tailscale Test Agent), with the same stdio redirection.
@@ -191,3 +200,47 @@ func (e *Env) generateFreeBSDUserData(n *Node) string {
 
 	return ud.String()
 }
+
+func tailscaledEnvPrefix(n *Node) string {
+	env := n.vnetNode.Env()
+	if len(env) == 0 {
+		return ""
+	}
+	parts := make([]string, 0, len(env))
+	for _, e := range env {
+		parts = append(parts, e.Key+"="+shell.Quote(e.Value))
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, " ") + " "
+}
+
+func ensureDebugSSHKey() error {
+	const keyPath = "/tmp/vmtest_key"
+	if privPEM, err := os.ReadFile(keyPath); err == nil {
+		if _, err := os.Stat(keyPath + ".pub"); err == nil {
+			return nil
+		}
+		signer, err := ssh.ParsePrivateKey(privPEM)
+		if err != nil {
+			return fmt.Errorf("parse %s: %w", keyPath, err)
+		}
+		return os.WriteFile(keyPath+".pub", ssh.MarshalAuthorizedKey(signer.PublicKey()), 0644)
+	}
+
+	pub, priv, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		return err
+	}
+	block, err := ssh.MarshalPrivateKey(priv, "vmtest")
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(keyPath, pem.EncodeToMemory(block), 0600); err != nil {
+		return err
+	}
+	sshPub, err := ssh.NewPublicKey(pub)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(keyPath+".pub", ssh.MarshalAuthorizedKey(sshPub), 0644)
+}

tstest/natlab/vmtest/vmtest.go

@@ -93,6 +93,7 @@ type Env struct {
 	allOnline                 bool // mark every peer as Online=true in MapResponses
 	peerRelayGrants           bool // grant peer-relay capabilities on the wildcard packet filter
 	selfSignedDERPCertPinning bool // serve test DERP map with sha256-raw cert pins
+	fakeACME                  bool // point tailscaled at vnet's fake ACME server
 
 	// Shared resource initialization (sync.Once for things multiple nodes share).
 	vnetOnce      sync.Once
@@ -394,6 +395,12 @@ func SelfSignedDERPCertPinning() EnvOption {
 	return envOptFunc(func(e *Env) { e.selfSignedDERPCertPinning = true })
 }
 
+// FakeACME returns an [EnvOption] that points nodes at vnet's in-process ACME
+// CA and configures the test control server to advertise MagicDNS cert domains.
+func FakeACME() EnvOption {
+	return envOptFunc(func(e *Env) { e.fakeACME = true })
+}
+
 // AddNetwork creates a new virtual network. Arguments follow the same pattern as
 // vnet.Config.AddNetwork (string IPs, NAT types, NetworkService values).
 func (e *Env) AddNetwork(opts ...any) *vnet.Network {
@@ -463,6 +470,12 @@ func (e *Env) AddNode(name string, opts ...any) *Node {
 			vnetOpts = append(vnetOpts, o)
 		}
 	}
+	if e.fakeACME {
+		vnetOpts = append(vnetOpts, vnet.TailscaledEnv{
+			Key:   "TS_DEBUG_ACME_DIRECTORY_URL",
+			Value: "http://acme.example/directory",
+		})
+	}
 
 	// macOS VMs require a macOS arm64 host (Apple Virtualization.framework via
 	// tailmac). Skip the test now rather than letting it proceed through the
@@ -810,6 +823,12 @@ func (e *Env) ControlServer() *testcontrol.Server {
 	return e.server.ControlServer()
 }
 
+// FakeACMERootPEM returns the root certificate for vnet's fake ACME CA.
+func (e *Env) FakeACMERootPEM() []byte {
+	e.initVnet()
+	return e.server.FakeACMERootPEM()
+}
+
 // BringUpMullvadWGServer brings up a userspace WireGuard server on n,
 // configured as a single-peer "Mullvad-style" exit-node target. The
 // server runs inside n's TTA process on a Linux TUN named "wg0".
@@ -1229,6 +1248,7 @@ func (e *Env) SSHExec(n *Node, cmd string) (string, error) {
 		"-o", "StrictHostKeyChecking=no",
 		"-o", "UserKnownHostsFile=/dev/null",
 		"-o", "ConnectTimeout=5",
+		"-o", "LogLevel=ERROR",
 		"-i", "/tmp/vmtest_key",
 		"-p", fmt.Sprintf("%d", n.sshPort),
 		"root@127.0.0.1",
@@ -1445,6 +1465,14 @@ func (e *Env) initVnet() {
 		if e.selfSignedDERPCertPinning {
 			e.server.ControlServer().DERPMap = e.buildSelfSignedDERPMap()
 		}
+		if e.fakeACME {
+			cs := e.server.ControlServer()
+			cs.MagicDNSDomain = "tailnet.test"
+			if cs.DNSConfig == nil {
+				cs.DNSConfig = new(tailcfg.DNSConfig)
+			}
+			cs.DNSConfig.Proxied = true
+		}
 	})
 }
 

tstest/natlab/vnet/fake_acme.go

@@ -0,0 +1,528 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vnet
+
+// This file implements a minimal fake ACME (RFC 8555) certificate authority
+// used by TestACMECertServeHTTPS in tstest/natlab/vmtest. It exists so that
+// natlab VM tests can exercise `tailscale cert` and `tailscale serve` end to
+// end without reaching out to Let's Encrypt.
+//
+// Only the parts of ACME exercised by that test are implemented: the dns-01
+// challenge flow, a single hard-coded account, no JWS signature verification,
+// no key rollover, no nonce tracking, no rate limiting. The TLS root is
+// freshly generated per server.
+//
+// If a future test needs more of the protocol, prefer switching to
+// https://github.com/letsencrypt/pebble (the official Let's Encrypt test
+// ACME server) rather than fleshing this file out further. The threshold for
+// "more complicated" should be low.
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	crand "crypto/rand"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"fmt"
+	"io"
+	"math/big"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"tailscale.com/util/httpm"
+)
+
+// fakeACMEServer is an in-process fake ACME (RFC 8555) CA used by natlab
+// VM tests. See the file-level comment for scope and limitations.
+//
+// The zero value is not usable; construct via [newFakeACMEServer].
+type fakeACMEServer struct {
+	// baseURL is the externally visible URL prefix at which the server is
+	// reachable (no trailing slash). All issued URLs are formed by appending
+	// to baseURL.
+	baseURL string
+
+	mu      sync.Mutex
+	rootKey *ecdsa.PrivateKey // CA signing key
+	rootDER []byte            // CA cert, DER-encoded
+	nextID  int64             // monotonically increasing ID source for orders/authzs/challenges/certs
+	// lookupTXT, if non-nil, returns the TXT records visible to the CA for
+	// dns-01 challenge validation. It is set by the surrounding [Server]
+	// after construction.
+	lookupTXT func(string) []string
+	orders    map[string]*fakeACMEOrder // order ID → order
+	authzs    map[string]*fakeACMEAuthz // authz ID → authz
+	certsPEM  map[string][]byte         // cert ID → issued cert chain PEM
+}
+
+// fakeACMEOrder is the in-memory state for one ACME order.
+//
+// Status is one of "pending", "ready", or "valid", matching RFC 8555 §7.1.6.
+// The terminal "invalid" state is not modeled.
+type fakeACMEOrder struct {
+	id          string
+	status      string
+	identifiers []fakeACMEIdentifier
+	authzURLs   []string
+	finalizeURL string
+	certURL     string // empty until status == "valid"
+}
+
+// fakeACMEAuthz is the in-memory state for one ACME authorization. Each
+// authorization carries a single dns-01 challenge; other challenge types
+// are not modeled.
+type fakeACMEAuthz struct {
+	id         string
+	status     string // "pending" or "valid"
+	identifier fakeACMEIdentifier
+	challenge  fakeACMEChallenge
+}
+
+// fakeACMEIdentifier identifies a domain to be authorized.
+// It is serialized as the ACME "identifier" JSON object.
+type fakeACMEIdentifier struct {
+	Type  string `json:"type"`  // always "dns" in practice
+	Value string `json:"value"` // domain name, possibly with a "*." wildcard prefix
+}
+
+// fakeACMEChallenge is the JSON shape of a single ACME challenge,
+// per RFC 8555 §8.
+type fakeACMEChallenge struct {
+	URL    string `json:"url"`
+	Type   string `json:"type"`  // always "dns-01"
+	Token  string `json:"token"` // not used to derive a real key authorization; just echoed back
+	Status string `json:"status"`
+}
+
+// newFakeACMEServer returns a new fake ACME server that will advertise
+// itself at baseURL. A fresh ECDSA P-256 CA key and a self-signed root
+// certificate are generated. The caller is responsible for actually serving
+// HTTP at baseURL and routing requests to [fakeACMEServer.ServeHTTP].
+func newFakeACMEServer(baseURL string) *fakeACMEServer {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader)
+	if err != nil {
+		panic(fmt.Sprintf("vnet: generating fake ACME root key: %v", err))
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber:          big.NewInt(1),
+		Subject:               pkix.Name{CommonName: "natlab fake ACME root"},
+		NotBefore:             time.Now().Add(-time.Hour),
+		NotAfter:              time.Now().Add(365 * 24 * time.Hour),
+		KeyUsage:              x509.KeyUsageCertSign | x509.KeyUsageCRLSign,
+		BasicConstraintsValid: true,
+		IsCA:                  true,
+	}
+	der, err := x509.CreateCertificate(crand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		panic(fmt.Sprintf("vnet: creating fake ACME root: %v", err))
+	}
+	return &fakeACMEServer{
+		baseURL:  strings.TrimRight(baseURL, "/"),
+		rootKey:  key,
+		rootDER:  der,
+		nextID:   1,
+		orders:   map[string]*fakeACMEOrder{},
+		authzs:   map[string]*fakeACMEAuthz{},
+		certsPEM: map[string][]byte{},
+	}
+}
+
+// directoryURL returns the ACME directory URL for s, suitable for setting
+// TS_DEBUG_ACME_DIRECTORY_URL in a tailscaled under test.
+func (s *fakeACMEServer) directoryURL() string {
+	return s.baseURL + "/directory"
+}
+
+// rootPEM returns the PEM-encoded root certificate that signs all certs
+// issued by s. Clients that want to verify those certs must trust it.
+func (s *fakeACMEServer) rootPEM() []byte {
+	return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: s.rootDER})
+}
+
+// ServeHTTP routes ACME protocol requests to the appropriate handler.
+// It is intended to be installed as the [http.Handler] for s.baseURL.
+func (s *fakeACMEServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Replay-Nonce", fmt.Sprintf("nonce-%d", time.Now().UnixNano()))
+	switch {
+	case r.Method == httpm.GET && r.URL.Path == "/directory":
+		writeACMEJSON(w, http.StatusOK, struct {
+			NewNonce   string `json:"newNonce"`
+			NewAccount string `json:"newAccount"`
+			NewOrder   string `json:"newOrder"`
+			RevokeCert string `json:"revokeCert"`
+		}{
+			NewNonce:   s.baseURL + "/new-nonce",
+			NewAccount: s.baseURL + "/new-account",
+			NewOrder:   s.baseURL + "/new-order",
+			RevokeCert: s.baseURL + "/revoke-cert",
+		})
+	case (r.Method == httpm.HEAD || r.Method == httpm.GET) && r.URL.Path == "/new-nonce":
+		w.WriteHeader(http.StatusOK)
+	case r.Method == httpm.POST && r.URL.Path == "/new-account":
+		s.serveNewAccount(w, r)
+	case r.Method == httpm.POST && r.URL.Path == "/new-order":
+		s.serveNewOrder(w, r)
+	case r.Method == httpm.POST && strings.HasPrefix(r.URL.Path, "/authz/"):
+		s.serveAuthz(w, r)
+	case r.Method == httpm.POST && strings.HasPrefix(r.URL.Path, "/challenge/"):
+		s.serveChallenge(w, r)
+	case r.Method == httpm.POST && strings.HasPrefix(r.URL.Path, "/order/") && strings.HasSuffix(r.URL.Path, "/finalize"):
+		s.serveFinalize(w, r)
+	case r.Method == httpm.POST && strings.HasPrefix(r.URL.Path, "/order/"):
+		s.serveOrder(w, r)
+	case r.Method == httpm.POST && strings.HasPrefix(r.URL.Path, "/cert/"):
+		s.serveCert(w, r)
+	default:
+		http.NotFound(w, r)
+	}
+}
+
+// serveNewAccount handles the ACME newAccount endpoint.
+//
+// The fake only supports a single account: every newAccount request returns
+// the same /account/1 URL, and no per-account state is tracked. Tests that
+// need multiple distinct accounts will need to extend this.
+func (s *fakeACMEServer) serveNewAccount(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		OnlyReturnExisting bool `json:"onlyReturnExisting"`
+	}
+	if err := decodeJWSPayload(r, &req); err != nil {
+		io.Copy(io.Discard, r.Body)
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if req.OnlyReturnExisting {
+		writeACMEProblemType(w, http.StatusBadRequest, "accountDoesNotExist", "account does not exist")
+		return
+	}
+	w.Header().Set("Location", s.baseURL+"/account/1")
+	writeACMEJSON(w, http.StatusCreated, struct {
+		Status string `json:"status"`
+	}{Status: "valid"})
+}
+
+// serveNewOrder handles the ACME newOrder endpoint. It allocates an order
+// and one authorization (with a single dns-01 challenge) per identifier in
+// the request, all in "pending" status.
+func (s *fakeACMEServer) serveNewOrder(w http.ResponseWriter, r *http.Request) {
+	var req struct {
+		Identifiers []fakeACMEIdentifier `json:"identifiers"`
+	}
+	if err := decodeJWSPayload(r, &req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	orderID := s.allocIDLocked()
+	orderURL := s.baseURL + "/order/" + orderID
+	o := &fakeACMEOrder{
+		id:          orderID,
+		status:      "pending",
+		identifiers: req.Identifiers,
+		finalizeURL: orderURL + "/finalize",
+	}
+	for _, ident := range req.Identifiers {
+		authzID := s.allocIDLocked()
+		chalID := s.allocIDLocked()
+		chal := fakeACMEChallenge{
+			URL:    s.baseURL + "/challenge/" + chalID,
+			Type:   "dns-01",
+			Token:  "token-" + chalID,
+			Status: "pending",
+		}
+		az := &fakeACMEAuthz{
+			id:         authzID,
+			status:     "pending",
+			identifier: ident,
+			challenge:  chal,
+		}
+		authzURL := s.baseURL + "/authz/" + authzID
+		s.authzs[authzID] = az
+		o.authzURLs = append(o.authzURLs, authzURL)
+	}
+	s.orders[orderID] = o
+	w.Header().Set("Location", orderURL)
+	writeACMEJSON(w, http.StatusCreated, s.orderResponseLocked(o))
+}
+
+// serveAuthz handles GET-via-POST of an authorization object.
+func (s *fakeACMEServer) serveAuthz(w http.ResponseWriter, r *http.Request) {
+	id := strings.TrimPrefix(r.URL.Path, "/authz/")
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	az := s.authzs[id]
+	if az == nil {
+		http.NotFound(w, r)
+		return
+	}
+	writeACMEJSON(w, http.StatusOK, s.authzResponseLocked(az))
+}
+
+// serveChallenge handles the client's "I'm ready, please validate" POST to
+// a challenge URL. It looks up the expected TXT record via s.lookupTXT and,
+// if any record is present, marks both the challenge and its enclosing
+// authorization as valid (and re-evaluates any pending orders).
+//
+// The TXT record contents are not validated against the JWK thumbprint; any
+// non-empty record satisfies the challenge. That is intentional for these
+// tests but is not how a real ACME server behaves.
+func (s *fakeACMEServer) serveChallenge(w http.ResponseWriter, r *http.Request) {
+	// Keep issuance pending long enough for `tailscale cert` to print the
+	// cert-pending health warning it is watching for.
+	time.Sleep(3 * time.Second)
+
+	id := strings.TrimPrefix(r.URL.Path, "/challenge/")
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	var az *fakeACMEAuthz
+	for _, a := range s.authzs {
+		if strings.TrimPrefix(a.challenge.URL, s.baseURL+"/challenge/") == id {
+			az = a
+			break
+		}
+	}
+	if az == nil {
+		http.NotFound(w, r)
+		return
+	}
+	name := "_acme-challenge." + strings.TrimPrefix(az.identifier.Value, "*.")
+	if s.lookupTXT == nil || len(s.lookupTXT(name)) == 0 {
+		writeACMEProblem(w, http.StatusForbidden, "dns TXT record not found")
+		return
+	}
+	az.status = "valid"
+	az.challenge.Status = "valid"
+	s.updateOrdersLocked()
+	writeACMEJSON(w, http.StatusOK, az.challenge)
+}
+
+// serveOrder handles GET-via-POST of an order object.
+func (s *fakeACMEServer) serveOrder(w http.ResponseWriter, r *http.Request) {
+	id := strings.TrimPrefix(r.URL.Path, "/order/")
+	id = strings.TrimSuffix(id, "/finalize")
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	o := s.orders[id]
+	if o == nil {
+		http.NotFound(w, r)
+		return
+	}
+	w.Header().Set("Location", s.baseURL+"/order/"+id)
+	writeACMEJSON(w, http.StatusOK, s.orderResponseLocked(o))
+}
+
+// serveFinalize handles a POST to /order/<id>/finalize. It parses the
+// supplied CSR, issues a leaf certificate signed by the fake root, and
+// transitions the order to "valid".
+func (s *fakeACMEServer) serveFinalize(w http.ResponseWriter, r *http.Request) {
+	id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/order/"), "/finalize")
+	var req struct {
+		CSR string `json:"csr"`
+	}
+	if err := decodeJWSPayload(r, &req); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	csrDER, err := base64.RawURLEncoding.DecodeString(req.CSR)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	csr, err := x509.ParseCertificateRequest(csrDER)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+	if err := csr.CheckSignature(); err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	o := s.orders[id]
+	if o == nil {
+		http.NotFound(w, r)
+		return
+	}
+	if o.status != "ready" && o.status != "valid" {
+		writeACMEProblem(w, http.StatusForbidden, "order is not ready")
+		return
+	}
+	certPEM, err := s.issueCertLocked(csr)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	certID := s.allocIDLocked()
+	o.status = "valid"
+	o.certURL = s.baseURL + "/cert/" + certID
+	s.certsPEM[certID] = certPEM
+	w.Header().Set("Location", s.baseURL+"/order/"+id)
+	writeACMEJSON(w, http.StatusOK, s.orderResponseLocked(o))
+}
+
+// serveCert returns the PEM-encoded issued certificate chain (leaf + root)
+// for a previously finalized order.
+func (s *fakeACMEServer) serveCert(w http.ResponseWriter, r *http.Request) {
+	id := strings.TrimPrefix(r.URL.Path, "/cert/")
+	s.mu.Lock()
+	cert := append([]byte(nil), s.certsPEM[id]...)
+	s.mu.Unlock()
+	if len(cert) == 0 {
+		http.NotFound(w, r)
+		return
+	}
+	w.Header().Set("Content-Type", "application/pem-certificate-chain")
+	w.WriteHeader(http.StatusOK)
+	w.Write(cert)
+}
+
+// allocIDLocked returns a fresh decimal ID. s.mu must be held.
+func (s *fakeACMEServer) allocIDLocked() string {
+	id := s.nextID
+	s.nextID++
+	return fmt.Sprint(id)
+}
+
+// updateOrdersLocked promotes any pending order whose authorizations are
+// all valid to the "ready" state. s.mu must be held.
+func (s *fakeACMEServer) updateOrdersLocked() {
+	for _, o := range s.orders {
+		if o.status != "pending" {
+			continue
+		}
+		ready := true
+		for _, u := range o.authzURLs {
+			id := strings.TrimPrefix(u, s.baseURL+"/authz/")
+			if s.authzs[id].status != "valid" {
+				ready = false
+			}
+		}
+		if ready {
+			o.status = "ready"
+		}
+	}
+}
+
+// orderResponseLocked returns the JSON-shaped view of o that ACME clients
+// expect. s.mu must be held.
+func (s *fakeACMEServer) orderResponseLocked(o *fakeACMEOrder) any {
+	return struct {
+		Status         string               `json:"status"`
+		Identifiers    []fakeACMEIdentifier `json:"identifiers"`
+		Authorizations []string             `json:"authorizations"`
+		Finalize       string               `json:"finalize"`
+		Certificate    string               `json:"certificate"`
+	}{
+		Status:         o.status,
+		Identifiers:    o.identifiers,
+		Authorizations: o.authzURLs,
+		Finalize:       o.finalizeURL,
+		Certificate:    o.certURL,
+	}
+}
+
+// authzResponseLocked returns the JSON-shaped view of az that ACME clients
+// expect. s.mu must be held.
+func (s *fakeACMEServer) authzResponseLocked(az *fakeACMEAuthz) any {
+	return struct {
+		Status     string              `json:"status"`
+		Identifier fakeACMEIdentifier  `json:"identifier"`
+		Challenges []fakeACMEChallenge `json:"challenges"`
+	}{
+		Status:     az.status,
+		Identifier: az.identifier,
+		Challenges: []fakeACMEChallenge{az.challenge},
+	}
+}
+
+// issueCertLocked signs a 24-hour leaf cert for csr using s's root and
+// returns the leaf-then-root PEM chain. s.mu must be held.
+func (s *fakeACMEServer) issueCertLocked(csr *x509.CertificateRequest) ([]byte, error) {
+	serial := big.NewInt(time.Now().UnixNano())
+	tmpl := &x509.Certificate{
+		SerialNumber: serial,
+		Subject:      csr.Subject,
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(24 * time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		DNSNames:     csr.DNSNames,
+	}
+	root, err := x509.ParseCertificate(s.rootDER)
+	if err != nil {
+		return nil, err
+	}
+	der, err := x509.CreateCertificate(crand.Reader, tmpl, root, csr.PublicKey, s.rootKey)
+	if err != nil {
+		return nil, err
+	}
+	var b []byte
+	b = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})
+	b = append(b, s.rootPEM()...)
+	return b, nil
+}
+
+// decodeJWSPayload extracts the inner JSON payload from an ACME JWS-wrapped
+// request body and unmarshals it into v. The JWS signature, protected
+// header, and key are not inspected: this is a fake server, and we trust
+// whatever the client sends.
+//
+// ACME (RFC 8555) JWS payloads use unpadded base64url encoding, so we
+// decode Payload as a string with [base64.RawURLEncoding] rather than as a
+// []byte field (which encoding/json would decode with [base64.StdEncoding]).
+func decodeJWSPayload(r *http.Request, v any) error {
+	var jws struct {
+		Payload string `json:"payload"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&jws); err != nil {
+		return err
+	}
+	if jws.Payload == "" {
+		return nil
+	}
+	payload, err := base64.RawURLEncoding.DecodeString(jws.Payload)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(payload, v)
+}
+
+// writeACMEJSON serializes v as JSON and writes it as an ACME response with
+// the given status code.
+func writeACMEJSON(w http.ResponseWriter, code int, v any) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(code)
+	json.NewEncoder(w).Encode(v)
+}
+
+// writeACMEProblem writes an ACME "rejectedIdentifier" problem document.
+// This is the catch-all error used when no more specific ACME error type
+// fits.
+func writeACMEProblem(w http.ResponseWriter, code int, detail string) {
+	writeACMEProblemType(w, code, "rejectedIdentifier", detail)
+}
+
+// writeACMEProblemType writes an ACME problem document with the given
+// problem type (the trailing component of urn:ietf:params:acme:error:*)
+// and detail message.
+func writeACMEProblemType(w http.ResponseWriter, code int, problemType, detail string) {
+	writeACMEJSON(w, code, struct {
+		Status int    `json:"status"`
+		Type   string `json:"type"`
+		Detail string `json:"detail"`
+	}{
+		Status: code,
+		Type:   "urn:ietf:params:acme:error:" + problemType,
+		Detail: detail,
+	})
+}

tstest/natlab/vnet/vip.go

@@ -21,6 +21,7 @@ var (
 	fakeSyslog            = newVIP("syslog.tailscale", 9)
 	fakeCloudInit         = newVIP("cloud-init.tailscale", 5) // serves cloud-init metadata/userdata per node
 	fakeFiles             = newVIP("files.tailscale", 6)      // serves binary files (tta, tailscale, tailscaled) to VMs
+	fakeACME              = newVIP("acme.example", 7)         // fake ACME CA for vmtests
 
 	// FakeDualStackWeb is a dual-stack webserver VIP used by
 	// TestExitNodeV4Only to verify that traffic works through an

tstest/natlab/vnet/vnet.go

@@ -445,6 +445,17 @@ func (n *network) acceptTCP(r *tcp.ForwarderRequest) {
 		return
 	}
 
+	if destPort == 80 && fakeACME.Match(destIP) && n.s.fakeACME != nil {
+		r.Complete(false)
+		tc := gonet.NewTCPConn(&wq, ep)
+		context.AfterFunc(n.s.shutdownCtx, func() { tc.SetDeadline(time.Now()) })
+		hs := &http.Server{Handler: n.s.fakeACME}
+		n.s.wg.Go(func() {
+			hs.Serve(netutil.NewOneConnListener(tc, nil))
+		})
+		return
+	}
+
 	var targetDial string
 	if n.s.derpIPs.Contains(destIP) {
 		targetDial = destIP.String() + ":" + strconv.Itoa(int(destPort))
@@ -831,6 +842,7 @@ type Server struct {
 
 	control    *testcontrol.Server
 	derps      []*derpServer
+	fakeACME   *fakeACMEServer
 	pcapWriter *pcapWriter
 
 	// writeMu serializes all writes to VM clients.
@@ -845,6 +857,7 @@ type Server struct {
 
 	cloudInitData map[int]*CloudInitData // node num → cloud-init config
 	fileContents  map[string][]byte      // filename → file bytes
+	dnsTXTRecords map[string][]string
 
 	// onDHCPEvent, if non-nil, is called when DHCP messages are processed.
 	// Parameters are: source MAC, node number, DHCP message type, assigned IP.
@@ -922,14 +935,21 @@ func New(c *Config) (*Server, error) {
 			DERPMap:         derpMap,
 			ExplicitBaseURL: "http://control.tailscale",
 		},
+		fakeACME: newFakeACMEServer("http://acme.example"),
 
 		blendReality: c.blendReality,
 		derpIPs:      set.Of[netip.Addr](),
 
-		nodeByMAC:    map[MAC]*node{},
+		nodeByMAC:     map[MAC]*node{},
-		networkByWAN: &bart.Table[*network]{},
+		networkByWAN:  &bart.Table[*network]{},
-		networks:     set.Of[*network](),
+		networks:      set.Of[*network](),
+		dnsTXTRecords: map[string][]string{},
 	}
+	s.control.OnSetDNS = func(req *tailcfg.SetDNSRequest) error {
+		s.setDNSRecord(req.Name, req.Value)
+		return nil
+	}
+	s.fakeACME.lookupTXT = s.lookupTXT
 	for _, host := range derpHostnames {
 		s.derps = append(s.derps, newDERPServer(host))
 	}
@@ -968,6 +988,28 @@ func (s *Server) DERPCertSHA256Hex(idx int) string {
 	return s.derps[idx].certSHA256Hex
 }
 
+// FakeACMEDirectoryURL returns the directory URL for vnet's in-process ACME CA.
+func (s *Server) FakeACMEDirectoryURL() string {
+	return s.fakeACME.directoryURL()
+}
+
+// FakeACMERootPEM returns the PEM-encoded root certificate for vnet's fake ACME CA.
+func (s *Server) FakeACMERootPEM() []byte {
+	return s.fakeACME.rootPEM()
+}
+
+func (s *Server) setDNSRecord(name, value string) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.dnsTXTRecords[name] = append(s.dnsTXTRecords[name], value)
+}
+
+func (s *Server) lookupTXT(name string) []string {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return append([]string(nil), s.dnsTXTRecords[name]...)
+}
+
 // CloudInitData holds the cloud-init configuration for a node.
 type CloudInitData struct {
 	MetaData      string
@@ -2270,7 +2312,7 @@ func (s *Server) shouldInterceptTCP(pkt gopacket.Packet) bool {
 	}
 
 	if tcp.DstPort == 80 || tcp.DstPort == 443 {
-		for _, v := range []virtualIP{fakeControl, fakeDERP1, fakeDERP2, fakeLogCatcher, fakeCloudInit, fakeFiles} {
+		for _, v := range []virtualIP{fakeControl, fakeDERP1, fakeDERP2, fakeLogCatcher, fakeCloudInit, fakeFiles, fakeACME} {
 			if v.Match(flow.dst) {
 				return true
 			}
@@ -2399,6 +2441,17 @@ func (s *Server) createDNSResponse(pkt gopacket.Packet) ([]byte, error) {
 					TTL:   60,
 				})
 			}
+		} else if q.Type == layers.DNSTypeTXT {
+			for _, txt := range s.lookupTXT(string(q.Name)) {
+				response.ANCount++
+				response.Answers = append(response.Answers, layers.DNSResourceRecord{
+					Name:  q.Name,
+					Type:  q.Type,
+					Class: q.Class,
+					TXTs:  [][]byte{[]byte(txt)},
+					TTL:   60,
+				})
+			}
 		}
 	}