ipn/localapi,tstest/natlab: fix debug derp TLS check for sha256-raw CertName

serveDebugDERPRegion built its TLS config with
ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), which for a
"sha256-raw:<hex>" CertName passed the raw fingerprint to Go's stock
verifier as a hostname; the handshake always failed with a hostname
mismatch. This is the second half of #15579; the first half (tailscaled
itself failing with "unexpected multiple certs presented") was fixed in

Extract a tlsConfigForNode helper that mirrors derphttp.Client.tlsClient
so that sha256-raw and domain-fronting CertName values are dispatched
to tlsdial.SetConfigExpectedCertHash and tlsdial.SetConfigExpectedCert
respectively, falling back to HostName when CertName is empty.

The core fix here was originally written by @imnuke in #19965; that PR
also added a unit test in ipn/localapi/debugderp_test.go which is
replaced in this commit by a new vmtest that exercises the whole stack:
vnet now serves a self-signed cert valid for each fake DERP node's
HostName and exposes its SHA-256 fingerprint, and vmtest grows a new
SelfSignedDERPCertPinning EnvOption that swaps the test DERP map's
nodes to CertName="sha256-raw:<hex>" with InsecureForTests cleared.
TestSelfSignedDERPHashPinning then stands up two hard-NAT'd nodes, has
them communicate over DERP, and calls DebugDERPRegion on each. Before
this fix the test fails with the exact x509 hostname-mismatch error
from the original bug; after, it passes.

Updates #15579

Change-Id: I61f38ffebc7ac5abc962639db1ae88f5cd8633b1
Co-authored-by: Nuke <nuke@imnuke.dev>
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

ipn/localapi/debugderp.go

@@ -15,6 +15,7 @@ import (
 	"net/http"
 	"net/netip"
 	"strconv"
+	"strings"
 	"time"
 
 	"tailscale.com/derp/derphttp"
@@ -22,11 +23,28 @@ import (
 	"tailscale.com/net/netaddr"
 	"tailscale.com/net/netns"
 	"tailscale.com/net/stun"
+	"tailscale.com/net/tlsdial"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 	"tailscale.com/types/nettype"
 )
 
+// tlsConfigForNode builds a *tls.Config for connecting to a DERP node,
+// mirroring the logic in derphttp.Client.tlsClient so that sha256-raw cert
+// pinning and domain-fronting CertName values are handled correctly.
+func tlsConfigForNode(node *tailcfg.DERPNode) *tls.Config {
+	conf := tlsdial.Config(nil, nil)
+	conf.ServerName = node.HostName
+	if node.CertName != "" {
+		if suf, ok := strings.CutPrefix(node.CertName, "sha256-raw:"); ok {
+			tlsdial.SetConfigExpectedCertHash(conf, suf)
+		} else {
+			tlsdial.SetConfigExpectedCert(conf, node.CertName)
+		}
+	}
+	return conf
+}
+
 func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) {
 	if !h.PermitWrite {
 		http.Error(w, "debug access denied", http.StatusForbidden)
@@ -100,9 +118,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) {
 			defer conn.Close()
 
 			// Upgrade to TLS and verify that works properly.
-			tlsConn := tls.Client(conn, &tls.Config{
+			tlsConn := tls.Client(conn, tlsConfigForNode(derpNode))
-				ServerName: cmp.Or(derpNode.CertName, derpNode.HostName),
-			})
 			if err := tlsConn.HandshakeContext(ctx); err != nil {
 				st.Errors = append(st.Errors, fmt.Sprintf("Error upgrading connection to node %q @ %q to TLS over IPv4: %v", derpNode.HostName, addr, err))
 			} else {
@@ -119,12 +135,7 @@ func (h *Handler) serveDebugDERPRegion(w http.ResponseWriter, r *http.Request) {
 			defer conn.Close()
 
 			// Upgrade to TLS and verify that works properly.
-			tlsConn := tls.Client(conn, &tls.Config{
+			tlsConn := tls.Client(conn, tlsConfigForNode(derpNode))
-				ServerName: cmp.Or(derpNode.CertName, derpNode.HostName),
-				// TODO(andrew-d): we should print more
-				// detailed failure information on if/why TLS
-				// verification fails
-			})
 			if err := tlsConn.HandshakeContext(ctx); err != nil {
 				st.Errors = append(st.Errors, fmt.Sprintf("Error upgrading connection to node %q @ %q to TLS over IPv6: %v", derpNode.HostName, addr, err))
 			} else {

tstest/natlab/vmtest/selfsignedderp_test.go

@@ -0,0 +1,60 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package vmtest_test
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"tailscale.com/tstest/natlab/vmtest"
+)
+
+// TestSelfSignedDERPHashPinning exercises the sha256-raw DERP cert pinning
+// code path end-to-end: tailscaled connects to its home DERP whose cert is
+// self-signed and pinned via CertName="sha256-raw:<hex>" (no separate
+// fronting CertName), the two nodes communicate over the resulting tailnet,
+// and `tailscale debug derp` against the same region succeeds.
+//
+// Both nodes sit behind hard NATs with no port mapping available so disco
+// cannot punch a direct path and the tailnet ping must traverse DERP, making
+// the sha256-raw pinning of the tailscaled→DERP path part of the assertion.
+//
+// The debug-derp half is the regression test for the bug fixed in PR #19965:
+// before that change, [ipn/localapi.serveDebugDERPRegion] passed the raw
+// sha256-raw fingerprint as the TLS ServerName and the handshake always
+// failed with a hostname mismatch.
+func TestSelfSignedDERPHashPinning(t *testing.T) {
+	env := vmtest.New(t, vmtest.SelfSignedDERPCertPinning())
+	n1 := hard(env)
+	n2 := hard(env)
+	env.Start()
+
+	if err := env.PingExpect(n1, n2, vmtest.PingRouteDERP, 60*time.Second); err != nil {
+		t.Fatalf("ping node-0 -> node-1: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+
+	for _, n := range []*vmtest.Node{n1, n2} {
+		rep, err := n.Agent().DebugDERPRegion(ctx, "1")
+		if err != nil {
+			t.Fatalf("[%s] DebugDERPRegion(1): %v", n.Name(), err)
+		}
+		t.Logf("[%s] DebugDERPRegion(1): info=%v warnings=%v errors=%v",
+			n.Name(), rep.Info, rep.Warnings, rep.Errors)
+		for _, e := range rep.Errors {
+			// The `hard` builder gives the node only an IPv4 LAN, so the
+			// DebugDERPRegion IPv6 probe predictably fails with
+			// "network is unreachable". That's orthogonal to the TLS
+			// verification path this test exists to cover.
+			if strings.Contains(e, "over IPv6") {
+				continue
+			}
+			t.Errorf("[%s] DebugDERPRegion(1) error: %s", n.Name(), e)
+		}
+	}
+}

tstest/natlab/vmtest/vmtest.go

@@ -89,9 +89,10 @@ type Env struct {
 
 	qemuProcs []*exec.Cmd // launched QEMU processes
 
-	sameTailnetUser bool // all nodes register as the same Tailnet user
+	sameTailnetUser           bool // all nodes register as the same Tailnet user
-	allOnline       bool // mark every peer as Online=true in MapResponses
+	allOnline                 bool // mark every peer as Online=true in MapResponses
-	peerRelayGrants bool // grant peer-relay capabilities on the wildcard packet filter
+	peerRelayGrants           bool // grant peer-relay capabilities on the wildcard packet filter
+	selfSignedDERPCertPinning bool // serve test DERP map with sha256-raw cert pins
 
 	// Shared resource initialization (sync.Once for things multiple nodes share).
 	vnetOnce      sync.Once
@@ -384,6 +385,15 @@ func PeerRelayGrants() EnvOption {
 	return envOptFunc(func(e *Env) { e.peerRelayGrants = true })
 }
 
+// SelfSignedDERPCertPinning returns an [EnvOption] that makes the test control
+// server advertise a DERP map whose nodes use CertName="sha256-raw:<hex>"
+// pinning against the self-signed certs vnet's fake DERP servers serve. This
+// exercises the sha256-raw verification path end-to-end (in tailscaled and in
+// `tailscale debug derp`) without involving a real CA.
+func SelfSignedDERPCertPinning() EnvOption {
+	return envOptFunc(func(e *Env) { e.selfSignedDERPCertPinning = true })
+}
+
 // AddNetwork creates a new virtual network. Arguments follow the same pattern as
 // vnet.Config.AddNetwork (string IPs, NAT types, NetworkService values).
 func (e *Env) AddNetwork(opts ...any) *vnet.Network {
@@ -1379,9 +1389,44 @@ func (e *Env) initVnet() {
 		if e.peerRelayGrants {
 			e.server.ControlServer().PeerRelayGrants = true
 		}
+		if e.selfSignedDERPCertPinning {
+			e.server.ControlServer().DERPMap = e.buildSelfSignedDERPMap()
+		}
 	})
 }
 
+// buildSelfSignedDERPMap returns a DERP map identical in structure to the
+// stock test map (same regions, hostnames, virtual IPs) but with each node's
+// CertName set to "sha256-raw:<hex>" pinning the actual self-signed cert
+// served by vnet's fake DERP server, and InsecureForTests cleared so the
+// pin is actually exercised. Nodes are matched to certs by HostName.
+func (e *Env) buildSelfSignedDERPMap() *tailcfg.DERPMap {
+	hostToHash := make(map[string]string, 2)
+	for i := range 2 {
+		hostToHash[e.server.DERPHostname(i)] = e.server.DERPCertSHA256Hex(i)
+	}
+	src := e.server.ControlServer().DERPMap
+	dm := &tailcfg.DERPMap{
+		Regions: make(map[int]*tailcfg.DERPRegion, len(src.Regions)),
+	}
+	for id, srcRegion := range src.Regions {
+		r := *srcRegion
+		r.Nodes = make([]*tailcfg.DERPNode, len(srcRegion.Nodes))
+		for i, srcNode := range srcRegion.Nodes {
+			n := *srcNode
+			hash, ok := hostToHash[n.HostName]
+			if !ok {
+				e.t.Fatalf("buildSelfSignedDERPMap: no cert hash for HostName %q", n.HostName)
+			}
+			n.InsecureForTests = false
+			n.CertName = "sha256-raw:" + hash
+			r.Nodes[i] = &n
+		}
+		dm.Regions[id] = &r
+	}
+	return dm
+}
+
 // ensureQEMUSocket creates the Unix stream socket for QEMU VMs. Called once.
 func (e *Env) ensureQEMUSocket() {
 	e.qemuSockOnce.Do(func() {

tstest/natlab/vnet/vnet.go

@@ -14,7 +14,13 @@ package vnet
 import (
 	"bytes"
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	crand "crypto/rand"
+	"crypto/sha256"
 	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
@@ -23,10 +29,10 @@ import (
 	"iter"
 	"log"
 	"maps"
+	"math/big"
 	"math/rand/v2"
 	"net"
 	"net/http"
-	"net/http/httptest"
 	"net/netip"
 	"os/exec"
 	"strconv"
@@ -734,16 +740,23 @@ type derpServer struct {
 	srv       *derpserver.Server
 	handler   http.Handler
 	tlsConfig *tls.Config
+	// certSHA256Hex is the SHA-256 hex fingerprint of the leaf certificate
+	// served by this DERP server. It is the value tests pin against when
+	// they configure a custom DERP map with CertName="sha256-raw:<hex>".
+	certSHA256Hex string
 }
 
-func newDERPServer() *derpServer {
+// newDERPServer returns a derpServer whose TLS cert is a freshly generated
-	// Just to get a self-signed TLS cert:
+// self-signed ECDSA cert valid for hostname. Tests that use a stock test DERP
-	ts := httptest.NewTLSServer(nil)
+// map with InsecureForTests=true ignore the cert content entirely; tests that
-	ts.Close()
+// want to exercise sha256-raw cert pinning can read the certSHA256Hex via
-
+// [Server.DERPCertSHA256Hex].
+func newDERPServer(hostname string) *derpServer {
+	tlsConfig, certHex := selfSignedDERPCert(hostname)
 	ds := &derpServer{
-		srv:       derpserver.New(key.NewNode(), logger.Discard),
+		srv:           derpserver.New(key.NewNode(), logger.Discard),
-		tlsConfig: ts.TLS, // self-signed; test client configure to not check
+		tlsConfig:     tlsConfig,
+		certSHA256Hex: certHex,
 	}
 	var mux http.ServeMux
 	mux.Handle("/derp", derpserver.Handler(ds.srv))
@@ -753,6 +766,40 @@ func newDERPServer() *derpServer {
 	return ds
 }
 
+// selfSignedDERPCert builds a self-signed ECDSA P-256 cert valid for hostname
+// and returns a *tls.Config that serves it, along with the SHA-256 hex digest
+// of the cert's DER bytes.
+func selfSignedDERPCert(hostname string) (*tls.Config, string) {
+	key, err := ecdsa.GenerateKey(elliptic.P256(), crand.Reader)
+	if err != nil {
+		panic(fmt.Sprintf("vnet: generating DERP cert key: %v", err))
+	}
+	tmpl := &x509.Certificate{
+		SerialNumber: big.NewInt(1),
+		Subject:      pkix.Name{CommonName: hostname},
+		NotBefore:    time.Now().Add(-time.Hour),
+		NotAfter:     time.Now().Add(365 * 24 * time.Hour),
+		KeyUsage:     x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:  []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		DNSNames:     []string{hostname},
+	}
+	if ip := net.ParseIP(hostname); ip != nil {
+		tmpl.IPAddresses = []net.IP{ip}
+		tmpl.DNSNames = nil
+	}
+	der, err := x509.CreateCertificate(crand.Reader, tmpl, tmpl, &key.PublicKey, key)
+	if err != nil {
+		panic(fmt.Sprintf("vnet: creating DERP cert: %v", err))
+	}
+	cfg := &tls.Config{
+		Certificates: []tls.Certificate{{
+			Certificate: [][]byte{der},
+			PrivateKey:  key,
+		}},
+	}
+	return cfg, fmt.Sprintf("%x", sha256.Sum256(der))
+}
+
 type Server struct {
 	shutdownCtx    context.Context
 	shutdownCancel context.CancelFunc
@@ -810,6 +857,11 @@ func (s *Server) SetDHCPCallback(fn func(MAC, int, layers.DHCPMsgType, netip.Add
 	s.onDHCPEvent = fn
 }
 
+// derpHostnames are the SNI/HostName values vnet's fake DERP servers identify
+// as. They are also used to issue the per-DERP self-signed certificate so that
+// hostname verification succeeds for tests that pin via sha256-raw.
+var derpHostnames = []string{"derp1.tailscale", "derp2.tailscale"}
+
 var derpMap = &tailcfg.DERPMap{
 	Regions: map[int]*tailcfg.DERPRegion{
 		1: {
@@ -820,7 +872,7 @@ var derpMap = &tailcfg.DERPMap{
 				{
 					Name:             "1a",
 					RegionID:         1,
-					HostName:         "derp1.tailscale",
+					HostName:         derpHostnames[0],
 					IPv4:             fakeDERP1.v4.String(),
 					IPv6:             fakeDERP1.v6.String(),
 					InsecureForTests: true,
@@ -836,7 +888,7 @@ var derpMap = &tailcfg.DERPMap{
 				{
 					Name:             "2a",
 					RegionID:         2,
-					HostName:         "derp2.tailscale",
+					HostName:         derpHostnames[1],
 					IPv4:             fakeDERP2.v4.String(),
 					IPv6:             fakeDERP2.v6.String(),
 					InsecureForTests: true,
@@ -865,8 +917,8 @@ func New(c *Config) (*Server, error) {
 		networkByWAN: &bart.Table[*network]{},
 		networks:     set.Of[*network](),
 	}
-	for range 2 {
+	for _, host := range derpHostnames {
-		s.derps = append(s.derps, newDERPServer())
+		s.derps = append(s.derps, newDERPServer(host))
 	}
 	if err := s.initFromConfig(c); err != nil {
 		return nil, err
@@ -889,6 +941,20 @@ func (s *Server) ControlServer() *testcontrol.Server {
 	return s.control
 }
 
+// DERPHostname returns the SNI/HostName used by vnet's idx'th fake DERP
+// server. idx must be 0 or 1.
+func (s *Server) DERPHostname(idx int) string {
+	return derpHostnames[idx]
+}
+
+// DERPCertSHA256Hex returns the SHA-256 hex fingerprint of the self-signed
+// TLS certificate served by vnet's idx'th fake DERP server. It is the value
+// to pin against in a [tailcfg.DERPNode.CertName] formatted as
+// "sha256-raw:<hex>". idx must be 0 or 1.
+func (s *Server) DERPCertSHA256Hex(idx int) string {
+	return s.derps[idx].certSHA256Hex
+}
+
 // CloudInitData holds the cloud-init configuration for a node.
 type CloudInitData struct {
 	MetaData      string