ipn/localapi,tstest/natlab: fix debug derp TLS check for sha256-raw CertName

serveDebugDERPRegion built its TLS config with
ServerName: cmp.Or(derpNode.CertName, derpNode.HostName), which for a
"sha256-raw:<hex>" CertName passed the raw fingerprint to Go's stock
verifier as a hostname; the handshake always failed with a hostname
mismatch. This is the second half of #15579; the first half (tailscaled
itself failing with "unexpected multiple certs presented") was fixed in

Extract a tlsConfigForNode helper that mirrors derphttp.Client.tlsClient
so that sha256-raw and domain-fronting CertName values are dispatched
to tlsdial.SetConfigExpectedCertHash and tlsdial.SetConfigExpectedCert
respectively, falling back to HostName when CertName is empty.

The core fix here was originally written by @imnuke in #19965; that PR
also added a unit test in ipn/localapi/debugderp_test.go which is
replaced in this commit by a new vmtest that exercises the whole stack:
vnet now serves a self-signed cert valid for each fake DERP node's
HostName and exposes its SHA-256 fingerprint, and vmtest grows a new
SelfSignedDERPCertPinning EnvOption that swaps the test DERP map's
nodes to CertName="sha256-raw:<hex>" with InsecureForTests cleared.
TestSelfSignedDERPHashPinning then stands up two hard-NAT'd nodes, has
them communicate over DERP, and calls DebugDERPRegion on each. Before
this fix the test fails with the exact x509 hostname-mismatch error
from the original bug; after, it passes.

Updates #15579

Change-Id: I61f38ffebc7ac5abc962639db1ae88f5cd8633b1
Co-authored-by: Nuke <nuke@imnuke.dev>
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

tstest/natlab/vmtest/vmtest.go

@@ -89,9 +89,10 @@ type Env struct {
 
 	qemuProcs []*exec.Cmd // launched QEMU processes
 
-	sameTailnetUser bool // all nodes register as the same Tailnet user
-	allOnline       bool // mark every peer as Online=true in MapResponses
-	peerRelayGrants bool // grant peer-relay capabilities on the wildcard packet filter
+	sameTailnetUser           bool // all nodes register as the same Tailnet user
+	allOnline                 bool // mark every peer as Online=true in MapResponses
+	peerRelayGrants           bool // grant peer-relay capabilities on the wildcard packet filter
+	selfSignedDERPCertPinning bool // serve test DERP map with sha256-raw cert pins
 
 	// Shared resource initialization (sync.Once for things multiple nodes share).
 	vnetOnce      sync.Once
@@ -384,6 +385,15 @@ func PeerRelayGrants() EnvOption {
 	return envOptFunc(func(e *Env) { e.peerRelayGrants = true })
 }
 
+// SelfSignedDERPCertPinning returns an [EnvOption] that makes the test control
+// server advertise a DERP map whose nodes use CertName="sha256-raw:<hex>"
+// pinning against the self-signed certs vnet's fake DERP servers serve. This
+// exercises the sha256-raw verification path end-to-end (in tailscaled and in
+// `tailscale debug derp`) without involving a real CA.
+func SelfSignedDERPCertPinning() EnvOption {
+	return envOptFunc(func(e *Env) { e.selfSignedDERPCertPinning = true })
+}
+
 // AddNetwork creates a new virtual network. Arguments follow the same pattern as
 // vnet.Config.AddNetwork (string IPs, NAT types, NetworkService values).
 func (e *Env) AddNetwork(opts ...any) *vnet.Network {
@@ -1379,9 +1389,44 @@ func (e *Env) initVnet() {
 		if e.peerRelayGrants {
 			e.server.ControlServer().PeerRelayGrants = true
 		}
+		if e.selfSignedDERPCertPinning {
+			e.server.ControlServer().DERPMap = e.buildSelfSignedDERPMap()
+		}
 	})
 }
 
+// buildSelfSignedDERPMap returns a DERP map identical in structure to the
+// stock test map (same regions, hostnames, virtual IPs) but with each node's
+// CertName set to "sha256-raw:<hex>" pinning the actual self-signed cert
+// served by vnet's fake DERP server, and InsecureForTests cleared so the
+// pin is actually exercised. Nodes are matched to certs by HostName.
+func (e *Env) buildSelfSignedDERPMap() *tailcfg.DERPMap {
+	hostToHash := make(map[string]string, 2)
+	for i := range 2 {
+		hostToHash[e.server.DERPHostname(i)] = e.server.DERPCertSHA256Hex(i)
+	}
+	src := e.server.ControlServer().DERPMap
+	dm := &tailcfg.DERPMap{
+		Regions: make(map[int]*tailcfg.DERPRegion, len(src.Regions)),
+	}
+	for id, srcRegion := range src.Regions {
+		r := *srcRegion
+		r.Nodes = make([]*tailcfg.DERPNode, len(srcRegion.Nodes))
+		for i, srcNode := range srcRegion.Nodes {
+			n := *srcNode
+			hash, ok := hostToHash[n.HostName]
+			if !ok {
+				e.t.Fatalf("buildSelfSignedDERPMap: no cert hash for HostName %q", n.HostName)
+			}
+			n.InsecureForTests = false
+			n.CertName = "sha256-raw:" + hash
+			r.Nodes[i] = &n
+		}
+		dm.Regions[id] = &r
+	}
+	return dm
+}
+
 // ensureQEMUSocket creates the Unix stream socket for QEMU VMs. Called once.
 func (e *Env) ensureQEMUSocket() {
 	e.qemuSockOnce.Do(func() {