net/udprelay: add tailscaled_peer_relay_endpoints gauge (#18265)

New gauge reflects endpoints state via labels:
- open, when both peers are connected and ready to talk, and
- connecting. when at least one peer hasn't connected yet.

Corresponding client metrics are logged as
- udprelay_endpoints_connecting
- udprelay_endpoints_open

Updates tailscale/corp#30820

Change-Id: Idb1baa90a38c97847e14f9b2390093262ad0ea23

Signed-off-by: Alex Valiushko <alexvaliushko@tailscale.com>

net/udprelay/metrics.go

@@ -22,6 +22,17 @@ var (
 	cMetricForwarded46Bytes = clientmetric.NewAggregateCounter("udprelay_forwarded_bytes_udp4_udp6")
 	cMetricForwarded64Bytes = clientmetric.NewAggregateCounter("udprelay_forwarded_bytes_udp6_udp4")
 	cMetricForwarded66Bytes = clientmetric.NewAggregateCounter("udprelay_forwarded_bytes_udp6_udp6")
+
+	// cMetricEndpoints is initialized here with no other writes, making it safe for concurrent reads.
+	//
+	// [clientmetric.Gauge] does not let us embed existing counters, so
+	// [metrics.updateEndpoint] records data into client and user gauges independently.
+	//
+	// Transitions to and from [endpointClosed] are not recorded.
+	cMetricEndpoints = map[endpointState]*clientmetric.Metric{
+		endpointConnecting: clientmetric.NewGauge("udprelay_endpoints_connecting"),
+		endpointOpen:       clientmetric.NewGauge("udprelay_endpoints_open"),
+	}
 )
 
 type transport string
@@ -36,6 +47,10 @@ type forwardedLabel struct {
 	transportOut transport `prom:"transport_out"`
 }
 
+type endpointLabel struct {
+	state endpointState `prom:"state"`
+}
+
 type metrics struct {
 	forwarded44Packets expvar.Int
 	forwarded46Packets expvar.Int
@@ -46,6 +61,11 @@ type metrics struct {
 	forwarded46Bytes expvar.Int
 	forwarded64Bytes expvar.Int
 	forwarded66Bytes expvar.Int
+
+	// endpoints are set in [registerMetrics] and safe for concurrent reads.
+	//
+	// Transitions to and from [endpointClosed] are not recorded
+	endpoints map[endpointState]*expvar.Int
 }
 
 // registerMetrics publishes user and client metric counters for peer relay server.
@@ -65,6 +85,12 @@ func registerMetrics(reg *usermetric.Registry) *metrics {
 			"counter",
 			"Number of bytes forwarded via Peer Relay",
 		)
+		uMetricEndpoints = usermetric.NewMultiLabelMapWithRegistry[endpointLabel](
+			reg,
+			"tailscaled_peer_relay_endpoints",
+			"gauge",
+			"Number of allocated Peer Relay endpoints",
+		)
 		forwarded44 = forwardedLabel{transportIn: transportUDP4, transportOut: transportUDP4}
 		forwarded46 = forwardedLabel{transportIn: transportUDP4, transportOut: transportUDP6}
 		forwarded64 = forwardedLabel{transportIn: transportUDP6, transportOut: transportUDP4}
@@ -83,6 +109,13 @@ func registerMetrics(reg *usermetric.Registry) *metrics {
 	uMetricForwardedBytes.Set(forwarded64, &m.forwarded64Bytes)
 	uMetricForwardedBytes.Set(forwarded66, &m.forwarded66Bytes)
 
+	m.endpoints = map[endpointState]*expvar.Int{
+		endpointConnecting: {},
+		endpointOpen:       {},
+	}
+	uMetricEndpoints.Set(endpointLabel{endpointOpen}, m.endpoints[endpointOpen])
+	uMetricEndpoints.Set(endpointLabel{endpointConnecting}, m.endpoints[endpointConnecting])
+
 	// Publish client metrics.
 	cMetricForwarded44Packets.Register(&m.forwarded44Packets)
 	cMetricForwarded46Packets.Register(&m.forwarded46Packets)
@@ -96,6 +129,26 @@ func registerMetrics(reg *usermetric.Registry) *metrics {
 	return m
 }
 
+type endpointUpdater interface {
+	updateEndpoint(before, after endpointState)
+}
+
+// updateEndpoint updates the endpoints gauge according to states left and entered.
+// It records client-metric gauges independently, see [cMetricEndpoints] doc.
+func (m *metrics) updateEndpoint(before, after endpointState) {
+	if before == after {
+		return
+	}
+	if uMetricEndpointsBefore, ok := m.endpoints[before]; ok && before != endpointClosed {
+		uMetricEndpointsBefore.Add(-1)
+		cMetricEndpoints[before].Add(-1)
+	}
+	if uMetricEndpointsAfter, ok := m.endpoints[after]; ok && after != endpointClosed {
+		uMetricEndpointsAfter.Add(1)
+		cMetricEndpoints[after].Add(1)
+	}
+}
+
 // countForwarded records user and client metrics according to the
 // inbound and outbound address families.
 func (m *metrics) countForwarded(in4, out4 bool, bytes, packets int64) {
@@ -114,8 +167,7 @@ func (m *metrics) countForwarded(in4, out4 bool, bytes, packets int64) {
 	}
 }
 
-// deregisterMetrics unregisters the underlying expvar counters
-// from clientmetrics.
+// deregisterMetrics clears clientmetrics counters and resets gauges to zero.
 func deregisterMetrics() {
 	cMetricForwarded44Packets.UnregisterAll()
 	cMetricForwarded46Packets.UnregisterAll()
@@ -125,4 +177,7 @@ func deregisterMetrics() {
 	cMetricForwarded46Bytes.UnregisterAll()
 	cMetricForwarded64Bytes.UnregisterAll()
 	cMetricForwarded66Bytes.UnregisterAll()
+	for _, v := range cMetricEndpoints {
+		v.Set(0)
+	}
 }