util/clientmetric, wgengine/watchdog: report watchdog errors in user/client metrics (#18591)

fixes tailscale/corp#36708

Sets up a set of metrics to report watchdog timeouts for wgengine and
reports an event for any watchdog timeout.

Signed-off-by: Jonathan Nobels <jonathan@tailscale.com>

wgengine/watchdog_test.go

@@ -1,10 +1,13 @@
 // Copyright (c) Tailscale Inc & contributors
 // SPDX-License-Identifier: BSD-3-Clause
 
+//go:build !js
+
 package wgengine
 
 import (
 	"runtime"
+	"sync"
 	"testing"
 	"time"
 
@@ -44,3 +47,95 @@ func TestWatchdog(t *testing.T) {
 		e.Close()
 	})
 }
+
+func TestWatchdogMetrics(t *testing.T) {
+	tests := []struct {
+		name       string
+		events     []watchdogEvent
+		wantCounts map[watchdogEvent]int64
+	}{
+		{
+			name:   "single event types",
+			events: []watchdogEvent{RequestStatus, PeerForIPEvent, Ping},
+			wantCounts: map[watchdogEvent]int64{
+				RequestStatus:  1,
+				PeerForIPEvent: 1,
+				Ping:           1,
+			},
+		},
+		{
+			name:   "repeated events",
+			events: []watchdogEvent{RequestStatus, RequestStatus, Ping, RequestStatus},
+			wantCounts: map[watchdogEvent]int64{
+				RequestStatus: 3,
+				Ping:          1,
+			},
+		},
+	}
+
+	// For swallowing fatalf calls and stack traces
+	logf := func(format string, args ...any) {}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			clearMetrics(t)
+			bus := eventbustest.NewBus(t)
+			ht := health.NewTracker(bus)
+			reg := new(usermetric.Registry)
+			e, err := NewFakeUserspaceEngine(logf, 0, ht, reg, bus)
+			if err != nil {
+				t.Fatal(err)
+			}
+			e = NewWatchdog(e)
+			w := e.(*watchdogEngine)
+			w.maxWait = 1 * time.Microsecond
+			w.logf = logf
+			w.fatalf = logf
+
+			var wg sync.WaitGroup
+			wg.Add(len(tt.events))
+
+			for _, ev := range tt.events {
+				blocked := make(chan struct{})
+				w.watchdog(ev, func() {
+					defer wg.Done()
+					<-blocked
+				})
+				close(blocked)
+			}
+			wg.Wait()
+
+			// Check individual event counts
+			for ev, want := range tt.wantCounts {
+				m, ok := watchdogMetrics[ev]
+				if !ok {
+					t.Fatalf("no metric found for event %q", ev)
+				}
+				got := m.Value()
+				if got != want {
+					t.Errorf("got %d metric events for %q, want %d", got, ev, want)
+				}
+			}
+
+			// Check total count for Any
+			m, ok := watchdogMetrics[Any]
+			if !ok {
+				t.Fatalf("no Any metric found")
+			}
+			got := m.Value()
+			if got != int64(len(tt.events)) {
+				t.Errorf("got %d metric events for Any, want %d", got, len(tt.events))
+			}
+		})
+	}
+}
+
+func clearMetrics(t *testing.T) {
+	t.Helper()
+	if watchdogMetrics == nil {
+		return
+	}
+	for _, m := range watchdogMetrics {
+		m.Set(0)
+	}
+}