control/controlknobs,feature/*,ipn/ipnlocal,tailcfg: add runtimemetrics

Emit runtime metrics as clientmetrics when the
NodeAttrEmitRuntimeMetrics NodeCapability is present.

We start small with just 2 metrics: heap bytes and total process memory.

Updates tailscale/corp#39434

Signed-off-by: Jordan Whited <jordan@tailscale.com>

feature/buildfeatures/feature_runtimemetrics_disabled.go

@@ -0,0 +1,13 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Code generated by gen.go; DO NOT EDIT.
+
+//go:build ts_omit_runtimemetrics
+
+package buildfeatures
+
+// HasRuntimeMetrics is whether the binary was built with support for modular feature "Support emission of runtime/metrics as clientmetrics".
+// Specifically, it's whether the binary was NOT built with the "ts_omit_runtimemetrics" build tag.
+// It's a const so it can be used for dead code elimination.
+const HasRuntimeMetrics = false

feature/buildfeatures/feature_runtimemetrics_enabled.go

@@ -0,0 +1,13 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Code generated by gen.go; DO NOT EDIT.
+
+//go:build !ts_omit_runtimemetrics
+
+package buildfeatures
+
+// HasRuntimeMetrics is whether the binary was built with support for modular feature "Support emission of runtime/metrics as clientmetrics".
+// Specifically, it's whether the binary was NOT built with the "ts_omit_runtimemetrics" build tag.
+// It's a const so it can be used for dead code elimination.
+const HasRuntimeMetrics = true

feature/condregister/maybe_runtimemetrics.go

@@ -0,0 +1,8 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !ts_omit_runtimemetrics
+
+package condregister
+
+import _ "tailscale.com/feature/runtimemetrics"

feature/featuretags/featuretags.go

@@ -239,6 +239,10 @@ var Features = map[FeatureTag]FeatureMeta{
 		Sym:  "RouteCheck",
 		Desc: "Support checking the reachability of overlapping routers, for choosing between multiple network paths to the same IP address",
 	},
+	"runtimemetrics": {
+		Sym:  "RuntimeMetrics",
+		Desc: "Support emission of runtime/metrics as clientmetrics",
+	},
 	"sdnotify": {
 		Sym:  "SDNotify",
 		Desc: "systemd notification support",

feature/runtimemetrics/runtimemetrics.go

@@ -0,0 +1,132 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+// Package runtimemetrics exports select runtime/metrics as [tailscale.com/util/clientmetric]'s.
+package runtimemetrics
+
+import (
+	"runtime/metrics"
+	"sync"
+	"time"
+
+	"tailscale.com/feature"
+	"tailscale.com/ipn/ipnlocal"
+	"tailscale.com/util/clientmetric"
+)
+
+func init() {
+	feature.Register("runtimemetrics")
+	ipnlocal.HookSetRuntimeMetricsEnabled.Set(setEnabled)
+}
+
+var (
+	setEnabledMu  sync.Mutex // guards runningPoller
+	runningPoller *poller    // non-nil when running, otherwise nil
+)
+
+func setEnabled(enabled bool) {
+	setEnabledMu.Lock()
+	defer setEnabledMu.Unlock()
+	if enabled {
+		if runningPoller != nil {
+			return
+		}
+		runningPoller = newPoller()
+	} else {
+		if runningPoller == nil {
+			return
+		}
+		runningPoller.close()
+		runningPoller = nil
+	}
+}
+
+type poller struct {
+	closeOnce sync.Once
+	closeCh   chan struct{}
+	wg        sync.WaitGroup
+}
+
+func newPoller() *poller {
+	p := &poller{
+		closeCh: make(chan struct{}),
+	}
+	p.wg.Go(p.run)
+	return p
+}
+
+func (p *poller) close() {
+	p.closeOnce.Do(func() {
+		close(p.closeCh)
+		p.wg.Wait()
+	})
+}
+
+const (
+	// pollInterval is how frequently [poller] polls Go runtime metrics. Its
+	// value mirrors [tailscale.com/util/clientmetric.minMetricEncodeInterval],
+	// which is the minimum interval between clientmetrics emissions.
+	pollInterval = 15 * time.Second
+)
+
+type sampleNameClientmetric struct {
+	sampleName       string               // [metrics.Sample.Name]
+	clientmetricName string               // passed to clientmetric.New...
+	metric           *clientmetric.Metric // lazy init on first pollAndEmit
+}
+
+var clientmetrics = []sampleNameClientmetric{
+	{
+		// Memory occupied by live objects and dead objects that have not
+		// yet been marked free by the garbage collector.
+		sampleName:       "/memory/classes/heap/objects:bytes",
+		clientmetricName: "runtimemetrics_memory_heap_objects_bytes",
+	},
+	{
+		// All memory mapped by the Go runtime into the current process
+		// as read-write. Note that this does not include memory mapped
+		// by code called via cgo or via the syscall package. Sum of all
+		// metrics in /memory/classes.
+		sampleName:       "/memory/classes/total:bytes",
+		clientmetricName: "runtimemetrics_memory_total_bytes",
+	},
+}
+
+var registerClientmetricsOnce sync.Once
+
+func exportSamples(samples []metrics.Sample) {
+	registerClientmetricsOnce.Do(func() {
+		for i := range clientmetrics {
+			clientmetrics[i].metric = clientmetric.NewGauge(clientmetrics[i].clientmetricName)
+		}
+	})
+	for i := range samples {
+		if samples[i].Value.Kind() != metrics.KindUint64 {
+			continue
+		}
+		clientmetrics[i].metric.Set(int64(samples[i].Value.Uint64()))
+	}
+}
+
+func (p *poller) pollAndEmit() {
+	samples := make([]metrics.Sample, len(clientmetrics))
+	for i := range clientmetrics {
+		samples[i].Name = clientmetrics[i].sampleName
+	}
+	metrics.Read(samples)
+	exportSamples(samples)
+}
+
+func (p *poller) run() {
+	ticker := time.NewTicker(pollInterval)
+	defer ticker.Stop()
+	p.pollAndEmit() // pollAndEmit immediately
+	for {
+		select {
+		case <-p.closeCh:
+			return
+		case <-ticker.C:
+			p.pollAndEmit()
+		}
+	}
+}

feature/runtimemetrics/runtimemetrics_test.go

@@ -0,0 +1,117 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package runtimemetrics
+
+import (
+	"testing"
+	"testing/synctest"
+	"time"
+)
+
+func TestSetEnabledEndToEnd(t *testing.T) {
+	synctest.Test(t, syncTestSetEnabledEndToEnd)
+}
+
+func syncTestSetEnabledEndToEnd(t *testing.T) {
+	getPoller := func() *poller {
+		setEnabledMu.Lock()
+		defer setEnabledMu.Unlock()
+		return runningPoller
+	}
+
+	if p := getPoller(); p != nil {
+		t.Fatalf("runningPoller not nil at test start: %p", p)
+	}
+	t.Cleanup(func() { setEnabled(false) })
+
+	// disabled -> enabled: starts a poller that immediately polls.
+	setEnabled(true)
+	p1 := getPoller()
+	if p1 == nil {
+		t.Fatal("runningPoller nil after setEnabled(true)")
+	}
+
+	// Wait for the immediate pollAndEmit to finish and the goroutine to
+	// block on the ticker.
+	synctest.Wait()
+
+	// Lazy metric registration must have happened and values must be set
+	// from a real runtime/metrics read. Both currently-tracked metrics
+	// (heap objects + total memory) are always >0 in a running Go process.
+	for i, cm := range clientmetrics {
+		if cm.metric == nil {
+			t.Fatalf("clientmetrics[%d] (%s) metric nil after first poll", i, cm.sampleName)
+		}
+		if got := cm.metric.Value(); got <= 0 {
+			t.Errorf("clientmetrics[%d] (%s) = %d after first poll, want > 0",
+				i, cm.clientmetricName, got)
+		}
+	}
+
+	// setEnabled(true) while enabled is idempotent: same poller instance.
+	setEnabled(true)
+	if p := getPoller(); p != p1 {
+		t.Fatalf("setEnabled(true) replaced poller: got %p, want %p", p, p1)
+	}
+
+	// Overwrite each gauge with a sentinel so we can prove the next tick
+	// re-reads runtime values.
+	const sentinel = int64(-1)
+	for _, cm := range clientmetrics {
+		cm.metric.Set(sentinel)
+	}
+
+	// Advance virtual time one tick. The poller's ticker fires and pollAndEmit
+	// runs again.
+	time.Sleep(pollInterval)
+	synctest.Wait()
+
+	for i, cm := range clientmetrics {
+		if got := cm.metric.Value(); got == sentinel {
+			t.Errorf("clientmetrics[%d] (%s) still sentinel %d after tick; ticker did not fire",
+				i, cm.clientmetricName, got)
+		} else if got <= 0 {
+			t.Errorf("clientmetrics[%d] (%s) = %d after tick, want > 0",
+				i, cm.clientmetricName, got)
+		}
+	}
+
+	// enabled -> disabled: stops the poller; wg.Wait inside close() means
+	// the goroutine has exited by the time setEnabled returns.
+	setEnabled(false)
+	if p := getPoller(); p != nil {
+		t.Fatalf("runningPoller %p still set after setEnabled(false)", p)
+	}
+
+	// After disabling, gauges must remain at their last polled values
+	// indefinitely (no further ticks should overwrite them).
+	for _, cm := range clientmetrics {
+		cm.metric.Set(sentinel)
+	}
+	time.Sleep(10 * pollInterval)
+	synctest.Wait()
+	for i, cm := range clientmetrics {
+		if got := cm.metric.Value(); got != sentinel {
+			t.Errorf("clientmetrics[%d] (%s) = %d after disabling; poller goroutine still running?",
+				i, cm.clientmetricName, got)
+		}
+	}
+
+	// disabled -> disabled: still a no-op.
+	setEnabled(false)
+	if p := getPoller(); p != nil {
+		t.Fatalf("runningPoller %p set after second setEnabled(false)", p)
+	}
+
+	// Re-enable creates a fresh poller, not the closed one.
+	setEnabled(true)
+	synctest.Wait()
+	p2 := getPoller()
+	if p2 == nil {
+		t.Fatal("runningPoller nil on re-enable")
+	}
+	if p2 == p1 {
+		t.Fatal("re-enable returned previously-closed poller")
+	}
+}