wgengine,util/execqueue: wait for in-flight linkChange before closing

ExecQueue.Shutdown does not wait for a function that is already
executing, so Close could tear down magicConn, dns, wgdev, and tundev
while a queued linkChange was still using them, panicking during
shutdown. Add ExecQueue.ShutdownAndWait, which discards queued
functions that have not started and waits for the in-flight one, and
use it in Close with a bounded context before tearing anything down.
The eventbus client is closed first and is the queue's only producer,
so no new work can arrive after the drain.

Updates #17641

Change-Id: I0350bcb59c1ee4b0dcac88cf66b93828466c8c98
Signed-off-by: Adel-Ayoub <adelayoub.maaziz@gmail.com>

wgengine/userspace_test.go

@@ -12,6 +12,7 @@ import (
 	"slices"
 	"sync"
 	"testing"
+	"time"
 
 	"github.com/tailscale/wireguard-go/device"
 	"go4.org/mem"
@@ -643,3 +644,40 @@ func TestLinkChangeReapplyPreservesMagicDNSRoutes(t *testing.T) {
 			logger.AsJSON(initial), logger.AsJSON(after))
 	}
 }
+
+// TestCloseWaitsForLinkChange tests that Close waits for in-flight
+// linkChangeQueue work to finish before tearing down the subsystems
+// that linkChange uses.
+//
+// See https://github.com/tailscale/tailscale/issues/17641.
+func TestCloseWaitsForLinkChange(t *testing.T) {
+	bus := eventbustest.NewBus(t)
+
+	ht := health.NewTracker(bus)
+	reg := new(usermetric.Registry)
+	e, err := NewFakeUserspaceEngine(t.Logf, 0, ht, reg, bus)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	started := make(chan struct{})
+	release := make(chan struct{})
+	done := make(chan struct{})
+	e.(*userspaceEngine).linkChangeQueue.Add(func() {
+		close(started)
+		<-release
+		close(done)
+	})
+	<-started
+
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		close(release)
+	}()
+	e.Close()
+	select {
+	case <-done:
+	default:
+		t.Fatal("Close returned with link change work still in flight")
+	}
+}