wgengine,util/execqueue: wait for in-flight linkChange before closing

ExecQueue.Shutdown does not wait for a function that is already
executing, so Close could tear down magicConn, dns, wgdev, and tundev
while a queued linkChange was still using them, panicking during
shutdown. Add ExecQueue.ShutdownAndWait, which discards queued
functions that have not started and waits for the in-flight one, and
use it in Close with a bounded context before tearing anything down.
The eventbus client is closed first and is the queue's only producer,
so no new work can arrive after the drain.

Updates #17641

Change-Id: I0350bcb59c1ee4b0dcac88cf66b93828466c8c98
Signed-off-by: Adel-Ayoub <adelayoub.maaziz@gmail.com>

util/execqueue/execqueue_test.go

@@ -7,6 +7,8 @@ import (
 	"context"
 	"sync/atomic"
 	"testing"
+	"testing/synctest"
+	"time"
 )
 
 func TestExecQueue(t *testing.T) {
@@ -29,3 +31,55 @@ func TestExecQueueRunSyncLocking(t *testing.T) {
 		q.Shutdown()
 	})
 }
+
+func TestShutdownAndWait(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		q := &ExecQueue{}
+		started := make(chan struct{})
+		release := make(chan struct{})
+		var finished, ranPending atomic.Bool
+		q.Add(func() {
+			close(started)
+			<-release
+			finished.Store(true)
+		})
+		q.Add(func() { ranPending.Store(true) })
+		<-started
+
+		// The fake clock only advances once ShutdownAndWait below is
+		// blocked, so the release cannot fire early.
+		go func() {
+			time.Sleep(time.Second)
+			close(release)
+		}()
+		if err := q.ShutdownAndWait(context.Background()); err != nil {
+			t.Fatalf("ShutdownAndWait: %v", err)
+		}
+		if !finished.Load() {
+			t.Error("ShutdownAndWait returned before the in-flight function completed")
+		}
+		if ranPending.Load() {
+			t.Error("pending function ran after shutdown")
+		}
+	})
+}
+
+func TestShutdownAndWaitTimeout(t *testing.T) {
+	synctest.Test(t, func(t *testing.T) {
+		q := &ExecQueue{}
+		started := make(chan struct{})
+		release := make(chan struct{})
+		q.Add(func() {
+			close(started)
+			<-release
+		})
+		<-started
+
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+		defer cancel()
+		if err := q.ShutdownAndWait(ctx); err == nil {
+			t.Error("ShutdownAndWait = nil; want deadline exceeded")
+		}
+		close(release)
+	})
+}