cmd/k8s-operator: use correct tailnet client for L7 & L3 ingresses (#18749)

* cmd/k8s-operator: use correct tailnet client for L7 & L3 ingresses

This commit fixes a bug when using multi-tailnet within the operator
to spin up L7 & L3 ingresses where the client used to create the
tailscale services was not switching depending on the tailnet used
by the proxygroup backing the service/ingress.

Updates: https://github.com/tailscale/corp/issues/34561

Signed-off-by: David Bond <davidsbond93@gmail.com>

* cmd/k8s-operator: adding server url to proxygroups when a custom tailnet has been specified

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
(cherry picked from commit 3b21ac5504e713e32dfcd43d9ee21e7e712ac200)

---------

Signed-off-by: David Bond <davidsbond93@gmail.com>
Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
Co-authored-by: chaosinthecrd <tom@tmlabs.co.uk>

cmd/k8s-operator/api-server-proxy-pg_test.go

@@ -16,8 +16,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
 	"tailscale.com/internal/client/tailscale"
-	"tailscale.com/ipn/ipnstate"
 	tsoperator "tailscale.com/k8s-operator"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
 	"tailscale.com/kube/k8s-proxy/conf"
@@ -107,14 +107,6 @@ func TestAPIServerProxyReconciler(t *testing.T) {
 	}
 	ft.CreateOrUpdateVIPService(t.Context(), ingressTSSvc)
 
-	lc := &fakeLocalClient{
-		status: &ipnstate.Status{
-			CurrentTailnet: &ipnstate.TailnetStatus{
-				MagicDNSSuffix: "ts.net",
-			},
-		},
-	}
-
 	r := &KubeAPIServerTSServiceReconciler{
 		Client:      fc,
 		tsClient:    ft,
@@ -122,7 +114,6 @@ func TestAPIServerProxyReconciler(t *testing.T) {
 		tsNamespace: ns,
 		logger:      zap.Must(zap.NewDevelopment()).Sugar(),
 		recorder:    record.NewFakeRecorder(10),
-		lc:          lc,
 		clock:       tstest.NewClock(tstest.ClockOpts{}),
 		operatorID:  "self-id",
 	}
@@ -147,6 +138,20 @@ func TestAPIServerProxyReconciler(t *testing.T) {
 	if err := ft.DeleteVIPService(t.Context(), "svc:"+pgName); err != nil {
 		t.Fatalf("deleting initial Tailscale Service: %v", err)
 	}
+
+	// Create the state secret for the ProxyGroup without services being advertised.
+	mustCreate(t, fc, &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "test-pg-0",
+			Namespace: ns,
+			Labels:    pgSecretLabels(pgName, kubetypes.LabelSecretTypeState),
+		},
+		Data: map[string][]byte{
+			"_current-profile": []byte("test"),
+			"test":             []byte(`{"Config":{"NodeID":"node-foo", "UserProfile": {"LoginName": "test-pg.ts.net" }}}`),
+		},
+	})
+
 	expectReconciled(t, r, "", pgName)
 
 	tsSvc, err := ft.GetVIPService(t.Context(), "svc:"+pgName)
@@ -190,17 +195,19 @@ func TestAPIServerProxyReconciler(t *testing.T) {
 	expectEqual(t, fc, pg, omitPGStatusConditionMessages) // Unchanged status.
 
 	// Simulate Pod prefs updated with advertised services; should see Configured condition updated to true.
-	mustCreate(t, fc, &corev1.Secret{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "test-pg-0",
-			Namespace: ns,
-			Labels:    pgSecretLabels(pgName, kubetypes.LabelSecretTypeState),
-		},
-		Data: map[string][]byte{
-			"_current-profile": []byte("profile-foo"),
-			"profile-foo":      []byte(`{"AdvertiseServices":["svc:test-pg"],"Config":{"NodeID":"node-foo"}}`),
-		},
+	mustUpdate(t, fc, ns, "test-pg-0", func(o *corev1.Secret) {
+		var p prefs
+		if err = json.Unmarshal(o.Data["test"], &p); err != nil {
+			t.Errorf("failed to unmarshal preferences: %v", err)
+		}
+
+		p.AdvertiseServices = []string{"svc:test-pg"}
+		o.Data["test"], err = json.Marshal(p)
+		if err != nil {
+			t.Errorf("failed to marshal preferences: %v", err)
+		}
 	})
+
 	expectReconciled(t, r, "", pgName)
 	tsoperator.SetProxyGroupCondition(pg, tsapi.KubeAPIServerProxyConfigured, metav1.ConditionTrue, reasonKubeAPIServerProxyConfigured, "", 1, r.clock, r.logger)
 	pg.Status.URL = "https://" + defaultDomain