cmd/k8s-operator: simplify scope of e2e tests (#17076)

Removes ACL edits from e2e tests in favour of trying to simplify the
tests and separate the actual test logic from the environment setup
logic as much as possible. Also aims to fit in with the requirements
that will generally be filled anyway for most devs working on the
operator; in particular using tags that fit in with our documentation.

Updates tailscale/corp#32085

Change-Id: I7659246e39ec0b7bcc4ec0a00c6310f25fe6fac2

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/k8s-operator/e2e/ingress_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -17,45 +18,63 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	kube "tailscale.com/k8s-operator"
 	"tailscale.com/tstest"
+	"tailscale.com/types/ptr"
+	"tailscale.com/util/httpm"
 )
 
 // See [TestMain] for test requirements.
 func TestIngress(t *testing.T) {
-	if tsClient == nil {
-		t.Skip("TestIngress requires credentials for a tailscale client")
+	if apiClient == nil {
+		t.Skip("TestIngress requires TS_API_CLIENT_SECRET set")
 	}
 
-	ctx := context.Background()
 	cfg := config.GetConfigOrDie()
 	cl, err := client.New(cfg, client.Options{})
 	if err != nil {
 		t.Fatal(err)
 	}
 	// Apply nginx
-	createAndCleanup(t, ctx, cl, &corev1.Pod{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      "nginx",
-			Namespace: "default",
-			Labels: map[string]string{
-				"app.kubernetes.io/name": "nginx",
-			},
-		},
-		Spec: corev1.PodSpec{
-			Containers: []corev1.Container{
-				{
-					Name:  "nginx",
-					Image: "nginx",
+	createAndCleanup(t, cl,
+		&appsv1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "nginx",
+				Namespace: "default",
+				Labels: map[string]string{
+					"app.kubernetes.io/name": "nginx",
 				},
 			},
-		},
-	})
+			Spec: appsv1.DeploymentSpec{
+				Replicas: ptr.To[int32](1),
+				Selector: &metav1.LabelSelector{
+					MatchLabels: map[string]string{
+						"app.kubernetes.io/name": "nginx",
+					},
+				},
+				Template: corev1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"app.kubernetes.io/name": "nginx",
+						},
+					},
+					Spec: corev1.PodSpec{
+						Containers: []corev1.Container{
+							{
+								Name:  "nginx",
+								Image: "nginx",
+							},
+						},
+					},
+				},
+			},
+		})
 	// Apply service to expose it as ingress
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-ingress",
 			Namespace: "default",
 			Annotations: map[string]string{
-				"tailscale.com/expose": "true",
+				"tailscale.com/expose":      "true",
+				"tailscale.com/proxy-class": "prod",
 			},
 		},
 		Spec: corev1.ServiceSpec{
@@ -71,10 +90,10 @@ func TestIngress(t *testing.T) {
 			},
 		},
 	}
-	createAndCleanup(t, ctx, cl, svc)
+	createAndCleanup(t, cl, svc)
 
 	// TODO: instead of timing out only when test times out, cancel context after 60s or so.
-	if err := wait.PollUntilContextCancel(ctx, time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) {
+	if err := wait.PollUntilContextCancel(t.Context(), time.Millisecond*100, true, func(ctx context.Context) (done bool, err error) {
 		maybeReadySvc := &corev1.Service{ObjectMeta: objectMeta("default", "test-ingress")}
 		if err := get(ctx, cl, maybeReadySvc); err != nil {
 			return false, err
@@ -89,17 +108,20 @@ func TestIngress(t *testing.T) {
 	}
 
 	var resp *http.Response
-	if err := tstest.WaitFor(time.Second*60, func() error {
+	if err := tstest.WaitFor(time.Minute, func() error {
 		// TODO(tomhjp): Get the tailnet DNS name from the associated secret instead.
 		// If we are not the first tailnet node with the requested name, we'll get
 		// a -N suffix.
-		resp, err = tsClient.HTTPClient.Get(fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name))
+		req, err := http.NewRequest(httpm.GET, fmt.Sprintf("http://%s-%s:80", svc.Namespace, svc.Name), nil)
 		if err != nil {
 			return err
 		}
-		return nil
+		ctx, cancel := context.WithTimeout(t.Context(), time.Second)
+		defer cancel()
+		resp, err = tailnetClient.HTTPClient().Do(req.WithContext(ctx))
+		return err
 	}); err != nil {
-		t.Fatalf("error trying to reach service: %v", err)
+		t.Fatalf("error trying to reach Service: %v", err)
 	}
 
 	if resp.StatusCode != http.StatusOK {