cmd/k8s-operator,k8s-operator: define ProxyGroupPolicy reconciler (#18654)

This commit implements a reconciler for the new `ProxyGroupPolicy`
custom resource. When created, all `ProxyGroupPolicy` resources
within the same namespace are merged into two `ValidatingAdmissionPolicy`
resources, one for egress and one for ingress.

These policies use CEL expressions to limit the usage of the
"tailscale.com/proxy-group" annotation on `Service` and `Ingress`
resources on create & update.

Included here is also a new e2e test that ensures that resources that
violate the policy return an error on creation, and that once the
policy is changed to allow them they can be created.

Closes: https://github.com/tailscale/corp/issues/36830

Signed-off-by: David Bond <davidsbond93@gmail.com>

cmd/k8s-operator/e2e/doc.go

@@ -24,5 +24,5 @@
 //
 // * go
 // * container runtime with the docker daemon API available
-// * devcontrol: ./tool/go run ./cmd/devcontrol --generate-test-devices=k8s-operator-e2e --scenario-output-dir=/tmp/k8s-operator-e2e --test-dns=http://localhost:8055
+// * devcontrol: ./tool/go run --tags=tailscale_saas ./cmd/devcontrol --generate-test-devices=k8s-operator-e2e --scenario-output-dir=/tmp/k8s-operator-e2e --test-dns=http://localhost:8055
 package e2e

cmd/k8s-operator/e2e/ingress_test.go

@@ -14,6 +14,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+
 	kube "tailscale.com/k8s-operator"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"

cmd/k8s-operator/e2e/main_test.go

@@ -54,12 +54,29 @@ func createAndCleanup(t *testing.T, cl client.Client, obj client.Object) {
 	t.Cleanup(func() {
 		// Use context.Background() for cleanup, as t.Context() is cancelled
 		// just before cleanup functions are called.
-		if err := cl.Delete(context.Background(), obj); err != nil {
+		if err = cl.Delete(context.Background(), obj); err != nil {
 			t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
 		}
 	})
 }
 
+func createAndCleanupErr(t *testing.T, cl client.Client, obj client.Object) error {
+	t.Helper()
+
+	err := cl.Create(t.Context(), obj)
+	if err != nil {
+		return err
+	}
+
+	t.Cleanup(func() {
+		if err = cl.Delete(context.Background(), obj); err != nil {
+			t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
+		}
+	})
+
+	return nil
+}
+
 func get(ctx context.Context, cl client.Client, obj client.Object) error {
 	return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)
 }

cmd/k8s-operator/e2e/pebble.go

@@ -12,6 +12,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"tailscale.com/types/ptr"
 )
 

cmd/k8s-operator/e2e/proxy_test.go

@@ -16,6 +16,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"tailscale.com/ipn"
 	"tailscale.com/tstest"
 )

cmd/k8s-operator/e2e/proxygrouppolicy_test.go

@@ -0,0 +1,161 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+)
+
+// See [TestMain] for test requirements.
+func TestProxyGroupPolicy(t *testing.T) {
+	if tnClient == nil {
+		t.Skip("TestProxyGroupPolicy requires a working tailnet client")
+	}
+
+	// Apply deny-all policy
+	denyAllPolicy := &tsapi.ProxyGroupPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "deny-all",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: tsapi.ProxyGroupPolicySpec{
+			Ingress: []string{},
+			Egress:  []string{},
+		},
+	}
+
+	createAndCleanup(t, kubeClient, denyAllPolicy)
+	<-time.After(time.Second * 2)
+
+	// Attempt to create an egress Service within the default namespace, the above policy should
+	// reject it.
+	egressService := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "egress-to-proxy-group",
+			Namespace: metav1.NamespaceDefault,
+			Annotations: map[string]string{
+				"tailscale.com/tailnet-fqdn": "test.something.ts.net",
+				"tailscale.com/proxy-group":  "test",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Ports: []corev1.ServicePort{
+				{
+					Port:     8080,
+					Protocol: corev1.ProtocolTCP,
+					Name:     "http",
+				},
+			},
+		},
+	}
+
+	err := createAndCleanupErr(t, kubeClient, egressService)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"):
+	case err != nil:
+		t.Fatalf("expected forbidden error, got: %v", err)
+	default:
+		t.Fatal("expected error when creating egress service")
+	}
+
+	// Attempt to create an ingress Service within the default namespace, the above policy should
+	// reject it.
+	ingressService := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ingress-to-proxy-group",
+			Namespace: metav1.NamespaceDefault,
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+			Ports: []corev1.ServicePort{
+				{
+					Port:     8080,
+					Protocol: corev1.ProtocolTCP,
+					Name:     "http",
+				},
+			},
+		},
+	}
+
+	err = createAndCleanupErr(t, kubeClient, ingressService)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"):
+	case err != nil:
+		t.Fatalf("expected forbidden error, got: %v", err)
+	default:
+		t.Fatal("expected error when creating ingress service")
+	}
+
+	// Attempt to create an Ingress within the default namespace, the above policy should reject it
+	ingress := &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ingress-to-proxy-group",
+			Namespace: metav1.NamespaceDefault,
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "nginx",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 80,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{
+					Hosts: []string{"nginx"},
+				},
+			},
+		},
+	}
+
+	err = createAndCleanupErr(t, kubeClient, ingress)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"):
+	case err != nil:
+		t.Fatalf("expected forbidden error, got: %v", err)
+	default:
+		t.Fatal("expected error when creating ingress")
+	}
+
+	// Add policy to allow ingress/egress using the "test" proxy-group. This should be merged with the deny-all
+	// policy so they do not conflict.
+	allowTestPolicy := &tsapi.ProxyGroupPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "allow-test",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: tsapi.ProxyGroupPolicySpec{
+			Ingress: []string{"test"},
+			Egress:  []string{"test"},
+		},
+	}
+
+	createAndCleanup(t, kubeClient, allowTestPolicy)
+	<-time.After(time.Second * 2)
+
+	// With this policy in place, the above ingress/egress resources should be allowed to be created.
+	createAndCleanup(t, kubeClient, egressService)
+	createAndCleanup(t, kubeClient, ingressService)
+	createAndCleanup(t, kubeClient, ingress)
+}

cmd/k8s-operator/e2e/setup.go

@@ -52,6 +52,7 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster"
 	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
 	"sigs.k8s.io/kind/pkg/cmd"
+
 	"tailscale.com/internal/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"

cmd/k8s-operator/e2e/ssh.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	tailscaleroot "tailscale.com"
 	"tailscale.com/types/ptr"
 )