cmd/k8s-operator,k8s-operator: define ProxyGroupPolicy reconciler (#18654)

This commit implements a reconciler for the new `ProxyGroupPolicy` custom resource. When created, all `ProxyGroupPolicy` resources within the same namespace are merged into two `ValidatingAdmissionPolicy` resources, one for egress and one for ingress. These policies use CEL expressions to limit the usage of the "tailscale.com/proxy-group" annotation on `Service` and `Ingress` resources on create & update. Included here is also a new e2e test that ensures that resources that violate the policy return an error on creation, and that once the policy is changed to allow them they can be created. Closes: https://github.com/tailscale/corp/issues/36830 Signed-off-by: David Bond <davidsbond93@gmail.com>
2026-02-18 09:34:55 +00:00
parent f4aea70f7a
commit eb3d35c8b5
18 changed files with 987 additions and 24 deletions
@@ -832,6 +832,7 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/
        tailscale.com/k8s-operator/apis                              from tailscale.com/k8s-operator/apis/v1alpha1
        tailscale.com/k8s-operator/apis/v1alpha1                     from tailscale.com/cmd/k8s-operator+
        tailscale.com/k8s-operator/reconciler                        from tailscale.com/k8s-operator/reconciler/tailnet
+        tailscale.com/k8s-operator/reconciler/proxygrouppolicy       from tailscale.com/cmd/k8s-operator
        tailscale.com/k8s-operator/reconciler/tailnet                from tailscale.com/cmd/k8s-operator
        tailscale.com/k8s-operator/sessionrecording                  from tailscale.com/k8s-operator/api-proxy
        tailscale.com/k8s-operator/sessionrecording/spdy             from tailscale.com/k8s-operator/sessionrecording
@@ -9,3 +9,4 @@
 /proxygroup.yaml
 /recorder.yaml
 /tailnet.yaml 
+/proxygrouppolicy.yaml
@@ -40,6 +40,9 @@ rules:
 - apiGroups: ["tailscale.com"]
  resources: ["tailnets", "tailnets/status"]
  verbs: ["get", "list", "watch", "update"]
+- apiGroups: ["tailscale.com"]
+  resources: ["proxygrouppolicies", "proxygrouppolicies/status"]
+  verbs: ["get", "list", "watch", "update"]
 - apiGroups: ["tailscale.com"]
  resources: ["recorders", "recorders/status"]
  verbs: ["get", "list", "watch", "update"]
@@ -47,6 +50,9 @@ rules:
  resources: ["customresourcedefinitions"]
  verbs: ["get", "list", "watch"]
  resourceNames: ["servicemonitors.monitoring.coreos.com"]
+- apiGroups: ["admissionregistration.k8s.io"]
+  resources: ["validatingadmissionpolicies", "validatingadmissionpolicybindings"]
+  verbs: ["list", "create", "delete", "update", "get", "watch"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -19,10 +19,6 @@ spec:
        - jsonPath: .metadata.creationTimestamp
          name: Age
          type: date
-        - description: Status of the deployed ProxyGroupPolicy resources.
-          jsonPath: .status.conditions[?(@.type == "ProxyGroupPolicyReady")].reason
-          name: Status
-          type: string
      name: v1alpha1
      schema:
        openAPIV3Schema:
@@ -3290,6 +3290,142 @@ spec:
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
+metadata:
+    annotations:
+        controller-gen.kubebuilder.io/version: v0.17.0
+    name: proxygrouppolicies.tailscale.com
+spec:
+    group: tailscale.com
+    names:
+        kind: ProxyGroupPolicy
+        listKind: ProxyGroupPolicyList
+        plural: proxygrouppolicies
+        shortNames:
+            - pgp
+        singular: proxygrouppolicy
+    scope: Namespaced
+    versions:
+        - additionalPrinterColumns:
+            - jsonPath: .metadata.creationTimestamp
+              name: Age
+              type: date
+          name: v1alpha1
+          schema:
+            openAPIV3Schema:
+                properties:
+                    apiVersion:
+                        description: |-
+                            APIVersion defines the versioned schema of this representation of an object.
+                            Servers should convert recognized schemas to the latest internal value, and
+                            may reject unrecognized values.
+                            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+                        type: string
+                    kind:
+                        description: |-
+                            Kind is a string value representing the REST resource this object represents.
+                            Servers may infer this from the endpoint the client submits requests to.
+                            Cannot be updated.
+                            In CamelCase.
+                            More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+                        type: string
+                    metadata:
+                        type: object
+                    spec:
+                        description: |-
+                            Spec describes the desired state of the ProxyGroupPolicy.
+                            More info:
+                            https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
+                        properties:
+                            egress:
+                                description: |-
+                                    Names of ProxyGroup resources that can be used by Service resources within this namespace. An empty list
+                                    denotes that no egress via ProxyGroups is allowed within this namespace.
+                                items:
+                                    type: string
+                                type: array
+                            ingress:
+                                description: |-
+                                    Names of ProxyGroup resources that can be used by Ingress resources within this namespace. An empty list
+                                    denotes that no ingress via ProxyGroups is allowed within this namespace.
+                                items:
+                                    type: string
+                                type: array
+                        type: object
+                    status:
+                        description: |-
+                            Status describes the status of the ProxyGroupPolicy. This is set
+                            and managed by the Tailscale operator.
+                        properties:
+                            conditions:
+                                items:
+                                    description: Condition contains details for one aspect of the current state of this API Resource.
+                                    properties:
+                                        lastTransitionTime:
+                                            description: |-
+                                                lastTransitionTime is the last time the condition transitioned from one status to another.
+                                                This should be when the underlying condition changed.  If that is not known, then using the time when the API field changed is acceptable.
+                                            format: date-time
+                                            type: string
+                                        message:
+                                            description: |-
+                                                message is a human readable message indicating details about the transition.
+                                                This may be an empty string.
+                                            maxLength: 32768
+                                            type: string
+                                        observedGeneration:
+                                            description: |-
+                                                observedGeneration represents the .metadata.generation that the condition was set based upon.
+                                                For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date
+                                                with respect to the current state of the instance.
+                                            format: int64
+                                            minimum: 0
+                                            type: integer
+                                        reason:
+                                            description: |-
+                                                reason contains a programmatic identifier indicating the reason for the condition's last transition.
+                                                Producers of specific condition types may define expected values and meanings for this field,
+                                                and whether the values are considered a guaranteed API.
+                                                The value should be a CamelCase string.
+                                                This field may not be empty.
+                                            maxLength: 1024
+                                            minLength: 1
+                                            pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                            type: string
+                                        status:
+                                            description: status of the condition, one of True, False, Unknown.
+                                            enum:
+                                                - "True"
+                                                - "False"
+                                                - Unknown
+                                            type: string
+                                        type:
+                                            description: type of condition in CamelCase or in foo.example.com/CamelCase.
+                                            maxLength: 316
+                                            pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                            type: string
+                                    required:
+                                        - lastTransitionTime
+                                        - message
+                                        - reason
+                                        - status
+                                        - type
+                                    type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                    - type
+                                x-kubernetes-list-type: map
+                        type: object
+                required:
+                    - metadata
+                    - spec
+                type: object
+          served: true
+          storage: true
+          subresources:
+            status: {}
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
 metadata:
    annotations:
        controller-gen.kubebuilder.io/version: v0.17.0
@@ -5318,6 +5454,16 @@ rules:
        - list
        - watch
        - update
+    - apiGroups:
+        - tailscale.com
+      resources:
+        - proxygrouppolicies
+        - proxygrouppolicies/status
+      verbs:
+        - get
+        - list
+        - watch
+        - update
    - apiGroups:
        - tailscale.com
      resources:
@@ -5338,6 +5484,18 @@ rules:
        - get
        - list
        - watch
+    - apiGroups:
+        - admissionregistration.k8s.io
+      resources:
+        - validatingadmissionpolicies
+        - validatingadmissionpolicybindings
+      verbs:
+        - list
+        - create
+        - delete
+        - update
+        - get
+        - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -24,5 +24,5 @@
 //
 // * go
 // * container runtime with the docker daemon API available
-// * devcontrol: ./tool/go run ./cmd/devcontrol --generate-test-devices=k8s-operator-e2e --scenario-output-dir=/tmp/k8s-operator-e2e --test-dns=http://localhost:8055
+// * devcontrol: ./tool/go run --tags=tailscale_saas ./cmd/devcontrol --generate-test-devices=k8s-operator-e2e --scenario-output-dir=/tmp/k8s-operator-e2e --test-dns=http://localhost:8055
 package e2e
@@ -14,6 +14,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+
 	kube "tailscale.com/k8s-operator"
 	"tailscale.com/tstest"
 	"tailscale.com/types/ptr"
@@ -54,12 +54,29 @@ func createAndCleanup(t *testing.T, cl client.Client, obj client.Object) {
 	t.Cleanup(func() {
 		// Use context.Background() for cleanup, as t.Context() is cancelled
 		// just before cleanup functions are called.
-		if err := cl.Delete(context.Background(), obj); err != nil {
+		if err = cl.Delete(context.Background(), obj); err != nil {
 			t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
 		}
 	})
 }

+func createAndCleanupErr(t *testing.T, cl client.Client, obj client.Object) error {
+	t.Helper()
+
+	err := cl.Create(t.Context(), obj)
+	if err != nil {
+		return err
+	}
+
+	t.Cleanup(func() {
+		if err = cl.Delete(context.Background(), obj); err != nil {
+			t.Errorf("error cleaning up %s %s/%s: %s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName(), err)
+		}
+	})
+
+	return nil
+}
+
 func get(ctx context.Context, cl client.Client, obj client.Object) error {
 	return cl.Get(ctx, client.ObjectKeyFromObject(obj), obj)
 }
@@ -12,6 +12,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"tailscale.com/types/ptr"
 )

@@ -16,6 +16,7 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	"tailscale.com/ipn"
 	"tailscale.com/tstest"
 )
@@ -0,0 +1,161 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package e2e
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/types/ptr"
+)
+
+// See [TestMain] for test requirements.
+func TestProxyGroupPolicy(t *testing.T) {
+	if tnClient == nil {
+		t.Skip("TestProxyGroupPolicy requires a working tailnet client")
+	}
+
+	// Apply deny-all policy
+	denyAllPolicy := &tsapi.ProxyGroupPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "deny-all",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: tsapi.ProxyGroupPolicySpec{
+			Ingress: []string{},
+			Egress:  []string{},
+		},
+	}
+
+	createAndCleanup(t, kubeClient, denyAllPolicy)
+	<-time.After(time.Second * 2)
+
+	// Attempt to create an egress Service within the default namespace, the above policy should
+	// reject it.
+	egressService := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "egress-to-proxy-group",
+			Namespace: metav1.NamespaceDefault,
+			Annotations: map[string]string{
+				"tailscale.com/tailnet-fqdn": "test.something.ts.net",
+				"tailscale.com/proxy-group":  "test",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			ExternalName: "placeholder",
+			Type:         corev1.ServiceTypeExternalName,
+			Ports: []corev1.ServicePort{
+				{
+					Port:     8080,
+					Protocol: corev1.ProtocolTCP,
+					Name:     "http",
+				},
+			},
+		},
+	}
+
+	err := createAndCleanupErr(t, kubeClient, egressService)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"):
+	case err != nil:
+		t.Fatalf("expected forbidden error, got: %v", err)
+	default:
+		t.Fatal("expected error when creating egress service")
+	}
+
+	// Attempt to create an ingress Service within the default namespace, the above policy should
+	// reject it.
+	ingressService := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ingress-to-proxy-group",
+			Namespace: metav1.NamespaceDefault,
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test",
+			},
+		},
+		Spec: corev1.ServiceSpec{
+			Type:              corev1.ServiceTypeLoadBalancer,
+			LoadBalancerClass: ptr.To("tailscale"),
+			Ports: []corev1.ServicePort{
+				{
+					Port:     8080,
+					Protocol: corev1.ProtocolTCP,
+					Name:     "http",
+				},
+			},
+		},
+	}
+
+	err = createAndCleanupErr(t, kubeClient, ingressService)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"):
+	case err != nil:
+		t.Fatalf("expected forbidden error, got: %v", err)
+	default:
+		t.Fatal("expected error when creating ingress service")
+	}
+
+	// Attempt to create an Ingress within the default namespace, the above policy should reject it
+	ingress := &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ingress-to-proxy-group",
+			Namespace: metav1.NamespaceDefault,
+			Annotations: map[string]string{
+				"tailscale.com/proxy-group": "test",
+			},
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ptr.To("tailscale"),
+			DefaultBackend: &networkingv1.IngressBackend{
+				Service: &networkingv1.IngressServiceBackend{
+					Name: "nginx",
+					Port: networkingv1.ServiceBackendPort{
+						Number: 80,
+					},
+				},
+			},
+			TLS: []networkingv1.IngressTLS{
+				{
+					Hosts: []string{"nginx"},
+				},
+			},
+		},
+	}
+
+	err = createAndCleanupErr(t, kubeClient, ingress)
+	switch {
+	case err != nil && strings.Contains(err.Error(), "ValidatingAdmissionPolicy"):
+	case err != nil:
+		t.Fatalf("expected forbidden error, got: %v", err)
+	default:
+		t.Fatal("expected error when creating ingress")
+	}
+
+	// Add policy to allow ingress/egress using the "test" proxy-group. This should be merged with the deny-all
+	// policy so they do not conflict.
+	allowTestPolicy := &tsapi.ProxyGroupPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "allow-test",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: tsapi.ProxyGroupPolicySpec{
+			Ingress: []string{"test"},
+			Egress:  []string{"test"},
+		},
+	}
+
+	createAndCleanup(t, kubeClient, allowTestPolicy)
+	<-time.After(time.Second * 2)
+
+	// With this policy in place, the above ingress/egress resources should be allowed to be created.
+	createAndCleanup(t, kubeClient, egressService)
+	createAndCleanup(t, kubeClient, ingressService)
+	createAndCleanup(t, kubeClient, ingress)
+}
@@ -52,6 +52,7 @@ import (
 	"sigs.k8s.io/kind/pkg/cluster"
 	"sigs.k8s.io/kind/pkg/cluster/nodeutils"
 	"sigs.k8s.io/kind/pkg/cmd"
+
 	"tailscale.com/internal/client/tailscale"
 	"tailscale.com/ipn"
 	"tailscale.com/ipn/store/mem"
@@ -24,6 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/client-go/rest"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
 	tailscaleroot "tailscale.com"
 	"tailscale.com/types/ptr"
 )
@@ -20,20 +20,22 @@ import (
 )

 const (
-	operatorDeploymentFilesPath   = "cmd/k8s-operator/deploy"
-	connectorCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
-	proxyClassCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
-	dnsConfigCRDPath              = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
-	recorderCRDPath               = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
-	proxyGroupCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
-	tailnetCRDPath                = operatorDeploymentFilesPath + "/crds/tailscale.com_tailnets.yaml"
-	helmTemplatesPath             = operatorDeploymentFilesPath + "/chart/templates"
-	connectorCRDHelmTemplatePath  = helmTemplatesPath + "/connector.yaml"
-	proxyClassCRDHelmTemplatePath = helmTemplatesPath + "/proxyclass.yaml"
-	dnsConfigCRDHelmTemplatePath  = helmTemplatesPath + "/dnsconfig.yaml"
-	recorderCRDHelmTemplatePath   = helmTemplatesPath + "/recorder.yaml"
-	proxyGroupCRDHelmTemplatePath = helmTemplatesPath + "/proxygroup.yaml"
-	tailnetCRDHelmTemplatePath    = helmTemplatesPath + "/tailnet.yaml"
+	operatorDeploymentFilesPath         = "cmd/k8s-operator/deploy"
+	connectorCRDPath                    = operatorDeploymentFilesPath + "/crds/tailscale.com_connectors.yaml"
+	proxyClassCRDPath                   = operatorDeploymentFilesPath + "/crds/tailscale.com_proxyclasses.yaml"
+	dnsConfigCRDPath                    = operatorDeploymentFilesPath + "/crds/tailscale.com_dnsconfigs.yaml"
+	recorderCRDPath                     = operatorDeploymentFilesPath + "/crds/tailscale.com_recorders.yaml"
+	proxyGroupCRDPath                   = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygroups.yaml"
+	tailnetCRDPath                      = operatorDeploymentFilesPath + "/crds/tailscale.com_tailnets.yaml"
+	proxyGroupPolicyCRDPath             = operatorDeploymentFilesPath + "/crds/tailscale.com_proxygrouppolicies.yaml"
+	helmTemplatesPath                   = operatorDeploymentFilesPath + "/chart/templates"
+	connectorCRDHelmTemplatePath        = helmTemplatesPath + "/connector.yaml"
+	proxyClassCRDHelmTemplatePath       = helmTemplatesPath + "/proxyclass.yaml"
+	dnsConfigCRDHelmTemplatePath        = helmTemplatesPath + "/dnsconfig.yaml"
+	recorderCRDHelmTemplatePath         = helmTemplatesPath + "/recorder.yaml"
+	proxyGroupCRDHelmTemplatePath       = helmTemplatesPath + "/proxygroup.yaml"
+	tailnetCRDHelmTemplatePath          = helmTemplatesPath + "/tailnet.yaml"
+	proxyGroupPolicyCRDHelmTemplatePath = helmTemplatesPath + "/proxygrouppolicy.yaml"

 	helmConditionalStart = "{{ if .Values.installCRDs -}}\n"
 	helmConditionalEnd   = "{{- end -}}"
@@ -157,6 +159,7 @@ func generate(baseDir string) error {
 		{recorderCRDPath, recorderCRDHelmTemplatePath},
 		{proxyGroupCRDPath, proxyGroupCRDHelmTemplatePath},
 		{tailnetCRDPath, tailnetCRDHelmTemplatePath},
+		{proxyGroupPolicyCRDPath, proxyGroupPolicyCRDHelmTemplatePath},
 	} {
 		if err := addCRDToHelm(crd.crdPath, crd.templatePath); err != nil {
 			return fmt.Errorf("error adding %s CRD to Helm templates: %w", crd.crdPath, err)
@@ -173,6 +176,8 @@ func cleanup(baseDir string) error {
 		dnsConfigCRDHelmTemplatePath,
 		recorderCRDHelmTemplatePath,
 		proxyGroupCRDHelmTemplatePath,
+		tailnetCRDHelmTemplatePath,
+		proxyGroupPolicyCRDHelmTemplatePath,
 	} {
 		if err := os.Remove(filepath.Join(baseDir, path)); err != nil && !os.IsNotExist(err) {
 			return fmt.Errorf("error cleaning up %s: %w", path, err)
@@ -54,6 +54,7 @@ import (
 	"tailscale.com/ipn/store/kubestore"
 	apiproxy "tailscale.com/k8s-operator/api-proxy"
 	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
+	"tailscale.com/k8s-operator/reconciler/proxygrouppolicy"
 	"tailscale.com/k8s-operator/reconciler/tailnet"
 	"tailscale.com/kube/kubetypes"
 	"tailscale.com/tsnet"
@@ -337,6 +338,14 @@ func runReconcilers(opts reconcilerOpts) {
 		startlog.Fatalf("could not register tailnet reconciler: %v", err)
 	}

+	proxyGroupPolicyOptions := proxygrouppolicy.ReconcilerOptions{
+		Client: mgr.GetClient(),
+	}
+
+	if err = proxygrouppolicy.NewReconciler(proxyGroupPolicyOptions).Register(mgr); err != nil {
+		startlog.Fatalf("could not register proxygrouppolicy reconciler: %v", err)
+	}
+
 	svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler)
 	svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc"))
 	// If a ProxyClass changes, enqueue all Services labeled with that