From be0e460a203880aa9943893fa632ccc68491c746 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 20 Jul 2026 16:37:15 +0100 Subject: [PATCH] cmd/k8s-operator,k8s-operator: Kubernetes Peer Relays (#20495) This commit contains the Kubernetes implementation of peer relays via the new `PeerRelay` CRD. It's a mega branch consisting of the commits of other PRs gone into this work: 1. https://github.com/tailscale/tailscale/pull/20211 2. https://github.com/tailscale/tailscale/pull/20329 3. https://github.com/tailscale/tailscale/pull/20423 4. https://github.com/tailscale/tailscale/pull/20503 An instance of the `PeerRelay` CRD deploys a `StatefulSet` of containerboot instances configured to advertise themselves as peer relays using the IP addresses configured via `LoadBalancer` services on each cloud provider (with some AWS specifics as it's less automatic than its competing cloud providers). Per replica, a `LoadBalancer` type `Service` resource is provisioned and its IP address is used to configure the respective relay. This has been tested with success in AWS, GCP & Azure and provides additional modification to `Service` resources via the CRD for any other kinds of deployment environments. It also contains some work that may appear to be duplication of what already exists within `cmd/k8s-operator` so we can start building an appropriate migration path for `Connector`, `ProxyGroup` etc into respective `k8s-operator/reconciler/*` packages. Closes https://github.com/tailscale/corp/issues/34524 --- cmd/k8s-operator/depaware.txt | 4 +- .../deploy/chart/templates/operator-rbac.yaml | 3 + .../deploy/crds/tailscale.com_peerrelays.yaml | 264 +++ .../deploy/manifests/operator.yaml | 10 + cmd/k8s-operator/operator.go | 14 + k8s-operator/api.md | 152 ++ k8s-operator/apis/v1alpha1/register.go | 2 + k8s-operator/apis/v1alpha1/types_peerrelay.go | 168 ++ .../apis/v1alpha1/zz_generated.deepcopy.go | 193 +++ k8s-operator/conditions.go | 8 + k8s-operator/reconciler/peerrelay/authkey.go | 18 + k8s-operator/reconciler/peerrelay/device.go | 66 + .../reconciler/peerrelay/peerrelay.go | 572 +++++++ .../reconciler/peerrelay/peerrelay_test.go | 1510 +++++++++++++++++ k8s-operator/reconciler/peerrelay/service.go | 189 +++ .../reconciler/peerrelay/statefulset.go | 101 ++ k8s-operator/reconciler/reconciler.go | 56 + k8s-operator/reconciler/reconciler_test.go | 99 ++ k8s-operator/reconciler/tailscaled/authkey.go | 56 + .../reconciler/tailscaled/proxyclass.go | 115 ++ .../reconciler/tailscaled/statefulset.go | 223 +++ kube/kubetypes/types.go | 1 + 22 files changed, 3823 insertions(+), 1 deletion(-) create mode 100644 cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml create mode 100644 k8s-operator/apis/v1alpha1/types_peerrelay.go create mode 100644 k8s-operator/reconciler/peerrelay/authkey.go create mode 100644 k8s-operator/reconciler/peerrelay/device.go create mode 100644 k8s-operator/reconciler/peerrelay/peerrelay.go create mode 100644 k8s-operator/reconciler/peerrelay/peerrelay_test.go create mode 100644 k8s-operator/reconciler/peerrelay/service.go create mode 100644 k8s-operator/reconciler/peerrelay/statefulset.go create mode 100644 k8s-operator/reconciler/tailscaled/authkey.go create mode 100644 k8s-operator/reconciler/tailscaled/proxyclass.go create mode 100644 k8s-operator/reconciler/tailscaled/statefulset.go diff --git a/cmd/k8s-operator/depaware.txt b/cmd/k8s-operator/depaware.txt index c1dd97db0..652cce2f0 100644 --- a/cmd/k8s-operator/depaware.txt +++ b/cmd/k8s-operator/depaware.txt @@ -761,9 +761,11 @@ tailscale.com/cmd/k8s-operator dependencies: (generated by github.com/tailscale/ tailscale.com/k8s-operator/api-proxy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/apis from tailscale.com/k8s-operator/apis/v1alpha1 tailscale.com/k8s-operator/apis/v1alpha1 from tailscale.com/cmd/k8s-operator+ - tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet + tailscale.com/k8s-operator/reconciler from tailscale.com/k8s-operator/reconciler/tailnet+ + tailscale.com/k8s-operator/reconciler/peerrelay from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/proxygrouppolicy from tailscale.com/cmd/k8s-operator tailscale.com/k8s-operator/reconciler/tailnet from tailscale.com/cmd/k8s-operator + tailscale.com/k8s-operator/reconciler/tailscaled from tailscale.com/k8s-operator/reconciler/peerrelay tailscale.com/k8s-operator/sessionrecording from tailscale.com/k8s-operator/api-proxy tailscale.com/k8s-operator/sessionrecording/spdy from tailscale.com/k8s-operator/sessionrecording tailscale.com/k8s-operator/sessionrecording/tsrecorder from tailscale.com/k8s-operator/sessionrecording+ diff --git a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml index 08dea80a5..847109c40 100644 --- a/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml +++ b/cmd/k8s-operator/deploy/chart/templates/operator-rbac.yaml @@ -40,6 +40,9 @@ rules: - apiGroups: ["tailscale.com"] resources: ["tailnets", "tailnets/status"] verbs: ["get", "list", "watch", "update"] +- apiGroups: ["tailscale.com"] + resources: ["peerrelays", "peerrelays/status"] + verbs: ["get", "list", "watch", "update"] - apiGroups: ["tailscale.com"] resources: ["proxygrouppolicies", "proxygrouppolicies/status"] verbs: ["get", "list", "watch", "update"] diff --git a/cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml b/cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml new file mode 100644 index 000000000..7b8247b7d --- /dev/null +++ b/cmd/k8s-operator/deploy/crds/tailscale.com_peerrelays.yaml @@ -0,0 +1,264 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: peerrelays.tailscale.com +spec: + group: tailscale.com + names: + kind: PeerRelay + listKind: PeerRelayList + plural: peerrelays + shortNames: + - pr + singular: peerrelay + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: Status of the deployed PeerRelay resources. + jsonPath: .status.conditions[?(@.type == "PeerRelayReady")].reason + name: Status + type: string + - description: Public addresses the peer relay replicas are reachable on. + jsonPath: .status.endpoints[*].address + name: Endpoints + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + type: object + required: + - metadata + - spec + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + Spec describes the desired state of the PeerRelay. + More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + type: object + properties: + aws: + description: |- + AWS contains configuration for pinning each replica to a specific AWS Elastic IP and subnet. Only meaningful + when running on EKS with the AWS Load Balancer Controller. When set, the per-replica values override any + aws-load-balancer-eip-allocations or aws-load-balancer-subnets values supplied via spec.service.annotations. + type: object + required: + - elasticIPs + properties: + elasticIPs: + description: |- + ElasticIPs pins each replica to a specific AWS EIP allocation and subnet. Only meaningful when Network Load + Balancers are provisioned by the AWS Load Balancer Controller. ElasticIPs supplies one allocation-subnet pair + per replica: replica N uses ElasticIPs[N]. The list must be at least as long as spec.replicas so every replica + has a distinct EIP; extra entries are permitted so that scale-up doesn't immediately trip validation. + + When set, the reconciler stamps + service.beta.kubernetes.io/aws-load-balancer-eip-allocations and + service.beta.kubernetes.io/aws-load-balancer-subnets on each per-replica Service, overriding any values in + spec.service.annotations. + type: array + minItems: 1 + items: + description: PeerRelayAWSElasticIP pairs an EIP allocation with the subnet in the same AZ. + type: object + required: + - allocationID + - subnetID + properties: + allocationID: + description: |- + AllocationID is the AWS EIP allocation ID (e.g. eipalloc-0123abcd) whose public IP this replica is reachable + on. Stamped as service.beta.kubernetes.io/aws-load-balancer-eip-allocations on the replica's Service. + type: string + pattern: ^eipalloc-[0-9a-f]+$ + subnetID: + description: |- + SubnetID is the AWS subnet in the same availability zone as AllocationID (e.g. subnet-0123abcd). Stamped as + service.beta.kubernetes.io/aws-load-balancer-subnets on the replica's Service so the NLB is provisioned in + the same AZ as the EIP. + type: string + pattern: ^subnet-[0-9a-f]+$ + x-kubernetes-list-type: atomic + hostnamePrefix: + description: |- + HostnamePrefix specifies the hostname prefix for each + replica. Each device will have the integer number + from its StatefulSet pod appended to this prefix to form the full hostname. + HostnamePrefix can contain lower case letters, numbers and dashes, it + must not start with a dash and must be between 1 and 62 characters long. + type: string + pattern: ^[a-z0-9][a-z0-9-]{0,61}$ + proxyClass: + description: |- + ProxyClass is the name of the ProxyClass custom resource that + contains configuration options that should be applied to the + resources created for this PeerRelay. If unset, the operator will + create resources with the default configuration. + type: string + replicas: + description: |- + Replicas specifies how many devices to create. Set this to enable + high availability for peer relays. + https://tailscale.com/kb/1115/high-availability. Defaults to 1. + type: integer + format: int32 + default: 1 + minimum: 0 + service: + description: Service contains configuration values to modify the LoadBalancer service used to expose the peer relay. + type: object + properties: + annotations: + description: |- + Annotations to apply to the LoadBalancer service. Any annotations that conflict with those used by known + cloud providers to ensure IP addresses rather than DNS names are ignored. + type: object + additionalProperties: + type: string + tags: + description: |- + Tags that the Tailscale node will be tagged with. + Defaults to [tag:k8s]. + To autoapprove the device defined by a PeerRelay, + you can configure Tailscale ACLs to give these tags the necessary + permissions. + See https://tailscale.com/kb/1337/acl-syntax#autoapprovers. + If you specify custom tags here, you must also make the operator an owner of these tags. + See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + Tags cannot be changed once a PeerRelay node has been created. + Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + type: array + items: + type: string + pattern: ^tag:[a-zA-Z][a-zA-Z0-9-]*$ + tailnet: + description: |- + Tailnet specifies the tailnet this PeerRelay should join. If blank, the default tailnet is used. When set, this + name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + type: string + x-kubernetes-validations: + - rule: self == oldSelf + message: PeerRelay tailnet is immutable + x-kubernetes-validations: + - rule: '!has(self.aws) || !has(self.aws.elasticIPs) || self.aws.elasticIPs.size() >= self.replicas' + message: spec.aws.elasticIPs must contain at least one entry per replica + status: + description: |- + Status describes the status of the PeerRelay. This is set + and managed by the Tailscale operator. + type: object + properties: + conditions: + type: array + items: + description: Condition contains details for one aspect of the current state of this API Resource. + type: object + required: + - lastTransitionTime + - message + - reason + - status + - type + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + type: string + format: date-time + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + type: string + maxLength: 32768 + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + type: integer + format: int64 + minimum: 0 + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + type: string + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + status: + description: status of the condition, one of True, False, Unknown. + type: string + enum: + - "True" + - "False" + - Unknown + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + endpoints: + description: |- + Endpoints lists the public address:port pairs each peer relay replica is reachable on. There is one entry + per replica whose LoadBalancer Service has been assigned a public address; entries appear as the underlying + cloud provisions each Service. + type: array + items: + type: object + required: + - address + - port + - replica + properties: + address: + description: |- + Address is the public IP or hostname the cloud has allocated for this replica's LoadBalancer Service. + Peers reach this relay by connecting to Address:Port over UDP. + type: string + port: + description: Port is the UDP port the peer relay listens on. + type: integer + format: int32 + replica: + description: Replica is the zero-based index of the peer relay replica this endpoint targets. + type: integer + format: int32 + x-kubernetes-list-map-keys: + - replica + x-kubernetes-list-type: map + served: true + storage: true + subresources: + status: {} diff --git a/cmd/k8s-operator/deploy/manifests/operator.yaml b/cmd/k8s-operator/deploy/manifests/operator.yaml index 8068923d2..32d9f0896 100644 --- a/cmd/k8s-operator/deploy/manifests/operator.yaml +++ b/cmd/k8s-operator/deploy/manifests/operator.yaml @@ -6335,6 +6335,16 @@ rules: - list - watch - update + - apiGroups: + - tailscale.com + resources: + - peerrelays + - peerrelays/status + verbs: + - get + - list + - watch + - update - apiGroups: - tailscale.com resources: diff --git a/cmd/k8s-operator/operator.go b/cmd/k8s-operator/operator.go index af44dd4f7..377651810 100644 --- a/cmd/k8s-operator/operator.go +++ b/cmd/k8s-operator/operator.go @@ -55,6 +55,7 @@ import ( "tailscale.com/ipn/store/kubestore" apiproxy "tailscale.com/k8s-operator/api-proxy" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/peerrelay" "tailscale.com/k8s-operator/reconciler/proxygrouppolicy" "tailscale.com/k8s-operator/reconciler/tailnet" "tailscale.com/k8s-operator/tsclient" @@ -369,6 +370,19 @@ func runReconcilers(opts reconcilerOpts) { startlog.Fatalf("could not register proxygrouppolicy reconciler: %v", err) } + peerRelayOptions := peerrelay.ReconcilerOptions{ + Client: mgr.GetClient(), + TailscaleNamespace: opts.tailscaleNamespace, + ProxyImage: opts.proxyImage, + DefaultTags: strings.Split(opts.proxyTags, ","), + Clients: clients, + Logger: opts.log, + } + + if err = peerrelay.NewReconciler(peerRelayOptions).Register(mgr); err != nil { + startlog.Fatalf("could not register peerrelay reconciler: %v", err) + } + svcFilter := handler.EnqueueRequestsFromMapFunc(serviceHandler) svcChildFilter := handler.EnqueueRequestsFromMapFunc(managedResourceHandlerForType("svc")) // If a ProxyClass changes, enqueue all Services labeled with that diff --git a/k8s-operator/api.md b/k8s-operator/api.md index a32daec93..d4dc6f864 100644 --- a/k8s-operator/api.md +++ b/k8s-operator/api.md @@ -12,6 +12,8 @@ - [ConnectorList](#connectorlist) - [DNSConfig](#dnsconfig) - [DNSConfigList](#dnsconfiglist) +- [PeerRelay](#peerrelay) +- [PeerRelayList](#peerrelaylist) - [ProxyClass](#proxyclass) - [ProxyClassList](#proxyclasslist) - [ProxyGroup](#proxygroup) @@ -349,6 +351,7 @@ _Validation:_ _Appears in:_ - [ConnectorSpec](#connectorspec) +- [PeerRelaySpec](#peerrelayspec) - [ProxyGroupSpec](#proxygroupspec) @@ -536,6 +539,154 @@ _Appears in:_ | `selector` _object (keys:string, values:string)_ | A selector which will be used to select the node's that will have their `ExternalIP`'s advertised
by the ProxyGroup as Static Endpoints. | | | +#### PeerRelay + + + + + + + +_Appears in:_ +- [PeerRelayList](#peerrelaylist) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | | +| `kind` _string_ | `PeerRelay` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `spec` _[PeerRelaySpec](#peerrelayspec)_ | Spec describes the desired state of the PeerRelay.
More info:
https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status | | | +| `status` _[PeerRelayStatus](#peerrelaystatus)_ | Status describes the status of the PeerRelay. This is set
and managed by the Tailscale operator. | | | + + +#### PeerRelayAWS + + + +PeerRelayAWS contains AWS-specific configuration for a PeerRelay. + + + +_Appears in:_ +- [PeerRelaySpec](#peerrelayspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `elasticIPs` _[PeerRelayAWSElasticIP](#peerrelayawselasticip) array_ | ElasticIPs pins each replica to a specific AWS EIP allocation and subnet. Only meaningful when Network Load
Balancers are provisioned by the AWS Load Balancer Controller. ElasticIPs supplies one allocation-subnet pair
per replica: replica N uses ElasticIPs[N]. The list must be at least as long as spec.replicas so every replica
has a distinct EIP; extra entries are permitted so that scale-up doesn't immediately trip validation.
When set, the reconciler stamps
service.beta.kubernetes.io/aws-load-balancer-eip-allocations and
service.beta.kubernetes.io/aws-load-balancer-subnets on each per-replica Service, overriding any values in
spec.service.annotations. | | MinItems: 1
| + + +#### PeerRelayAWSElasticIP + + + +PeerRelayAWSElasticIP pairs an EIP allocation with the subnet in the same AZ. + + + +_Appears in:_ +- [PeerRelayAWS](#peerrelayaws) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `allocationID` _string_ | AllocationID is the AWS EIP allocation ID (e.g. eipalloc-0123abcd) whose public IP this replica is reachable
on. Stamped as service.beta.kubernetes.io/aws-load-balancer-eip-allocations on the replica's Service. | | Pattern: `^eipalloc-[0-9a-f]+$`
| +| `subnetID` _string_ | SubnetID is the AWS subnet in the same availability zone as AllocationID (e.g. subnet-0123abcd). Stamped as
service.beta.kubernetes.io/aws-load-balancer-subnets on the replica's Service so the NLB is provisioned in
the same AZ as the EIP. | | Pattern: `^subnet-[0-9a-f]+$`
| + + +#### PeerRelayEndpoint + + + + + + + +_Appears in:_ +- [PeerRelayStatus](#peerrelaystatus) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `replica` _integer_ | Replica is the zero-based index of the peer relay replica this endpoint targets. | | | +| `address` _string_ | Address is the public IP or hostname the cloud has allocated for this replica's LoadBalancer Service.
Peers reach this relay by connecting to Address:Port over UDP. | | | +| `port` _integer_ | Port is the UDP port the peer relay listens on. | | | + + +#### PeerRelayList + + + + + + + + + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `apiVersion` _string_ | `tailscale.com/v1alpha1` | | | +| `kind` _string_ | `PeerRelayList` | | | +| `kind` _string_ | Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds | | | +| `apiVersion` _string_ | APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources | | | +| `metadata` _[ListMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#listmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | +| `items` _[PeerRelay](#peerrelay) array_ | | | | + + +#### PeerRelayService + + + + + + + +_Appears in:_ +- [PeerRelaySpec](#peerrelayspec) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `annotations` _object (keys:string, values:string)_ | Annotations to apply to the LoadBalancer service. Any annotations that conflict with those used by known
cloud providers to ensure IP addresses rather than DNS names are ignored. | | | + + +#### PeerRelaySpec + + + + + + + +_Appears in:_ +- [PeerRelay](#peerrelay) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `tags` _[Tags](#tags)_ | Tags that the Tailscale node will be tagged with.
Defaults to [tag:k8s].
To autoapprove the device defined by a PeerRelay,
you can configure Tailscale ACLs to give these tags the necessary
permissions.
See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
If you specify custom tags here, you must also make the operator an owner of these tags.
See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
Tags cannot be changed once a PeerRelay node has been created.
Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. | | Pattern: `^tag:[a-zA-Z][a-zA-Z0-9-]*$`
Type: string
| +| `hostnamePrefix` _[HostnamePrefix](#hostnameprefix)_ | HostnamePrefix specifies the hostname prefix for each
replica. Each device will have the integer number
from its StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long. | | Pattern: `^[a-z0-9][a-z0-9-]{0,61}$`
Type: string
| +| `proxyClass` _string_ | ProxyClass is the name of the ProxyClass custom resource that
contains configuration options that should be applied to the
resources created for this PeerRelay. If unset, the operator will
create resources with the default configuration. | | | +| `replicas` _integer_ | Replicas specifies how many devices to create. Set this to enable
high availability for peer relays.
https://tailscale.com/kb/1115/high-availability. Defaults to 1. | 1 | Minimum: 0
| +| `tailnet` _string_ | Tailnet specifies the tailnet this PeerRelay should join. If blank, the default tailnet is used. When set, this
name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. | | | +| `service` _[PeerRelayService](#peerrelayservice)_ | Service contains configuration values to modify the LoadBalancer service used to expose the peer relay. | | | +| `aws` _[PeerRelayAWS](#peerrelayaws)_ | AWS contains configuration for pinning each replica to a specific AWS Elastic IP and subnet. Only meaningful
when running on EKS with the AWS Load Balancer Controller. When set, the per-replica values override any
aws-load-balancer-eip-allocations or aws-load-balancer-subnets values supplied via spec.service.annotations. | | | + + +#### PeerRelayStatus + + + + + + + +_Appears in:_ +- [PeerRelay](#peerrelay) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.3/#condition-v1-meta) array_ | | | | +| `endpoints` _[PeerRelayEndpoint](#peerrelayendpoint) array_ | Endpoints lists the public address:port pairs each peer relay replica is reachable on. There is one entry
per replica whose LoadBalancer Service has been assigned a public address; entries appear as the underlying
cloud provisions each Service. | | | + + #### Pod @@ -1233,6 +1384,7 @@ _Validation:_ _Appears in:_ - [ConnectorSpec](#connectorspec) +- [PeerRelaySpec](#peerrelayspec) - [ProxyGroupSpec](#proxygroupspec) - [RecorderSpec](#recorderspec) diff --git a/k8s-operator/apis/v1alpha1/register.go b/k8s-operator/apis/v1alpha1/register.go index 125d74198..367631631 100644 --- a/k8s-operator/apis/v1alpha1/register.go +++ b/k8s-operator/apis/v1alpha1/register.go @@ -71,6 +71,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &TailnetList{}, &ProxyGroupPolicy{}, &ProxyGroupPolicyList{}, + &PeerRelay{}, + &PeerRelayList{}, ) metav1.AddToGroupVersion(scheme, SchemeGroupVersion) diff --git a/k8s-operator/apis/v1alpha1/types_peerrelay.go b/k8s-operator/apis/v1alpha1/types_peerrelay.go new file mode 100644 index 000000000..251e21663 --- /dev/null +++ b/k8s-operator/apis/v1alpha1/types_peerrelay.go @@ -0,0 +1,168 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package v1alpha1 + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Code comments on these types should be treated as user facing documentation- +// they will appear on the PeerRelay CRD i.e. if someone runs kubectl explain peerrelay. + +var PeerRelayKind = "PeerRelay" + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster,shortName=pr +// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp" +// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "PeerRelayReady")].reason`,description="Status of the deployed PeerRelay resources." +// +kubebuilder:printcolumn:name="Endpoints",type="string",JSONPath=`.status.endpoints[*].address`,description="Public addresses the peer relay replicas are reachable on." + +type PeerRelay struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitzero"` + + // Spec describes the desired state of the PeerRelay. + // More info: + // https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status + Spec PeerRelaySpec `json:"spec"` + + // Status describes the status of the PeerRelay. This is set + // and managed by the Tailscale operator. + // +optional + Status PeerRelayStatus `json:"status"` +} + +// +kubebuilder:object:root=true + +type PeerRelayList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata"` + + Items []PeerRelay `json:"items"` +} + +// +kubebuilder:validation:XValidation:rule="!has(self.aws) || !has(self.aws.elasticIPs) || self.aws.elasticIPs.size() >= self.replicas",message="spec.aws.elasticIPs must contain at least one entry per replica" +type PeerRelaySpec struct { + // Tags that the Tailscale node will be tagged with. + // Defaults to [tag:k8s]. + // To autoapprove the device defined by a PeerRelay, + // you can configure Tailscale ACLs to give these tags the necessary + // permissions. + // See https://tailscale.com/kb/1337/acl-syntax#autoapprovers. + // If you specify custom tags here, you must also make the operator an owner of these tags. + // See https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator. + // Tags cannot be changed once a PeerRelay node has been created. + // Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$. + // +optional + Tags Tags `json:"tags,omitempty"` + + // HostnamePrefix specifies the hostname prefix for each + // replica. Each device will have the integer number + // from its StatefulSet pod appended to this prefix to form the full hostname. + // HostnamePrefix can contain lower case letters, numbers and dashes, it + // must not start with a dash and must be between 1 and 62 characters long. + // +optional + HostnamePrefix HostnamePrefix `json:"hostnamePrefix,omitzero"` + + // ProxyClass is the name of the ProxyClass custom resource that + // contains configuration options that should be applied to the + // resources created for this PeerRelay. If unset, the operator will + // create resources with the default configuration. + // +optional + ProxyClass string `json:"proxyClass,omitempty"` + + // Replicas specifies how many devices to create. Set this to enable + // high availability for peer relays. + // https://tailscale.com/kb/1115/high-availability. Defaults to 1. + // +optional + // +kubebuilder:validation:Minimum=0 + // +kubebuilder:default=1 + Replicas *int32 `json:"replicas,omitzero"` + + // Tailnet specifies the tailnet this PeerRelay should join. If blank, the default tailnet is used. When set, this + // name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set. + // +optional + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PeerRelay tailnet is immutable" + Tailnet string `json:"tailnet,omitempty"` + + // Service contains configuration values to modify the LoadBalancer service used to expose the peer relay. + // +optional + Service *PeerRelayService `json:"service,omitzero"` + + // AWS contains configuration for pinning each replica to a specific AWS Elastic IP and subnet. Only meaningful + // when running on EKS with the AWS Load Balancer Controller. When set, the per-replica values override any + // aws-load-balancer-eip-allocations or aws-load-balancer-subnets values supplied via spec.service.annotations. + // +optional + AWS *PeerRelayAWS `json:"aws,omitzero"` +} + +type PeerRelayService struct { + // Annotations to apply to the LoadBalancer service. Any annotations that conflict with those used by known + // cloud providers to ensure IP addresses rather than DNS names are ignored. + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// PeerRelayAWS contains AWS-specific configuration for a PeerRelay. +type PeerRelayAWS struct { + // ElasticIPs pins each replica to a specific AWS EIP allocation and subnet. Only meaningful when Network Load + // Balancers are provisioned by the AWS Load Balancer Controller. ElasticIPs supplies one allocation-subnet pair + // per replica: replica N uses ElasticIPs[N]. The list must be at least as long as spec.replicas so every replica + // has a distinct EIP; extra entries are permitted so that scale-up doesn't immediately trip validation. + // + // When set, the reconciler stamps + // service.beta.kubernetes.io/aws-load-balancer-eip-allocations and + // service.beta.kubernetes.io/aws-load-balancer-subnets on each per-replica Service, overriding any values in + // spec.service.annotations. + // +listType=atomic + // +kubebuilder:validation:MinItems=1 + ElasticIPs []PeerRelayAWSElasticIP `json:"elasticIPs"` +} + +// PeerRelayAWSElasticIP pairs an EIP allocation with the subnet in the same AZ. +type PeerRelayAWSElasticIP struct { + // AllocationID is the AWS EIP allocation ID (e.g. eipalloc-0123abcd) whose public IP this replica is reachable + // on. Stamped as service.beta.kubernetes.io/aws-load-balancer-eip-allocations on the replica's Service. + // +kubebuilder:validation:Pattern=`^eipalloc-[0-9a-f]+$` + AllocationID string `json:"allocationID"` + + // SubnetID is the AWS subnet in the same availability zone as AllocationID (e.g. subnet-0123abcd). Stamped as + // service.beta.kubernetes.io/aws-load-balancer-subnets on the replica's Service so the NLB is provisioned in + // the same AZ as the EIP. + // +kubebuilder:validation:Pattern=`^subnet-[0-9a-f]+$` + SubnetID string `json:"subnetID"` +} + +type PeerRelayStatus struct { + // +listType=map + // +listMapKey=type + // +optional + Conditions []metav1.Condition `json:"conditions"` + + // Endpoints lists the public address:port pairs each peer relay replica is reachable on. There is one entry + // per replica whose LoadBalancer Service has been assigned a public address; entries appear as the underlying + // cloud provisions each Service. + // +listType=map + // +listMapKey=replica + // +optional + Endpoints []PeerRelayEndpoint `json:"endpoints,omitempty"` +} + +type PeerRelayEndpoint struct { + // Replica is the zero-based index of the peer relay replica this endpoint targets. + Replica int32 `json:"replica"` + + // Address is the public IP or hostname the cloud has allocated for this replica's LoadBalancer Service. + // Peers reach this relay by connecting to Address:Port over UDP. + Address string `json:"address"` + + // Port is the UDP port the peer relay listens on. + Port int32 `json:"port"` +} + +// PeerRelayReady is set to True if the PeerRelay is available for use by operator workloads. +const PeerRelayReady ConditionType = `PeerRelayReady` diff --git a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go index b401c6d87..4f4c1ee57 100644 --- a/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go +++ b/k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go @@ -550,6 +550,199 @@ func (in *NodePortConfig) DeepCopy() *NodePortConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelay) DeepCopyInto(out *PeerRelay) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelay. +func (in *PeerRelay) DeepCopy() *PeerRelay { + if in == nil { + return nil + } + out := new(PeerRelay) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PeerRelay) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelayAWS) DeepCopyInto(out *PeerRelayAWS) { + *out = *in + if in.ElasticIPs != nil { + in, out := &in.ElasticIPs, &out.ElasticIPs + *out = make([]PeerRelayAWSElasticIP, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayAWS. +func (in *PeerRelayAWS) DeepCopy() *PeerRelayAWS { + if in == nil { + return nil + } + out := new(PeerRelayAWS) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelayAWSElasticIP) DeepCopyInto(out *PeerRelayAWSElasticIP) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayAWSElasticIP. +func (in *PeerRelayAWSElasticIP) DeepCopy() *PeerRelayAWSElasticIP { + if in == nil { + return nil + } + out := new(PeerRelayAWSElasticIP) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelayEndpoint) DeepCopyInto(out *PeerRelayEndpoint) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayEndpoint. +func (in *PeerRelayEndpoint) DeepCopy() *PeerRelayEndpoint { + if in == nil { + return nil + } + out := new(PeerRelayEndpoint) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelayList) DeepCopyInto(out *PeerRelayList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]PeerRelay, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayList. +func (in *PeerRelayList) DeepCopy() *PeerRelayList { + if in == nil { + return nil + } + out := new(PeerRelayList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *PeerRelayList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelayService) DeepCopyInto(out *PeerRelayService) { + *out = *in + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayService. +func (in *PeerRelayService) DeepCopy() *PeerRelayService { + if in == nil { + return nil + } + out := new(PeerRelayService) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelaySpec) DeepCopyInto(out *PeerRelaySpec) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(Tags, len(*in)) + copy(*out, *in) + } + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = new(PeerRelayService) + (*in).DeepCopyInto(*out) + } + if in.AWS != nil { + in, out := &in.AWS, &out.AWS + *out = new(PeerRelayAWS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelaySpec. +func (in *PeerRelaySpec) DeepCopy() *PeerRelaySpec { + if in == nil { + return nil + } + out := new(PeerRelaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PeerRelayStatus) DeepCopyInto(out *PeerRelayStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Endpoints != nil { + in, out := &in.Endpoints, &out.Endpoints + *out = make([]PeerRelayEndpoint, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayStatus. +func (in *PeerRelayStatus) DeepCopy() *PeerRelayStatus { + if in == nil { + return nil + } + out := new(PeerRelayStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Pod) DeepCopyInto(out *Pod) { *out = *in diff --git a/k8s-operator/conditions.go b/k8s-operator/conditions.go index 89b83dd5f..15fef5049 100644 --- a/k8s-operator/conditions.go +++ b/k8s-operator/conditions.go @@ -100,6 +100,14 @@ func SetTailnetCondition(tn *tsapi.Tailnet, conditionType tsapi.ConditionType, s tn.Status.Conditions = conds } +// SetPeerRelayCondition ensures that PeerRelay status has a condition with the +// given attributes. LastTransitionTime gets set every time condition's status +// changes. +func SetPeerRelayCondition(pr *tsapi.PeerRelay, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, clock tstime.Clock, logger *zap.SugaredLogger) { + conds := updateCondition(pr.Status.Conditions, conditionType, status, reason, message, pr.Generation, clock, logger) + pr.Status.Conditions = conds +} + func updateCondition(conds []metav1.Condition, conditionType tsapi.ConditionType, status metav1.ConditionStatus, reason, message string, gen int64, clock tstime.Clock, logger *zap.SugaredLogger) []metav1.Condition { newCondition := metav1.Condition{ Type: string(conditionType), diff --git a/k8s-operator/reconciler/peerrelay/authkey.go b/k8s-operator/reconciler/peerrelay/authkey.go new file mode 100644 index 000000000..ca0270d69 --- /dev/null +++ b/k8s-operator/reconciler/peerrelay/authkey.go @@ -0,0 +1,18 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package peerrelay + +import ( + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" +) + +func (r *Reconciler) peerRelayTags(pr *tsapi.PeerRelay) []string { + tags := pr.Spec.Tags.Stringify() + if len(tags) == 0 { + return r.defaultTags + } + return tags +} diff --git a/k8s-operator/reconciler/peerrelay/device.go b/k8s-operator/reconciler/peerrelay/device.go new file mode 100644 index 000000000..81061717d --- /dev/null +++ b/k8s-operator/reconciler/peerrelay/device.go @@ -0,0 +1,66 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package peerrelay + +import ( + "context" + "errors" + "fmt" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/tailscaled" + "tailscale.com/kube/kubetypes" +) + +func (r *Reconciler) deleteDevicesFrom(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, fromIdx int32) error { + if r.tsClients == nil { + return nil + } + + tsc, err := r.tsClients.For(pr.Spec.Tailnet) + if err != nil { + return fmt.Errorf("failed to resolve Tailscale API client for tailnet %q: %w", pr.Spec.Tailnet, err) + } + + labels := peerRelayLabels(pr.Name) + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeState + + var list corev1.SecretList + if err = r.List(ctx, &list, client.InNamespace(r.tailscaleNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("failed to list state Secrets: %w", err) + } + + var errs []error + for i := range list.Items { + s := &list.Items[i] + idx, ok := replicaIndexFromLabels(s.Labels) + if !ok || idx < fromIdx { + continue + } + + if deviceID := tailscaled.DeviceIDFromStateSecret(s); deviceID != "" { + logger.Debugf("deleting tailnet device %q", deviceID) + if err = tsc.Devices().Delete(ctx, deviceID); err != nil && !tailscaleclient.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed to delete tailnet device %q: %w", deviceID, err)) + continue + } + } + + logger.Debugf("deleting state Secret %q", s.Name) + if err = r.Delete(ctx, s); err != nil && !apierrors.IsNotFound(err) { + errs = append(errs, fmt.Errorf("failed to delete state Secret %q: %w", s.Name, err)) + } + } + + return errors.Join(errs...) +} diff --git a/k8s-operator/reconciler/peerrelay/peerrelay.go b/k8s-operator/reconciler/peerrelay/peerrelay.go new file mode 100644 index 000000000..a20b52304 --- /dev/null +++ b/k8s-operator/reconciler/peerrelay/peerrelay.go @@ -0,0 +1,572 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package peerrelay provides reconciliation logic for the PeerRelay custom resource definition. It is responsible +// for managing the lifecycle of PeerRelay devices, including the StatefulSet and Service resources used to expose +// them. +package peerrelay + +import ( + "cmp" + "context" + "errors" + "fmt" + "net" + "net/netip" + "reflect" + "slices" + "sync" + "time" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + operatorutils "tailscale.com/k8s-operator" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/tailscaled" + "tailscale.com/kube/kubetypes" + "tailscale.com/tstime" + "tailscale.com/util/clientmetric" + "tailscale.com/util/set" +) + +type ( + // The Reconciler type is a reconcile.TypedReconciler implementation used to manage the reconciliation of + // PeerRelay custom resources. + Reconciler struct { + client.Client + + tailscaleNamespace string + proxyImage string + defaultTags []string + tsClients tailscaled.ClientProvider + resolver func(ctx context.Context, network, host string) ([]netip.Addr, error) + logger *zap.SugaredLogger + clock tstime.Clock + + // Metrics related fields + mu sync.Mutex + peerRelays set.Slice[types.UID] + } + + // The ReconcilerOptions type contains configuration values for the Reconciler. + ReconcilerOptions struct { + // The client for interacting with the Kubernetes API. + Client client.Client + // The namespace the operator is installed in. PeerRelay-managed resources (Services, StatefulSets, etc.) + // are created within this namespace. + TailscaleNamespace string + // ProxyImage is the container image used for the tailscaled pods that back each peer relay replica. + ProxyImage string + // DefaultTags is the tag list applied to freshly minted auth keys when a PeerRelay hasn't set its own + // spec.tags. Must be non-empty at construction time. + DefaultTags []string + // Clients resolves the Tailscale API client for a given tailnet name. Used to mint auth keys for each + // replica. Blank tailnet returns the operator's default client. + Clients tailscaled.ClientProvider + // Resolver is used to convert LoadBalancer Service hostnames to concrete IPs when the cloud + // controller doesn't populate Ingress[].IP directly (e.g. AWS NLBs). Defaults to a resolver backed by + // net.DefaultResolver when unset. + Resolver func(ctx context.Context, network string, host string) ([]netip.Addr, error) + // The logger to use for this Reconciler. + Logger *zap.SugaredLogger + // Clock is used to stamp condition transitions. Defaults to a real clock when unset. + Clock tstime.Clock + } +) + +const ( + reconcilerName = "peerrelay-reconciler" + fieldOwner client.FieldOwner = "peerrelay-reconciler" +) + +// Constants for condition reasons. +const ( + ReasonEndpointsPending = "EndpointsPending" + ReasonPodsPending = "PodsPending" + ReasonAWSConfigInvalid = "AWSConfigInvalid" + ReasonTailnetUnavailable = "TailnetUnavailable" + ReasonReady = "PeerRelayReady" +) + +var ( + // gaugePeerRelayResources tracks the overall number of PeerRelay resources currently managed by this operator + // instance. + gaugePeerRelayResources = clientmetric.NewGauge(kubetypes.MetricPeerRelayCount) +) + +// NewReconciler returns a new instance of the Reconciler type. It watches specifically for changes to PeerRelay +// custom resources. The ReconcilerOptions can be used to modify the behaviour of the Reconciler. +func NewReconciler(options ReconcilerOptions) *Reconciler { + clock := options.Clock + if clock == nil { + clock = tstime.DefaultClock{} + } + + resolver := options.Resolver + if resolver == nil { + resolver = net.DefaultResolver.LookupNetIP + } + + return &Reconciler{ + Client: options.Client, + tailscaleNamespace: options.TailscaleNamespace, + proxyImage: options.ProxyImage, + defaultTags: options.DefaultTags, + tsClients: options.Clients, + resolver: resolver, + logger: options.Logger.Named(reconcilerName), + clock: clock, + } +} + +// Register the Reconciler onto the given manager.Manager implementation. It watches PeerRelay resources directly, +// the child resources it manages (Services, StatefulSets, Secrets) so external drift or cloud controller updates +// enqueue a reconcile for the owning PeerRelay, and ProxyClass so config changes propagate to referring +// PeerRelays. +func (r *Reconciler) Register(mgr manager.Manager) error { + enqueue := handler.EnqueueRequestsFromMapFunc(reconciler.EnqueueForChild(parentTypePeerRelay)) + return builder. + ControllerManagedBy(mgr). + For(&tsapi.PeerRelay{}). + Watches(&corev1.Service{}, enqueue). + Watches(&appsv1.StatefulSet{}, enqueue). + Watches(&corev1.Secret{}, enqueue). + Watches(&tsapi.ProxyClass{}, handler.EnqueueRequestsFromMapFunc(r.enqueuePeerRelaysForProxyClass)). + Named(reconcilerName). + Complete(r) +} + +func (r *Reconciler) enqueuePeerRelaysForProxyClass(ctx context.Context, o client.Object) []reconcile.Request { + pc, ok := o.(*tsapi.ProxyClass) + if !ok { + return nil + } + + var list tsapi.PeerRelayList + if err := r.List(ctx, &list); err != nil { + r.logger.Errorf("failed to list PeerRelays for ProxyClass %q change: %v", pc.Name, err) + return nil + } + + var reqs []reconcile.Request + for _, pr := range list.Items { + if pr.Spec.ProxyClass == pc.Name { + reqs = append(reqs, reconcile.Request{NamespacedName: types.NamespacedName{Name: pr.Name}}) + } + } + return reqs +} + +// Reconcile is invoked when a change occurs to PeerRelay resources within the cluster. On create/update, it ensures +// one LoadBalancer Service exists per replica. On delete, all managed Services are removed before the finalizer is +// released. +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + logger := r.logger.With("PeerRelay", req.Name) + logger.Debug("starting reconcile") + defer logger.Debug("reconcile finished") + + var pr tsapi.PeerRelay + err := r.Get(ctx, req.NamespacedName, &pr) + switch { + case apierrors.IsNotFound(err): + logger.Debug("PeerRelay not found, assuming it was deleted") + return reconcile.Result{}, nil + case err != nil: + return reconcile.Result{}, fmt.Errorf("failed to get PeerRelay %q: %w", req.NamespacedName, err) + } + + if r.tsClients != nil { + if _, err = r.tsClients.For(pr.Spec.Tailnet); err != nil { + return r.reportTailnetUnavailable(ctx, logger, &pr, err) + } + } + + if !pr.DeletionTimestamp.IsZero() { + return r.delete(ctx, logger, &pr) + } + + return r.createOrUpdate(ctx, logger, &pr) +} + +func (r *Reconciler) reportTailnetUnavailable(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, tsErr error) (reconcile.Result, error) { + operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonTailnetUnavailable, tsErr.Error(), r.clock, logger) + if err := r.Status().Update(ctx, pr); err != nil { + return reconcile.Result{}, errors.Join(tsErr, fmt.Errorf("failed to update PeerRelay status: %w", err)) + } + + return reconcile.Result{}, tsErr +} + +func (r *Reconciler) createOrUpdate(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay) (reconcile.Result, error) { + if !slices.Contains(pr.Finalizers, reconciler.FinalizerName) { + reconciler.SetFinalizer(pr) + if err := r.Update(ctx, pr); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to add finalizer to PeerRelay %q: %w", pr.Name, err) + } + } + + r.mu.Lock() + if !r.peerRelays.Contains(pr.UID) { + r.peerRelays.Add(pr.UID) + logger.Infof("now managing PeerRelay %q", pr.Name) + } + r.mu.Unlock() + gaugePeerRelayResources.Set(int64(r.peerRelays.Len())) + + replicas := int32(1) + if pr.Spec.Replicas != nil { + replicas = *pr.Spec.Replicas + } + + // Belt-and-braces: CEL on the CRD enforces this at admission, but we also validate here to guard against older + // clusters without CEL, resources created before the CRD schema landed, or hand-edited status paths. If the user + // hasn't supplied enough EIPs for the requested replica count we refuse to touch existing state and surface the + // condition so they can fix the spec. + if pr.Spec.AWS != nil && int32(len(pr.Spec.AWS.ElasticIPs)) < replicas { + message := fmt.Sprintf("spec.aws.elasticIPs has %d entries but spec.replicas is %d", len(pr.Spec.AWS.ElasticIPs), replicas) + operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonAWSConfigInvalid, message, r.clock, logger) + if err := r.Status().Update(ctx, pr); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update PeerRelay status for %q: %w", pr.Name, err) + } + return reconcile.Result{}, nil + } + + for i := int32(0); i < replicas; i++ { + desired := r.peerRelayService(pr, i) + if err := r.ensureService(ctx, logger, desired); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to apply Service %q: %w", desired.Name, err) + } + } + + // Read the LB addresses assigned by the cloud so each pod's config file can advertise its own public endpoint + // via RelayServerStaticEndpoints. On first reconcile the LBs aren't provisioned yet , endpointsByReplica ends + // up empty and the configs are written without static endpoints; the Watches-triggered reconcile that fires + // when the LB IP lands will fill them in. + endpoints, err := r.readEndpoints(ctx, logger, pr) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to read endpoints for PeerRelay %q: %w", pr.Name, err) + } + + endpointsByReplica := make(map[int32]tsapi.PeerRelayEndpoint, len(endpoints)) + for _, ep := range endpoints { + endpointsByReplica[ep.Replica] = ep + } + + for i := int32(0); i < replicas; i++ { + var endpoint *tsapi.PeerRelayEndpoint + if ep, ok := endpointsByReplica[i]; ok { + endpoint = &ep + } + + if err = r.ensureStateSecret(ctx, logger, pr, i); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to apply state Secret for PeerRelay %q replica %d: %w", pr.Name, i, err) + } + + if err = r.ensureConfigSecret(ctx, logger, pr, i, endpoint); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to apply config Secret for PeerRelay %q replica %d: %w", pr.Name, i, err) + } + } + + ss, err := r.ensureStatefulSet(ctx, logger, pr, replicas) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to apply StatefulSet for PeerRelay %q: %w", pr.Name, err) + } + + if err = r.deleteDevicesFrom(ctx, logger, pr, replicas); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to clean up scaled-down tailnet devices for PeerRelay %q: %w", pr.Name, err) + } + + if err = r.deleteServicesFrom(ctx, logger, pr, replicas); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to clean up scaled-down Services for PeerRelay %q: %w", pr.Name, err) + } + + if err = r.deleteConfigSecretsFrom(ctx, logger, pr, replicas); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to clean up scaled-down config Secrets for PeerRelay %q: %w", pr.Name, err) + } + + if err = r.writeStatus(ctx, logger, pr, endpoints, replicas, ss); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to update PeerRelay status for %q: %w", pr.Name, err) + } + + if !peerRelayReady(pr) { + return reconcile.Result{RequeueAfter: 30 * time.Second}, nil + } + + return reconcile.Result{}, nil +} + +func peerRelayReady(pr *tsapi.PeerRelay) bool { + for _, c := range pr.Status.Conditions { + if c.Type == string(tsapi.PeerRelayReady) { + return c.Status == metav1.ConditionTrue + } + } + + return false +} + +func (r *Reconciler) readEndpoints(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay) ([]tsapi.PeerRelayEndpoint, error) { + var list corev1.ServiceList + if err := r.List(ctx, &list, client.InNamespace(r.tailscaleNamespace), client.MatchingLabels(peerRelayLabels(pr.Name))); err != nil { + return nil, fmt.Errorf("failed to list Services: %w", err) + } + + prevByReplica := make(map[int32]tsapi.PeerRelayEndpoint, len(pr.Status.Endpoints)) + for _, ep := range pr.Status.Endpoints { + prevByReplica[ep.Replica] = ep + } + + var endpoints []tsapi.PeerRelayEndpoint + for i := range list.Items { + svc := &list.Items[i] + var prev *tsapi.PeerRelayEndpoint + if idx, ok := replicaIndexFromLabels(svc.Labels); ok { + if ep, ok := prevByReplica[idx]; ok { + prev = &ep + } + } + + if endpoint := r.peerRelayEndpoint(ctx, logger, svc, prev); endpoint != nil { + endpoints = append(endpoints, *endpoint) + } + } + + slices.SortFunc(endpoints, func(a, b tsapi.PeerRelayEndpoint) int { + return cmp.Compare(a.Replica, b.Replica) + }) + + return endpoints, nil +} + +func (r *Reconciler) writeStatus(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, endpoints []tsapi.PeerRelayEndpoint, replicas int32, ss *appsv1.StatefulSet) error { + prevStatus := pr.Status.DeepCopy() + + pr.Status.Endpoints = endpoints + + var readyReplicas int32 + if ss != nil { + readyReplicas = ss.Status.ReadyReplicas + } + + switch { + case int32(len(endpoints)) < replicas: + message := fmt.Sprintf("%d of %d replicas have a public IP", len(endpoints), replicas) + operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonEndpointsPending, message, r.clock, logger) + case readyReplicas < replicas: + message := fmt.Sprintf("%d of %d pods are ready", readyReplicas, replicas) + operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionFalse, ReasonPodsPending, message, r.clock, logger) + default: + operatorutils.SetPeerRelayCondition(pr, tsapi.PeerRelayReady, metav1.ConditionTrue, ReasonReady, ReasonReady, r.clock, logger) + } + + if reflect.DeepEqual(prevStatus, &pr.Status) { + return nil + } + + if err := r.Status().Update(ctx, pr); err != nil { + return fmt.Errorf("failed to update PeerRelay status: %w", err) + } + + return nil +} + +func (r *Reconciler) delete(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay) (reconcile.Result, error) { + logger.Infof("deleting PeerRelay %q", pr.Name) + + if err := r.deleteDevicesFrom(ctx, logger, pr, 0); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to delete tailnet devices for PeerRelay %q: %w", pr.Name, err) + } + + if err := r.deleteStatefulSet(ctx, logger, pr); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to delete StatefulSet for PeerRelay %q: %w", pr.Name, err) + } + + if err := r.deleteConfigSecretsFrom(ctx, logger, pr, 0); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to delete config Secrets for PeerRelay %q: %w", pr.Name, err) + } + + if err := r.deleteServicesFrom(ctx, logger, pr, 0); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to delete Services for PeerRelay %q: %w", pr.Name, err) + } + + reconciler.RemoveFinalizer(pr) + if err := r.Update(ctx, pr); err != nil { + return reconcile.Result{}, fmt.Errorf("failed to remove finalizer from PeerRelay %q: %w", pr.Name, err) + } + + r.mu.Lock() + r.peerRelays.Remove(pr.UID) + r.mu.Unlock() + gaugePeerRelayResources.Set(int64(r.peerRelays.Len())) + + return reconcile.Result{}, nil +} + +func (r *Reconciler) ensureService(ctx context.Context, logger *zap.SugaredLogger, desired *corev1.Service) error { + logger.Debugf("applying Service %q", desired.Name) + if err := r.Patch(ctx, desired, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply Service: %w", err) + } + return nil +} + +func (r *Reconciler) deleteServicesFrom(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, fromIdx int32) error { + var list corev1.ServiceList + if err := r.List(ctx, &list, client.InNamespace(r.tailscaleNamespace), client.MatchingLabels(peerRelayLabels(pr.Name))); err != nil { + return fmt.Errorf("failed to list Services: %w", err) + } + + for i := range list.Items { + svc := &list.Items[i] + idx, ok := replicaIndexFromLabels(svc.Labels) + if !ok || idx < fromIdx { + continue + } + + logger.Debugf("deleting Service %q", svc.Name) + if err := r.Delete(ctx, svc); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete Service %q: %w", svc.Name, err) + } + } + + return nil +} + +func (r *Reconciler) ensureConfigSecret(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, idx int32, endpoint *tsapi.PeerRelayEndpoint) error { + authKey, err := r.reuseOrMintAuthKey(ctx, pr, idx) + if err != nil { + return err + } + + desired, err := r.peerRelayConfigSecret(pr, idx, endpoint, authKey) + if err != nil { + return fmt.Errorf("failed to build config Secret: %w", err) + } + + logger.Debugf("applying config Secret %q", desired.Name) + if err = r.Patch(ctx, desired, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply config Secret: %w", err) + } + + return nil +} + +func (r *Reconciler) reuseOrMintAuthKey(ctx context.Context, pr *tsapi.PeerRelay, idx int32) (*string, error) { + var existing corev1.Secret + err := r.Get(ctx, types.NamespacedName{Namespace: r.tailscaleNamespace, Name: configSecretName(pr.Name, idx)}, &existing) + switch { + case apierrors.IsNotFound(err): + key, err := r.mintAuthKey(ctx, pr) + if err != nil { + return nil, err + } + return &key, nil + case err != nil: + return nil, fmt.Errorf("failed to get config Secret: %w", err) + } + + if existingKey := tailscaled.AuthKeyFromConfigSecret(&existing); existingKey != nil { + return existingKey, nil + } + + key, err := r.mintAuthKey(ctx, pr) + if err != nil { + return nil, err + } + + return &key, nil +} + +func (r *Reconciler) mintAuthKey(ctx context.Context, pr *tsapi.PeerRelay) (string, error) { + client, err := r.tsClients.For(pr.Spec.Tailnet) + if err != nil { + return "", fmt.Errorf("failed to resolve Tailscale API client for tailnet %q: %w", pr.Spec.Tailnet, err) + } + + return tailscaled.NewAuthKey(ctx, client, r.peerRelayTags(pr)) +} + +func (r *Reconciler) ensureStateSecret(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, idx int32) error { + desired := tailscaled.NewStateSecret(tailscaled.StateSecretOptions{ + Name: replicaName(pr.Name, idx), + Namespace: r.tailscaleNamespace, + Labels: peerRelayServiceLabels(pr.Name, idx), + }) + + logger.Debugf("applying state Secret %q", desired.Name) + if err := r.Patch(ctx, desired, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + return fmt.Errorf("failed to apply state Secret: %w", err) + } + + return nil +} + +func (r *Reconciler) deleteConfigSecretsFrom(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, fromIdx int32) error { + labels := peerRelayLabels(pr.Name) + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeConfig + + var list corev1.SecretList + if err := r.List(ctx, &list, client.InNamespace(r.tailscaleNamespace), client.MatchingLabels(labels)); err != nil { + return fmt.Errorf("failed to list config Secrets: %w", err) + } + + for i := range list.Items { + secret := &list.Items[i] + idx, ok := replicaIndexFromLabels(secret.Labels) + if !ok || idx < fromIdx { + continue + } + + logger.Debugf("deleting config Secret %q", secret.Name) + if err := r.Delete(ctx, secret); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete config Secret %q: %w", secret.Name, err) + } + } + + return nil +} + +func (r *Reconciler) ensureStatefulSet(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay, replicas int32) (*appsv1.StatefulSet, error) { + pc, err := r.getProxyClass(ctx, pr) + if err != nil { + return nil, err + } + + desired := r.peerRelayStatefulSet(pr, replicas, pc) + + logger.Debugf("applying StatefulSet %q", desired.Name) + if err = r.Patch(ctx, desired, client.Apply, fieldOwner, client.ForceOwnership); err != nil { + return nil, fmt.Errorf("failed to apply StatefulSet: %w", err) + } + + var current appsv1.StatefulSet + if err = r.Get(ctx, types.NamespacedName{Namespace: desired.Namespace, Name: desired.Name}, ¤t); err != nil { + return nil, fmt.Errorf("failed to get StatefulSet: %w", err) + } + + return ¤t, nil +} + +func (r *Reconciler) deleteStatefulSet(ctx context.Context, logger *zap.SugaredLogger, pr *tsapi.PeerRelay) error { + ss := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: resourceName(pr.Name), Namespace: r.tailscaleNamespace}, + } + logger.Debugf("deleting StatefulSet %q", ss.Name) + if err := r.Delete(ctx, ss); err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("failed to delete StatefulSet: %w", err) + } + return nil +} diff --git a/k8s-operator/reconciler/peerrelay/peerrelay_test.go b/k8s-operator/reconciler/peerrelay/peerrelay_test.go new file mode 100644 index 000000000..c1c559722 --- /dev/null +++ b/k8s-operator/reconciler/peerrelay/peerrelay_test.go @@ -0,0 +1,1510 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package peerrelay_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "sync" + "testing" + + "go.uber.org/zap" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler/peerrelay" + "tailscale.com/k8s-operator/tsclient" +) + +const ( + tailscaleNamespace = "tailscale" + testProxyImage = "tailscale/tailscale:test" +) + +func testResolver(_ context.Context, _ string, host string) ([]netip.Addr, error) { + r := map[string][]netip.Addr{ + "test-0.elb.amazonaws.com": {netip.MustParseAddr("203.0.113.10")}, + "test-1.elb.amazonaws.com": {netip.MustParseAddr("203.0.113.11")}, + } + + if addrs, ok := r[host]; ok { + return addrs, nil + } + + return nil, &net.DNSError{Err: "no such host", Name: host, IsNotFound: true} +} + +type expectedService struct { + Name string + Type corev1.ServiceType + Port int32 + NodePort int32 + Protocol corev1.Protocol + Selector map[string]string + Labels map[string]string + Annotations map[string]string + AbsentLabels []string + AbsentAnnotations []string +} + +type statefulSetSpec struct { + Replicas int32 + Image string +} + +func TestReconciler_Reconcile(t *testing.T) { + t.Parallel() + + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + tt := []struct { + Name string + Request reconcile.Request + PeerRelay *tsapi.PeerRelay + ExistingResources []client.Object + ExpectsError bool + ExpectedServices []expectedService + ExpectedEndpoints []tsapi.PeerRelayEndpoint + ExpectedReadyStatus metav1.ConditionStatus // asserted only when non-empty + ExpectedReadyReason string // asserted only when non-empty + ExpectStatefulSetGone bool // assert the StatefulSet does not exist + ExpectStatefulSetSpec *statefulSetSpec // asserted only when non-nil + ExpectFinalizer bool + ExpectPRDeleted bool + }{ + { + Name: "ignores-unknown-peer-relay", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "missing"}}, + }, + { + Name: "default-replicas", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + Type: corev1.ServiceTypeLoadBalancer, + Port: 41641, + Protocol: corev1.ProtocolUDP, + Selector: map[string]string{"statefulset.kubernetes.io/pod-name": "peerrelay-test-0"}, + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + "tailscale.com/peer-relay-replica": "0", + }, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", + "service.beta.kubernetes.io/aws-load-balancer-ip-address-type": "ipv4", + "service.beta.kubernetes.io/azure-load-balancer-internal": "false", + }, + }, + }, + ExpectFinalizer: true, + ExpectStatefulSetSpec: &statefulSetSpec{Replicas: 1, Image: testProxyImage}, + }, + { + Name: "multiple-replicas", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(3))}, + }, + ExpectedServices: []expectedService{ + {Name: "peerrelay-test-0", Labels: map[string]string{"tailscale.com/peer-relay-replica": "0"}}, + {Name: "peerrelay-test-1", Labels: map[string]string{"tailscale.com/peer-relay-replica": "1"}}, + {Name: "peerrelay-test-2", Labels: map[string]string{"tailscale.com/peer-relay-replica": "2"}}, + }, + ExpectStatefulSetSpec: &statefulSetSpec{Replicas: 3, Image: testProxyImage}, + }, + { + Name: "zero-replicas", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(0))}, + }, + ExpectStatefulSetSpec: &statefulSetSpec{Replicas: 0, Image: testProxyImage}, + }, + { + Name: "scale-down", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, + }, + ExistingResources: []client.Object{ + managedService("test", 0), + managedService("test", 1), + managedService("test", 2), + managedService("test", 3), + }, + ExpectedServices: []expectedService{ + {Name: "peerrelay-test-0"}, + {Name: "peerrelay-test-1"}, + }, + ExpectStatefulSetSpec: &statefulSetSpec{Replicas: 2, Image: testProxyImage}, + }, + { + Name: "scale-up", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(3))}, + }, + ExistingResources: []client.Object{ + managedService("test", 0), + }, + ExpectedServices: []expectedService{ + {Name: "peerrelay-test-0"}, + {Name: "peerrelay-test-1"}, + {Name: "peerrelay-test-2"}, + }, + }, + { + Name: "scoped", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(1))}, + }, + ExistingResources: []client.Object{ + // A Service belonging to a different PeerRelay must not be touched. + managedService("other", 5), + }, + ExpectedServices: []expectedService{ + {Name: "peerrelay-other-5"}, + {Name: "peerrelay-test-0"}, + }, + }, + { + Name: "user-annotations", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Service: &tsapi.PeerRelayService{Annotations: map[string]string{"example.com/custom": "value"}}, + }, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + Annotations: map[string]string{ + "example.com/custom": "value", + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", + "service.beta.kubernetes.io/azure-load-balancer-internal": "false", + }, + }, + }, + }, + { + Name: "cloud-annotations", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Service: &tsapi.PeerRelayService{Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internal", + "service.beta.kubernetes.io/azure-load-balancer-internal": "true", + }}, + }, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", + "service.beta.kubernetes.io/azure-load-balancer-internal": "false", + }, + }, + }, + }, + { + // The reconciler applies via server-side apply, so a drifted Service (wrong Spec.Type, wrong Ports) + // is restored on reconcile. Kubernetes owns the merge with fields belonging to other managers + // (cloud LB controller annotations, kube-proxy's NodePort) and preserves them — that contract is + // exercised in e2e tests where a real API server is available; here we only verify the drift is fixed. + Name: "drift-corrected", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(1))}, + }, + ExistingResources: []client.Object{ + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "peerrelay-test-0", + Namespace: tailscaleNamespace, + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + "tailscale.com/peer-relay-replica": "0", + }, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internal", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Ports: []corev1.ServicePort{ + {Name: "wrong", Protocol: corev1.ProtocolTCP, Port: 80}, + }, + }, + }, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + Type: corev1.ServiceTypeLoadBalancer, + Port: 41641, + Protocol: corev1.ProtocolUDP, + Labels: map[string]string{ + "tailscale.com/managed": "true", + }, + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", // drift corrected + }, + }, + }, + // No LB ingress seeded, so the PeerRelayReady condition stays Pending. + ExpectedReadyStatus: metav1.ConditionFalse, + ExpectedReadyReason: peerrelay.ReasonEndpointsPending, + }, + { + // GCP/Azure-style: the LB reports a plain IPv4 address; we surface it verbatim in status.endpoints. + Name: "endpoints-populated-from-lb-ip", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "1.2.3.4", ""), + managedServiceWithLB("test", 1, "5.6.7.8", ""), + // Seed a StatefulSet with both replicas Ready so the writeStatus precedence path can reach + // ReasonReady. Without this the fake client's fresh StatefulSet would have ReadyReplicas=0 and + // we'd land in PodsPending instead. + managedStatefulSet("test", 2, 2), + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}, {Name: "peerrelay-test-1"}}, + ExpectedEndpoints: []tsapi.PeerRelayEndpoint{ + {Replica: 0, Address: "1.2.3.4", Port: 41641}, + {Replica: 1, Address: "5.6.7.8", Port: 41641}, + }, + ExpectedReadyStatus: metav1.ConditionTrue, + ExpectedReadyReason: peerrelay.ReasonReady, + }, + { + // All LB IPs assigned but pods haven't reported Ready yet + Name: "pods-pending-blocks-ready", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "1.2.3.4", ""), + managedServiceWithLB("test", 1, "5.6.7.8", ""), + managedStatefulSet("test", 2, 1), // only 1 of 2 pods Ready + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}, {Name: "peerrelay-test-1"}}, + ExpectedEndpoints: []tsapi.PeerRelayEndpoint{ + {Replica: 0, Address: "1.2.3.4", Port: 41641}, + {Replica: 1, Address: "5.6.7.8", Port: 41641}, + }, + ExpectedReadyStatus: metav1.ConditionFalse, + ExpectedReadyReason: peerrelay.ReasonPodsPending, + }, + { + // Hostname-only LBs (AWS NLBs) no longer produce a hard error, the reconciler resolves the hostname + // to a stable IP and advertises that instead. Here the resolver returns a canned address for the AWS + // hostname; the endpoint should reflect the resolved IP. The eip-allocations annotation is what + // signals to the reconciler that hostname resolution is safe — this is our AWS opt-in contract. + Name: "hostname-only-lb-is-resolved-to-ip", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Service: &tsapi.PeerRelayService{ + Annotations: map[string]string{eipAllocationsAnnotation: "eipalloc-aaaa"}, + }, + }, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "", "test-0.elb.amazonaws.com"), + managedStatefulSet("test", 1, 1), + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}}, + ExpectedEndpoints: []tsapi.PeerRelayEndpoint{ + {Replica: 0, Address: "203.0.113.10", Port: 41641}, + }, + ExpectedReadyStatus: metav1.ConditionTrue, + ExpectedReadyReason: peerrelay.ReasonReady, + }, + { + // Unresolvable hostname (e.g. NXDOMAIN during LB provisioning) stays in Pending — no hard error, + // reconciler will retry when the hostname comes online. Service still carries the eip-allocations + // annotation so we know the resolution path was attempted (and not just skipped for being non-AWS). + Name: "unresolvable-hostname-stays-pending", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Service: &tsapi.PeerRelayService{ + Annotations: map[string]string{eipAllocationsAnnotation: "eipalloc-aaaa"}, + }, + }, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "", "unresolvable.example.invalid"), + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}}, + ExpectedReadyStatus: metav1.ConditionFalse, + ExpectedReadyReason: peerrelay.ReasonEndpointsPending, + }, + { + // A hostname-only Service without the EIP annotation is a Service whose backing LB has unstable IPs + // (unmanaged AWS NLB, most third-party providers). The reconciler refuses to resolve — advertising a + // transient IP would leave peers connecting to a dead endpoint if AWS shifts the underlying A records. + // Users get an explicit "pending" state and either add the annotation or accept the LB isn't usable. + Name: "hostname-without-eip-annotation-stays-pending", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "", "test-0.elb.amazonaws.com"), + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}}, + ExpectedReadyStatus: metav1.ConditionFalse, + ExpectedReadyReason: peerrelay.ReasonEndpointsPending, + }, + { + // Mixed batch: one replica has a direct IP, another has only a hostname that resolves (with EIP + // annotation propagated from the PeerRelay spec). Both should end up in status.endpoints. + Name: "mixed-ip-and-resolved-hostname", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Replicas: new(int32(2)), + Service: &tsapi.PeerRelayService{ + Annotations: map[string]string{eipAllocationsAnnotation: "eipalloc-bbbb"}, + }, + }, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "1.2.3.4", ""), + managedServiceWithLB("test", 1, "", "test-1.elb.amazonaws.com"), + managedStatefulSet("test", 2, 2), + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}, {Name: "peerrelay-test-1"}}, + ExpectedEndpoints: []tsapi.PeerRelayEndpoint{ + {Replica: 0, Address: "1.2.3.4", Port: 41641}, + {Replica: 1, Address: "203.0.113.11", Port: 41641}, + }, + ExpectedReadyStatus: metav1.ConditionTrue, + ExpectedReadyReason: peerrelay.ReasonReady, + }, + { + // Mid-provisioning: some LBs have addresses, some don't yet. Only the ready ones show up. + Name: "endpoints-partial-when-lb-not-ready", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(3))}, + }, + ExistingResources: []client.Object{ + managedServiceWithLB("test", 0, "1.2.3.4", ""), + managedService("test", 2), + }, + ExpectedServices: []expectedService{{Name: "peerrelay-test-0"}, {Name: "peerrelay-test-1"}, {Name: "peerrelay-test-2"}}, + ExpectedEndpoints: []tsapi.PeerRelayEndpoint{ + {Replica: 0, Address: "1.2.3.4", Port: 41641}, + }, + ExpectedReadyStatus: metav1.ConditionFalse, + ExpectedReadyReason: peerrelay.ReasonEndpointsPending, + }, + { + // spec.aws.elasticIPs fans out per-replica: each Service gets its OWN eip-allocations + subnets + // annotations from the array. This is the HA path on AWS where every replica needs a distinct EIP. + Name: "aws-elasticips-fan-out-per-replica", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Replicas: new(int32(2)), + AWS: &tsapi.PeerRelayAWS{ + ElasticIPs: []tsapi.PeerRelayAWSElasticIP{ + {AllocationID: "eipalloc-aaaa", SubnetID: "subnet-aaaa"}, + {AllocationID: "eipalloc-bbbb", SubnetID: "subnet-bbbb"}, + }, + }, + }, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + Annotations: map[string]string{ + eipAllocationsAnnotation: "eipalloc-aaaa", + subnetsAnnotation: "subnet-aaaa", + }, + }, + { + Name: "peerrelay-test-1", + Annotations: map[string]string{ + eipAllocationsAnnotation: "eipalloc-bbbb", + subnetsAnnotation: "subnet-bbbb", + }, + }, + }, + ExpectStatefulSetSpec: &statefulSetSpec{Replicas: 2, Image: testProxyImage}, + }, + { + // spec.aws.elasticIPs wins over any conflicting eip-allocations / subnets in spec.service.annotations. + // Users get a single source of truth: whatever they put in per-replica config is what lands on the Service. + Name: "aws-elasticips-override-shared-annotations", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Service: &tsapi.PeerRelayService{ + Annotations: map[string]string{ + eipAllocationsAnnotation: "eipalloc-shared-wrong", + subnetsAnnotation: "subnet-shared-wrong", + }, + }, + AWS: &tsapi.PeerRelayAWS{ + ElasticIPs: []tsapi.PeerRelayAWSElasticIP{ + {AllocationID: "eipalloc-perreplica", SubnetID: "subnet-perreplica"}, + }, + }, + }, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + Annotations: map[string]string{ + eipAllocationsAnnotation: "eipalloc-perreplica", + subnetsAnnotation: "subnet-perreplica", + }, + }, + }, + }, + { + Name: "aws-elasticips-clears-eip-annotations", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + }, + ExistingResources: []client.Object{ + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "peerrelay-test-0", + Namespace: tailscaleNamespace, + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + "tailscale.com/peer-relay-replica": "0", + }, + Annotations: map[string]string{ + eipAllocationsAnnotation: "eipalloc-stale", + subnetsAnnotation: "subnet-stale", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + }, + }, + }, + ExpectedServices: []expectedService{ + { + Name: "peerrelay-test-0", + AbsentAnnotations: []string{eipAllocationsAnnotation, subnetsAnnotation}, + }, + }, + }, + { + // Length mismatch trips the belt-and-braces check: reconciler refuses to create Services and surfaces + // AWSConfigInvalid so the user can fix the spec. Nothing is created, existing state is preserved. + Name: "aws-elasticips-insufficient", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ + Replicas: new(int32(2)), + AWS: &tsapi.PeerRelayAWS{ + ElasticIPs: []tsapi.PeerRelayAWSElasticIP{ + {AllocationID: "eipalloc-aaaa", SubnetID: "subnet-aaaa"}, + }, + }, + }, + }, + ExpectStatefulSetGone: true, + ExpectedReadyStatus: metav1.ConditionFalse, + ExpectedReadyReason: peerrelay.ReasonAWSConfigInvalid, + }, + { + Name: "deletion", + Request: reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}, + PeerRelay: &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + DeletionTimestamp: new(metav1.Now()), + }, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, + }, + ExistingResources: []client.Object{ + managedService("test", 0), + managedService("test", 1), + managedService("other", 0), + managedConfigSecret("test", 0), + managedConfigSecret("test", 1), + &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Namespace: tailscaleNamespace, + }, + }, + }, + ExpectedServices: []expectedService{{Name: "peerrelay-other-0"}}, + ExpectPRDeleted: true, + ExpectStatefulSetGone: true, + }, + } + + for _, tc := range tt { + t.Run(tc.Name, func(t *testing.T) { + builder := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). + WithInterceptorFuncs(applyPatchInterceptor()) + if tc.PeerRelay != nil { + builder = builder.WithObjects(tc.PeerRelay) + } + builder = builder.WithObjects(tc.ExistingResources...) + + fc := builder.Build() + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:test-peer-relay"}, + Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + _, err = r.Reconcile(t.Context(), tc.Request) + if tc.ExpectsError && err == nil { + t.Fatalf("expected error, got none") + } + if !tc.ExpectsError && err != nil { + t.Fatalf("expected no error, got %v", err) + } + + var svcs corev1.ServiceList + if err = fc.List(t.Context(), &svcs, client.InNamespace(tailscaleNamespace)); err != nil { + t.Fatal(err) + } + + gotByName := make(map[string]corev1.Service, len(svcs.Items)) + gotNames := make([]string, 0, len(svcs.Items)) + for _, svc := range svcs.Items { + gotByName[svc.Name] = svc + gotNames = append(gotNames, svc.Name) + } + + wantNames := make([]string, 0, len(tc.ExpectedServices)) + for _, want := range tc.ExpectedServices { + wantNames = append(wantNames, want.Name) + } + + slices.Sort(gotNames) + slices.Sort(wantNames) + if !slices.Equal(gotNames, wantNames) { + t.Fatalf("expected services %v, got %v", wantNames, gotNames) + } + + for _, want := range tc.ExpectedServices { + assertService(t, want, new(gotByName[want.Name])) + } + + if tc.PeerRelay == nil { + return + } + + var pr tsapi.PeerRelay + err = fc.Get(t.Context(), types.NamespacedName{Name: tc.PeerRelay.Name}, &pr) + switch { + case tc.ExpectPRDeleted: + if !apierrors.IsNotFound(err) { + t.Fatalf("expected PeerRelay to be gone, got %v", err) + } + case err != nil: + t.Fatalf("failed to refetch PeerRelay: %v", err) + case tc.ExpectFinalizer: + if !slices.Contains(pr.Finalizers, "tailscale.com/finalizer") { + t.Errorf("expected finalizer to be set, got %v", pr.Finalizers) + } + } + + if !slices.Equal(pr.Status.Endpoints, tc.ExpectedEndpoints) { + t.Errorf("expected status.endpoints %v, got %v", tc.ExpectedEndpoints, pr.Status.Endpoints) + } + + if tc.ExpectedReadyStatus != "" || tc.ExpectedReadyReason != "" { + cond := readyCondition(&pr) + if tc.ExpectedReadyStatus != "" && cond.Status != tc.ExpectedReadyStatus { + t.Errorf("expected PeerRelayReady status %s, got %q", tc.ExpectedReadyStatus, cond.Status) + } + if tc.ExpectedReadyReason != "" && cond.Reason != tc.ExpectedReadyReason { + t.Errorf("expected PeerRelayReady reason %s, got %q", tc.ExpectedReadyReason, cond.Reason) + } + } + + assertStatefulSet(t, fc, tc.Request.Name, tc.ExpectStatefulSetSpec, tc.ExpectStatefulSetGone) + configSecrets, stateSecrets := childSecretsFromServices(tc.Request.Name, tc.ExpectedServices) + assertConfigSecrets(t, fc, tc.Request.Name, configSecrets) + assertStateSecrets(t, fc, tc.Request.Name, stateSecrets) + }) + } +} + +func assertStatefulSet(t *testing.T, fc client.Client, prName string, want *statefulSetSpec, gone bool) { + t.Helper() + + stsName := "peerrelay-" + prName + var ss appsv1.StatefulSet + err := fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: stsName}, &ss) + switch { + case gone: + if !apierrors.IsNotFound(err) { + t.Errorf("expected StatefulSet %q to be absent, got err=%v", prName, err) + } + return + case want == nil: + return + case err != nil: + t.Fatalf("expected StatefulSet %q, got err %v", prName, err) + } + + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != want.Replicas { + got := "" + if ss.Spec.Replicas != nil { + got = fmt.Sprintf("%d", *ss.Spec.Replicas) + } + t.Errorf("expected StatefulSet replicas=%d, got %s", want.Replicas, got) + } + + if want.Image != "" { + if len(ss.Spec.Template.Spec.Containers) == 0 { + t.Fatalf("StatefulSet template has no containers") + } + if ss.Spec.Template.Spec.Containers[0].Image != want.Image { + t.Errorf("expected container image %q, got %q", want.Image, ss.Spec.Template.Spec.Containers[0].Image) + } + } +} + +func assertConfigSecrets(t *testing.T, fc client.Client, prName string, want []string) { + t.Helper() + assertSecretsForType(t, fc, prName, "config", "config Secrets", want) +} + +func assertStateSecrets(t *testing.T, fc client.Client, prName string, want []string) { + t.Helper() + assertSecretsForType(t, fc, prName, "state", "state Secrets", want) +} + +// childSecretsFromServices derives the expected config and state Secret names for the PeerRelay named prName from the +// list of expected Services. Because the reconciler creates one config Secret (named -config) and one state +// Secret (named ) per Service, spelling those out separately in every test case is redundant. +func childSecretsFromServices(prName string, services []expectedService) (configs, states []string) { + prefix := "peerrelay-" + prName + "-" + for _, s := range services { + if !strings.HasPrefix(s.Name, prefix) { + continue + } + configs = append(configs, s.Name+"-config") + states = append(states, s.Name) + } + return configs, states +} + +func assertSecretsForType(t *testing.T, fc client.Client, prName, secretType, label string, want []string) { + t.Helper() + + var list corev1.SecretList + if err := fc.List(t.Context(), &list, client.InNamespace(tailscaleNamespace), client.MatchingLabels(map[string]string{ + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": prName, + "tailscale.com/secret-type": secretType, + })); err != nil { + t.Fatal(err) + } + + got := make([]string, 0, len(list.Items)) + for _, s := range list.Items { + got = append(got, s.Name) + } + + slices.Sort(got) + sortedWant := slices.Clone(want) + slices.Sort(sortedWant) + + if !slices.Equal(got, sortedWant) { + t.Errorf("expected %s %v, got %v", label, sortedWant, got) + } +} + +func readyCondition(pr *tsapi.PeerRelay) metav1.Condition { + for _, cond := range pr.Status.Conditions { + if cond.Type == string(tsapi.PeerRelayReady) { + return cond + } + } + + return metav1.Condition{} +} + +func assertService(t *testing.T, want expectedService, got *corev1.Service) { + t.Helper() + + if want.Type != "" && got.Spec.Type != want.Type { + t.Errorf("Service %q: expected type %q, got %q", want.Name, want.Type, got.Spec.Type) + } + + if want.Port != 0 || want.Protocol != "" || want.NodePort != 0 { + if len(got.Spec.Ports) != 1 { + t.Fatalf("Service %q: expected exactly one port, got %d", want.Name, len(got.Spec.Ports)) + } + if want.Protocol != "" && got.Spec.Ports[0].Protocol != want.Protocol { + t.Errorf("Service %q: expected protocol %q, got %q", want.Name, want.Protocol, got.Spec.Ports[0].Protocol) + } + if want.Port != 0 && got.Spec.Ports[0].Port != want.Port { + t.Errorf("Service %q: expected port %d, got %d", want.Name, want.Port, got.Spec.Ports[0].Port) + } + if want.NodePort != 0 && got.Spec.Ports[0].NodePort != want.NodePort { + t.Errorf("Service %q: expected nodePort %d, got %d", want.Name, want.NodePort, got.Spec.Ports[0].NodePort) + } + } + + for k, v := range want.Selector { + if gotV := got.Spec.Selector[k]; gotV != v { + t.Errorf("Service %q: expected selector %q=%q, got %q", want.Name, k, v, gotV) + } + } + + for k, v := range want.Labels { + if gotV := got.Labels[k]; gotV != v { + t.Errorf("Service %q: expected label %q=%q, got %q", want.Name, k, v, gotV) + } + } + + for _, k := range want.AbsentLabels { + if v, ok := got.Labels[k]; ok { + t.Errorf("Service %q: expected label %q to be absent, got %q", want.Name, k, v) + } + } + + for k, v := range want.Annotations { + if gotV := got.Annotations[k]; gotV != v { + t.Errorf("Service %q: expected annotation %q=%q, got %q", want.Name, k, v, gotV) + } + } + + for _, k := range want.AbsentAnnotations { + if v, ok := got.Annotations[k]; ok { + t.Errorf("Service %q: expected annotation %q to be absent, got %q", want.Name, k, v) + } + } +} + +func managedService(prName string, idx int) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("peerrelay-%s-%d", prName, idx), + Namespace: tailscaleNamespace, + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": prName, + "tailscale.com/peer-relay-replica": fmt.Sprintf("%d", idx), + }, + }, + Spec: corev1.ServiceSpec{Type: corev1.ServiceTypeLoadBalancer}, + } +} + +func managedServiceWithLB(prName string, idx int, ip, hostname string) *corev1.Service { + svc := managedService(prName, idx) + svc.Status.LoadBalancer.Ingress = []corev1.LoadBalancerIngress{{IP: ip, Hostname: hostname}} + return svc +} + +const ( + eipAllocationsAnnotation = "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" + subnetsAnnotation = "service.beta.kubernetes.io/aws-load-balancer-subnets" +) + +func managedStatefulSet(prName string, replicas, ready int32) *appsv1.StatefulSet { + labels := map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": prName, + } + stsName := "peerrelay-" + prName + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: stsName, + Namespace: tailscaleNamespace, + Labels: labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &replicas, + ServiceName: stsName, + Selector: &metav1.LabelSelector{MatchLabels: labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: labels}, + Spec: corev1.PodSpec{}, + }, + }, + Status: appsv1.StatefulSetStatus{ReadyReplicas: ready}, + } +} + +func managedConfigSecret(prName string, idx int) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("peerrelay-%s-%d-config", prName, idx), + Namespace: tailscaleNamespace, + Labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": prName, + "tailscale.com/peer-relay-replica": fmt.Sprintf("%d", idx), + }, + }, + } +} + +func TestReconciler_TailscaledConfig(t *testing.T) { + t.Parallel() + + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + pr := &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, + } + + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}). + WithObjects( + pr, + managedServiceWithLB("test", 0, "1.2.3.4", ""), + managedService("test", 1), + ). + Build() + + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:test-peer-relay"}, + Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err := r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + got0 := readTailscaledConfig(t, fc, "peerrelay-test-0-config") + if got0.RelayServerPort == nil || *got0.RelayServerPort != 41641 { + t.Errorf("replica 0: expected RelayServerPort=41641, got %v", got0.RelayServerPort) + } + wantEndpoints := []netip.AddrPort{netip.MustParseAddrPort("1.2.3.4:41641")} + if !slices.Equal(got0.RelayServerStaticEndpoints, wantEndpoints) { + t.Errorf("replica 0: expected RelayServerStaticEndpoints=%v, got %v", wantEndpoints, got0.RelayServerStaticEndpoints) + } + if got0.Hostname == nil || *got0.Hostname != "test-0" { + t.Errorf("replica 0: expected hostname=test-0, got %v", got0.Hostname) + } + + got1 := readTailscaledConfig(t, fc, "peerrelay-test-1-config") + if got1.RelayServerPort == nil || *got1.RelayServerPort != 41641 { + t.Errorf("replica 1: expected RelayServerPort=41641, got %v", got1.RelayServerPort) + } + // Replica 1's LB has not been provisioned yet, so no static endpoints. + if len(got1.RelayServerStaticEndpoints) != 0 { + t.Errorf("replica 1: expected no RelayServerStaticEndpoints, got %v", got1.RelayServerStaticEndpoints) + } +} + +func readTailscaledConfig(t *testing.T, fc client.Client, secretName string) ipn.ConfigVAlpha { + t.Helper() + + var secret corev1.Secret + if err := fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: secretName}, &secret); err != nil { + t.Fatalf("failed to get config Secret %q: %v", secretName, err) + } + + if len(secret.Data) != 1 { + t.Fatalf("expected exactly one file in config Secret %q, got %d", secretName, len(secret.Data)) + } + + var conf ipn.ConfigVAlpha + for name, body := range secret.Data { + if err := json.Unmarshal(body, &conf); err != nil { + t.Fatalf("failed to unmarshal config file %q from Secret %q: %v", name, secretName, err) + } + } + + return conf +} + +func applyPatchInterceptor() interceptor.Funcs { + return interceptor.Funcs{ + Patch: func(ctx context.Context, cl client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if patch.Type() != types.ApplyPatchType { + return cl.Patch(ctx, obj, patch, opts...) + } + + key := client.ObjectKeyFromObject(obj) + existing := obj.DeepCopyObject().(client.Object) + if err := cl.Get(ctx, key, existing); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + + return cl.Create(ctx, obj) + } + + obj.SetResourceVersion(existing.GetResourceVersion()) + return cl.Update(ctx, obj) + }, + } +} + +type fakeClientProvider struct { + client tsclient.Client + err error +} + +func (p *fakeClientProvider) For(_ string) (tsclient.Client, error) { return p.client, p.err } + +type fakeTSClient struct { + tsclient.Client + + mu sync.Mutex + keyCalls []tailscaleclient.CreateKeyRequest + deviceDeletes []string + nextKey []string +} + +func (c *fakeTSClient) Keys() tsclient.KeyResource { return (*fakeKeys)(c) } +func (c *fakeTSClient) Devices() tsclient.DeviceResource { return (*fakeDevices)(c) } + +func (c *fakeTSClient) CreateAuthKeyCalls() []tailscaleclient.CreateKeyRequest { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.keyCalls) +} + +func (c *fakeTSClient) DeviceDeletes() []string { + c.mu.Lock() + defer c.mu.Unlock() + return slices.Clone(c.deviceDeletes) +} + +type fakeKeys fakeTSClient + +func (k *fakeKeys) CreateAuthKey(_ context.Context, req tailscaleclient.CreateKeyRequest) (*tailscaleclient.Key, error) { + c := (*fakeTSClient)(k) + c.mu.Lock() + defer c.mu.Unlock() + c.keyCalls = append(c.keyCalls, req) + + var key string + if len(c.nextKey) > 0 { + key, c.nextKey = c.nextKey[0], c.nextKey[1:] + } else { + key = fmt.Sprintf("auth-key-%d", len(c.keyCalls)) + } + return &tailscaleclient.Key{Key: key}, nil +} + +func (k *fakeKeys) List(_ context.Context, _ bool) ([]tailscaleclient.Key, error) { return nil, nil } + +type fakeDevices fakeTSClient + +func (d *fakeDevices) Delete(_ context.Context, id string) error { + c := (*fakeTSClient)(d) + c.mu.Lock() + defer c.mu.Unlock() + c.deviceDeletes = append(c.deviceDeletes, id) + return nil +} + +func (d *fakeDevices) List(_ context.Context, _ ...tailscaleclient.ListDevicesOptions) ([]tailscaleclient.Device, error) { + return nil, nil +} + +func (d *fakeDevices) Get(_ context.Context, _ string) (*tailscaleclient.Device, error) { + return nil, nil +} + +func TestReconciler_AuthKey_Lifecycle(t *testing.T) { + t.Parallel() + + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + t.Run("mints-key-on-first-reconcile", func(t *testing.T) { + pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}} + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}). + WithObjects(pr). + Build() + + tsc := &fakeTSClient{nextKey: []string{"tskey-abc"}} + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:k8s-peer-relay"}, + Clients: &fakeClientProvider{client: tsc}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err := r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + calls := tsc.CreateAuthKeyCalls() + if len(calls) != 1 { + t.Fatalf("expected 1 CreateAuthKey call, got %d", len(calls)) + } + gotTags := calls[0].Capabilities.Devices.Create.Tags + if !slices.Equal(gotTags, []string{"tag:k8s-peer-relay"}) { + t.Errorf("expected default tags, got %v", gotTags) + } + + conf := readTailscaledConfig(t, fc, "peerrelay-test-0-config") + if conf.AuthKey == nil || *conf.AuthKey != "tskey-abc" { + t.Errorf("expected AuthKey=tskey-abc in config, got %v", conf.AuthKey) + } + }) + + t.Run("reuses-existing-key-across-reconciles", func(t *testing.T) { + pr := &tsapi.PeerRelay{ObjectMeta: metav1.ObjectMeta{Name: "test"}} + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}). + WithObjects(pr). + Build() + + tsc := &fakeTSClient{nextKey: []string{"tskey-first", "tskey-second"}} + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:k8s-peer-relay"}, + Clients: &fakeClientProvider{client: tsc}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + if got := len(tsc.CreateAuthKeyCalls()); got != 1 { + t.Errorf("expected 1 CreateAuthKey call across two reconciles, got %d", got) + } + + conf := readTailscaledConfig(t, fc, "peerrelay-test-0-config") + if conf.AuthKey == nil || *conf.AuthKey != "tskey-first" { + t.Errorf("expected AuthKey preserved as tskey-first, got %v", conf.AuthKey) + } + }) + + t.Run("uses-peer-relay-specific-tags", func(t *testing.T) { + pr := &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Tags: tsapi.Tags{"tag:custom"}}, + } + + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}). + WithObjects(pr). + Build() + + tsc := &fakeTSClient{} + + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:k8s-peer-relay"}, + Clients: &fakeClientProvider{client: tsc}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + calls := tsc.CreateAuthKeyCalls() + if len(calls) != 1 { + t.Fatalf("expected 1 CreateAuthKey call, got %d", len(calls)) + } + gotTags := calls[0].Capabilities.Devices.Create.Tags + if !slices.Equal(gotTags, []string{"tag:custom"}) { + t.Errorf("expected pr-specific tags, got %v", gotTags) + } + }) +} + +func TestReconciler_DeletesTailnetDevices(t *testing.T) { + t.Parallel() + + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + // stateSecret seeds a Secret shaped like the ones the reconciler pre-creates: parent-resource labels + the + // tailscale.com/secret-type=state marker, containing (optionally) a device_id entry as if tailscaled had + // written it. + stateSecret := func(prName, name string, idx int32, deviceID string) *corev1.Secret { + labels := map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": prName, + "tailscale.com/peer-relay-replica": fmt.Sprintf("%d", idx), + "tailscale.com/secret-type": "state", + } + s := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: tailscaleNamespace, Labels: labels}, + } + if deviceID != "" { + s.Data = map[string][]byte{"device_id": []byte(deviceID)} + } + return s + } + + t.Run("full-delete-removes-all-devices-and-state-secrets", func(t *testing.T) { + pr := &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Finalizers: []string{"tailscale.com/finalizer"}, + DeletionTimestamp: new(metav1.Now()), + }, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(2))}, + } + + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). + WithObjects( + pr, + stateSecret("test", "peerrelay-test-0", 0, "device-aaa"), + stateSecret("test", "peerrelay-test-1", 1, ""), // pod never registered , no device_id + stateSecret("other", "peerrelay-other-0", 0, "device-should-not-touch"), + ). + Build() + + tsc := &fakeTSClient{} + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:test-peer-relay"}, + Clients: &fakeClientProvider{client: tsc}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + if got, want := tsc.DeviceDeletes(), []string{"device-aaa"}; !slices.Equal(got, want) { + t.Errorf("expected Devices().Delete calls %v, got %v", want, got) + } + + // Our state Secrets should be gone; the unrelated PeerRelay's state Secret should still be present. + for _, name := range []string{"peerrelay-test-0", "peerrelay-test-1"} { + var s corev1.Secret + if err = fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: name}, &s); !apierrors.IsNotFound(err) { + t.Errorf("expected state Secret %q gone, got err=%v", name, err) + } + } + var other corev1.Secret + if err = fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: "peerrelay-other-0"}, &other); err != nil { + t.Errorf("unexpected: state Secret for other PeerRelay was removed: %v", err) + } + }) + + t.Run("scale-down-removes-devices-for-removed-replicas-only", func(t *testing.T) { + pr := &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Replicas: new(int32(1))}, + } + + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). + WithObjects( + pr, + stateSecret("test", "peerrelay-test-0", 0, "device-still-here"), + stateSecret("test", "peerrelay-test-1", 1, "device-scaled-away-1"), + stateSecret("test", "peerrelay-test-2", 2, "device-scaled-away-2"), + ). + Build() + + tsc := &fakeTSClient{} + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:test-peer-relay"}, + Clients: &fakeClientProvider{client: tsc}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + got := tsc.DeviceDeletes() + slices.Sort(got) + want := []string{"device-scaled-away-1", "device-scaled-away-2"} + if !slices.Equal(got, want) { + t.Errorf("expected Devices().Delete calls %v, got %v", want, got) + } + + var kept corev1.Secret + if err = fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: "peerrelay-test-0"}, &kept); err != nil { + t.Errorf("expected replica 0 state Secret preserved: %v", err) + } + + for _, name := range []string{"peerrelay-test-1", "peerrelay-test-2"} { + var s corev1.Secret + if err = fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: name}, &s); !apierrors.IsNotFound(err) { + t.Errorf("expected state Secret %q gone after scale-down, got err=%v", name, err) + } + } + }) +} + +func TestReconciler_TailnetUnavailable(t *testing.T) { + t.Parallel() + + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + pr := &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{Tailnet: "missing"}, + } + + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}). + WithObjects(pr). + Build() + + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:test-peer-relay"}, + Clients: &fakeClientProvider{err: errors.New("tailnet missing: not ready")}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err == nil { + t.Fatal("expected reconcile to return the tailnet resolver error, got nil") + } + + var got tsapi.PeerRelay + if err = fc.Get(t.Context(), types.NamespacedName{Name: "test"}, &got); err != nil { + t.Fatal(err) + } + + cond := readyCondition(&got) + if cond.Status != metav1.ConditionFalse { + t.Errorf("expected PeerRelayReady=False, got %q", cond.Status) + } + if cond.Reason != peerrelay.ReasonTailnetUnavailable { + t.Errorf("expected reason=%s, got %q", peerrelay.ReasonTailnetUnavailable, cond.Reason) + } + if !strings.Contains(cond.Message, "not ready") { + t.Errorf("expected condition message to include resolver error, got %q", cond.Message) + } + + var svcs corev1.ServiceList + if err = fc.List(t.Context(), &svcs, client.InNamespace(tailscaleNamespace)); err != nil { + t.Fatal(err) + } + if len(svcs.Items) != 0 { + t.Errorf("expected no Services created while tailnet is unavailable, got %d", len(svcs.Items)) + } +} + +func TestReconciler_AppliesProxyClass(t *testing.T) { + t.Parallel() + + logger, err := zap.NewDevelopment() + if err != nil { + t.Fatal(err) + } + + pr := &tsapi.PeerRelay{ + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: tsapi.PeerRelaySpec{ProxyClass: "custom"}, + } + + pc := &tsapi.ProxyClass{ + ObjectMeta: metav1.ObjectMeta{Name: "custom"}, + Spec: tsapi.ProxyClassSpec{ + StatefulSet: &tsapi.StatefulSet{ + Labels: tsapi.Labels{ + "team": "networking", + "tailscale.com/parent-resource": "hijack-attempt", // must NOT overwrite reconciler-managed value + }, + Annotations: map[string]string{"observability.example.com/scrape": "true"}, + Pod: &tsapi.Pod{ + NodeSelector: map[string]string{"pool": "peer-relays"}, + TailscaleContainer: &tsapi.Container{ + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}, + Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("256Mi")}, + }, + Env: []tsapi.Env{{Name: "TS_DEBUG_FIREWALL_MODE", Value: "auto"}}, + }, + }, + }, + }, + } + + fc := fake.NewClientBuilder().WithInterceptorFuncs(applyPatchInterceptor()). + WithScheme(tsapi.GlobalScheme). + WithStatusSubresource(&tsapi.PeerRelay{}, &appsv1.StatefulSet{}). + WithObjects(pr, pc). + Build() + + r := peerrelay.NewReconciler(peerrelay.ReconcilerOptions{ + Client: fc, + TailscaleNamespace: tailscaleNamespace, + ProxyImage: testProxyImage, + DefaultTags: []string{"tag:test-peer-relay"}, + Clients: &fakeClientProvider{client: &fakeTSClient{}}, + Resolver: testResolver, + Logger: logger.Sugar(), + }) + + if _, err = r.Reconcile(t.Context(), reconcile.Request{NamespacedName: types.NamespacedName{Name: "test"}}); err != nil { + t.Fatal(err) + } + + var ss appsv1.StatefulSet + if err = fc.Get(t.Context(), types.NamespacedName{Namespace: tailscaleNamespace, Name: "peerrelay-test"}, &ss); err != nil { + t.Fatal(err) + } + + if got := ss.Labels["team"]; got != "networking" { + t.Errorf("expected StatefulSet label team=networking, got %q", got) + } + + if got := ss.Labels["tailscale.com/parent-resource"]; got != "test" { + t.Errorf("expected reconciler-managed parent-resource label preserved as %q, got %q", "test", got) + } + + if got := ss.Annotations["observability.example.com/scrape"]; got != "true" { + t.Errorf("expected StatefulSet annotation scrape=true, got %q", got) + } + + if got := ss.Spec.Template.Spec.NodeSelector["pool"]; got != "peer-relays" { + t.Errorf("expected Pod nodeSelector pool=peer-relays, got %q", got) + } + + if len(ss.Spec.Template.Spec.Containers) != 1 { + t.Fatalf("expected 1 container, got %d", len(ss.Spec.Template.Spec.Containers)) + } + + c := ss.Spec.Template.Spec.Containers[0] + if got, want := c.Resources.Requests[corev1.ResourceCPU], resource.MustParse("100m"); !got.Equal(want) { + t.Errorf("expected container CPU request %s, got %s", want.String(), got.String()) + } + + if got, want := c.Resources.Limits[corev1.ResourceMemory], resource.MustParse("256Mi"); !got.Equal(want) { + t.Errorf("expected container memory limit %s, got %s", want.String(), got.String()) + } + + var foundEnv bool + for _, e := range c.Env { + if e.Name == "TS_DEBUG_FIREWALL_MODE" && e.Value == "auto" { + foundEnv = true + break + } + } + + if !foundEnv { + t.Errorf("expected TS_DEBUG_FIREWALL_MODE=auto env, container env is %+v", c.Env) + } +} diff --git a/k8s-operator/reconciler/peerrelay/service.go b/k8s-operator/reconciler/peerrelay/service.go new file mode 100644 index 000000000..e8cd7b154 --- /dev/null +++ b/k8s-operator/reconciler/peerrelay/service.go @@ -0,0 +1,189 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package peerrelay + +import ( + "context" + "fmt" + "maps" + "net/netip" + "slices" + "strconv" + "time" + + "go.uber.org/zap" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" +) + +const ( + // labelReplicaIndex stores the replica index of a managed Service so it can be matched back to a specific + // peer relay instance. + labelReplicaIndex = "tailscale.com/peer-relay-replica" + + // parentTypePeerRelay is the value used for reconciler.LabelParentType on PeerRelay-managed resources. + parentTypePeerRelay = "peerrelay" + + // servicePortName names the UDP port exposed by each Service. Mostly cosmetic, but Kubernetes requires a name + // once a Service has more than one port; using a stable name keeps the door open for that. + servicePortName = "peerrelay" + + // servicePort is the UDP port that each peer relay container will listen on and that the LoadBalancer Service + // exposes externally. + servicePort = 41641 + + annotationEIPAllocations = "service.beta.kubernetes.io/aws-load-balancer-eip-allocations" + annotationSubnets = "service.beta.kubernetes.io/aws-load-balancer-subnets" +) + +// cloudAnnotations are the cloud-provider-specific annotations applied to every generated LoadBalancer Service to +// ensure the Service is provisioned with a publicly addressable IP rather than a DNS name. +var cloudAnnotations = map[string]string{ + // AWS: provision an internet-facing NLB in IP target mode via the AWS Load Balancer Controller. + "service.beta.kubernetes.io/aws-load-balancer-type": "external", + "service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip", + "service.beta.kubernetes.io/aws-load-balancer-scheme": "internet-facing", + "service.beta.kubernetes.io/aws-load-balancer-ip-address-type": "ipv4", + + // Azure: pin the LB to external. + "service.beta.kubernetes.io/azure-load-balancer-internal": "false", +} + +func peerRelayLabels(prName string) map[string]string { + return reconciler.Labels(parentTypePeerRelay, prName, "") +} + +func peerRelayServiceLabels(prName string, idx int32) map[string]string { + labels := peerRelayLabels(prName) + labels[labelReplicaIndex] = strconv.FormatInt(int64(idx), 10) + return labels +} + +func resourceName(prName string) string { + return "peerrelay-" + prName +} + +func replicaName(prName string, idx int32) string { + return fmt.Sprintf("%s-%d", resourceName(prName), idx) +} + +func peerRelayServiceAnnotations(pr *tsapi.PeerRelay, idx int32) map[string]string { + annotations := make(map[string]string, len(cloudAnnotations)) + + if pr.Spec.Service != nil { + maps.Copy(annotations, pr.Spec.Service.Annotations) + } + + maps.Copy(annotations, cloudAnnotations) + + // Per-replica AWS pinning always wins over anything in spec.service.annotations or the cloud defaults so users + // can rely on spec.aws.elasticIPs being the single source of truth for each replica's EIP + subnet. + if pr.Spec.AWS != nil && int(idx) < len(pr.Spec.AWS.ElasticIPs) { + eip := pr.Spec.AWS.ElasticIPs[idx] + annotations[annotationEIPAllocations] = eip.AllocationID + annotations[annotationSubnets] = eip.SubnetID + } + + return annotations +} + +func (r *Reconciler) peerRelayService(pr *tsapi.PeerRelay, idx int32) *corev1.Service { + name := replicaName(pr.Name, idx) + + return &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: r.tailscaleNamespace, + Labels: peerRelayServiceLabels(pr.Name, idx), + Annotations: peerRelayServiceAnnotations(pr, idx), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + // The Service targets the specific StatefulSet pod for this replica. The StatefulSet controller + // automatically sets this label on each pod. + Selector: map[string]string{ + "statefulset.kubernetes.io/pod-name": name, + }, + Ports: []corev1.ServicePort{ + { + Name: servicePortName, + Protocol: corev1.ProtocolUDP, + Port: servicePort, + TargetPort: intstr.FromInt32(servicePort), + }, + }, + }, + } +} + +func replicaIndexFromLabels(labels map[string]string) (int32, bool) { + raw, ok := labels[labelReplicaIndex] + if !ok { + return 0, false + } + + n, err := strconv.ParseInt(raw, 10, 32) + if err != nil { + return 0, false + } + + return int32(n), true +} + +func (r *Reconciler) peerRelayEndpoint(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service, prev *tsapi.PeerRelayEndpoint) *tsapi.PeerRelayEndpoint { + idx, ok := replicaIndexFromLabels(svc.Labels) + if !ok { + return nil + } + + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.IP != "" { + return &tsapi.PeerRelayEndpoint{Replica: idx, Address: ing.IP, Port: servicePort} + } + } + + // Just return nil if we're not dealing with AWS fun. + if _, ok = svc.Annotations[annotationEIPAllocations]; !ok { + return nil + } + + // If we were not able to obtain an IP address, we fall back to an IPv4 lookup. This is specifically for the case + // of AWS where NLB-backed Service resources are only ever given hostnames. We expect users to also provide + // an annotation with their elastic IP allocations so that there is only ever 1 IP address behind the hostname, so + // we perform a lookup so that the user doesn't also need to provide that IP address. + for _, ing := range svc.Status.LoadBalancer.Ingress { + if ing.Hostname == "" { + continue + } + + resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + addrs, err := r.resolver(resolveCtx, "ip4", ing.Hostname) + if err != nil || len(addrs) == 0 { + logger.Warnf("failed to resolve LoadBalancer hostname %q for Service %q: %v", ing.Hostname, svc.Name, err) + // Preserve the previously-known endpoint (if any) so that a failure here doesn't erase status.endpoints. + if prev != nil { + return prev + } + + continue + } + + slices.SortFunc(addrs, netip.Addr.Compare) + return &tsapi.PeerRelayEndpoint{Replica: idx, Address: addrs[0].String(), Port: servicePort} + } + + return nil +} diff --git a/k8s-operator/reconciler/peerrelay/statefulset.go b/k8s-operator/reconciler/peerrelay/statefulset.go new file mode 100644 index 000000000..ecf0ab854 --- /dev/null +++ b/k8s-operator/reconciler/peerrelay/statefulset.go @@ -0,0 +1,101 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package peerrelay + +import ( + "context" + "fmt" + "net/netip" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + + "tailscale.com/ipn" + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" + "tailscale.com/k8s-operator/reconciler" + "tailscale.com/k8s-operator/reconciler/tailscaled" + "tailscale.com/kube/kubetypes" +) + +func configSecretName(prName string, idx int32) string { + return replicaName(prName, idx) + "-config" +} + +func peerRelayHostname(pr *tsapi.PeerRelay, idx int32) string { + prefix := string(pr.Spec.HostnamePrefix) + if prefix == "" { + prefix = pr.Name + } + return fmt.Sprintf("%s-%d", prefix, idx) +} + +func peerRelayTailscaledConfig(pr *tsapi.PeerRelay, idx int32, endpoint *tsapi.PeerRelayEndpoint, authKey *string) ipn.ConfigVAlpha { + conf := ipn.ConfigVAlpha{ + Version: "alpha0", + AcceptDNS: "false", + AcceptRoutes: "false", + Locked: "false", + Hostname: new(peerRelayHostname(pr, idx)), + RelayServerPort: new(uint16(servicePort)), + AuthKey: authKey, + } + + if endpoint != nil { + if addr, err := netip.ParseAddr(endpoint.Address); err == nil { + conf.RelayServerStaticEndpoints = []netip.AddrPort{ + netip.AddrPortFrom(addr, uint16(endpoint.Port)), + } + } + } + + return conf +} + +func (r *Reconciler) peerRelayConfigSecret(pr *tsapi.PeerRelay, idx int32, endpoint *tsapi.PeerRelayEndpoint, authKey *string) (*corev1.Secret, error) { + labels := peerRelayServiceLabels(pr.Name, idx) + return tailscaled.NewConfigSecret(tailscaled.ConfigSecretOptions{ + Name: configSecretName(pr.Name, idx), + Namespace: r.tailscaleNamespace, + Labels: labels, + Config: peerRelayTailscaledConfig(pr, idx, endpoint, authKey), + }) +} + +func (r *Reconciler) peerRelayStatefulSet(pr *tsapi.PeerRelay, replicas int32, pc *tsapi.ProxyClass) *appsv1.StatefulSet { + labels := peerRelayLabels(pr.Name) + ss := tailscaled.NewStatefulSet(tailscaled.StatefulSetOptions{ + Name: resourceName(pr.Name), + Namespace: r.tailscaleNamespace, + Labels: labels, + Image: r.proxyImage, + Replicas: replicas, + ServiceAccountName: "proxies", + ConfigSecretNameFunc: func(idx int32) string { + return configSecretName(pr.Name, idx) + }, + }) + + return tailscaled.ApplyProxyClass(ss, pc, managedLabelKeys, nil) +} + +var managedLabelKeys = []string{ + kubetypes.LabelManaged, + reconciler.LabelParentType, + reconciler.LabelParentName, +} + +func (r *Reconciler) getProxyClass(ctx context.Context, pr *tsapi.PeerRelay) (*tsapi.ProxyClass, error) { + if pr.Spec.ProxyClass == "" { + return nil, nil + } + + var pc tsapi.ProxyClass + if err := r.Get(ctx, types.NamespacedName{Name: pr.Spec.ProxyClass}, &pc); err != nil { + return nil, fmt.Errorf("failed to get ProxyClass %q: %w", pr.Spec.ProxyClass, err) + } + return &pc, nil +} diff --git a/k8s-operator/reconciler/reconciler.go b/k8s-operator/reconciler/reconciler.go index fcad7201e..4453c9f18 100644 --- a/k8s-operator/reconciler/reconciler.go +++ b/k8s-operator/reconciler/reconciler.go @@ -8,14 +8,32 @@ package reconciler import ( + "context" "slices" + "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "tailscale.com/kube/kubetypes" ) const ( // FinalizerName is the common finalizer used across all Tailscale Kubernetes resources. FinalizerName = "tailscale.com/finalizer" + + // LabelParentType identifies which Tailscale CRD kind owns a managed resource. Every resource that a Tailscale + // CRD reconciler creates should carry this label alongside LabelParentName. + LabelParentType = "tailscale.com/parent-resource-type" + + // LabelParentName identifies the name of the Tailscale CRD that owns a managed resource. Combined with + // LabelParentType, this uniquely identifies the parent within its scope. + LabelParentName = "tailscale.com/parent-resource" + + // LabelParentNamespace identifies the namespace of the owning Tailscale CRD. It is only stamped when the parent + // CRD is namespaced; cluster-scoped parents omit it. + LabelParentNamespace = "tailscale.com/parent-resource-ns" ) // SetFinalizer adds the finalizer to the resource if not already present. @@ -37,3 +55,41 @@ func RemoveFinalizer(obj client.Object) { finalizers := obj.GetFinalizers() obj.SetFinalizers(append(finalizers[:idx], finalizers[idx+1:]...)) } + +// Labels returns the standard ownership labels stamped on every resource a Tailscale CRD reconciler creates: +// tailscale.com/managed=true, plus LabelParentType and LabelParentName. If parentNamespace is non-empty (i.e. the +// parent CRD is namespaced) it is stamped as LabelParentNamespace; cluster-scoped parents pass "". +func Labels(parentType, parentName, parentNamespace string) map[string]string { + labels := map[string]string{ + kubetypes.LabelManaged: "true", + LabelParentType: parentType, + LabelParentName: parentName, + } + if parentNamespace != "" { + labels[LabelParentNamespace] = parentNamespace + } + return labels +} + +// EnqueueForChild returns a handler.MapFunc that enqueues a reconcile.Request for the parent CRD of a managed child +// resource. It filters by tailscale.com/managed=true and LabelParentType, and derives the request's NamespacedName +// from LabelParentName plus LabelParentNamespace (blank namespace for cluster-scoped parents). Use it on Watches of +// child resources so drift or cloud-controller updates propagate to the owning CRD's reconciler. +func EnqueueForChild(parentType string) handler.MapFunc { + return func(_ context.Context, o client.Object) []reconcile.Request { + labels := o.GetLabels() + if labels[kubetypes.LabelManaged] != "true" || labels[LabelParentType] != parentType { + return nil + } + + name := labels[LabelParentName] + if name == "" { + return nil + } + + return []reconcile.Request{{NamespacedName: types.NamespacedName{ + Name: name, + Namespace: labels[LabelParentNamespace], + }}} + } +} diff --git a/k8s-operator/reconciler/reconciler_test.go b/k8s-operator/reconciler/reconciler_test.go index 2db77e7aa..48cb2f301 100644 --- a/k8s-operator/reconciler/reconciler_test.go +++ b/k8s-operator/reconciler/reconciler_test.go @@ -6,11 +6,14 @@ package reconciler_test import ( + "maps" "slices" "testing" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlreconcile "sigs.k8s.io/controller-runtime/pkg/reconcile" "tailscale.com/k8s-operator/reconciler" ) @@ -40,3 +43,99 @@ func TestFinalizers(t *testing.T) { t.Fatalf("object still has finalizer %q: %v", reconciler.FinalizerName, object.Finalizers) } } + +func TestLabels(t *testing.T) { + t.Parallel() + + t.Run("cluster-scoped-parent", func(t *testing.T) { + got := reconciler.Labels("peerrelay", "test", "") + want := map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + } + if !maps.Equal(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + }) + + t.Run("namespaced-parent", func(t *testing.T) { + got := reconciler.Labels("connector", "test", "kube-system") + want := map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "connector", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": "kube-system", + } + if !maps.Equal(got, want) { + t.Errorf("expected %v, got %v", want, got) + } + }) +} + +func TestEnqueueForChild(t *testing.T) { + t.Parallel() + + enqueue := reconciler.EnqueueForChild("peerrelay") + + tests := []struct { + name string + labels map[string]string + want []ctrlreconcile.Request + }{ + { + name: "matching-cluster-scoped", + labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + }, + want: []ctrlreconcile.Request{{NamespacedName: types.NamespacedName{Name: "test"}}}, + }, + { + name: "matching-namespaced", + labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + "tailscale.com/parent-resource-ns": "kube-system", + }, + want: []ctrlreconcile.Request{{NamespacedName: types.NamespacedName{Name: "test", Namespace: "kube-system"}}}, + }, + { + name: "not-managed", + labels: map[string]string{ + "tailscale.com/parent-resource-type": "peerrelay", + "tailscale.com/parent-resource": "test", + }, + }, + { + name: "wrong-parent-type", + labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "proxygroup", + "tailscale.com/parent-resource": "test", + }, + }, + { + name: "missing-parent-name", + labels: map[string]string{ + "tailscale.com/managed": "true", + "tailscale.com/parent-resource-type": "peerrelay", + }, + }, + { + name: "no-labels", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + obj := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Labels: tc.labels}} + got := enqueue(t.Context(), obj) + if !slices.Equal(got, tc.want) { + t.Errorf("expected %v, got %v", tc.want, got) + } + }) + } +} diff --git a/k8s-operator/reconciler/tailscaled/authkey.go b/k8s-operator/reconciler/tailscaled/authkey.go new file mode 100644 index 000000000..478b8cfa4 --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/authkey.go @@ -0,0 +1,56 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailscaled + +import ( + "context" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + + tailscaleclient "tailscale.com/client/tailscale/v2" + + "tailscale.com/ipn" + "tailscale.com/k8s-operator/tsclient" +) + +// ClientProvider returns a Tailscale API client for the given tailnet name. A blank name should return the +// operator's default client. +type ClientProvider interface { + For(tailnet string) (tsclient.Client, error) +} + +// NewAuthKey mints a single-use, preauthorized tailnet auth key with the given tags. The key is intended for one +// tailscaled pod to consume on first startup; callers should not persist or share it. +func NewAuthKey(ctx context.Context, client tsclient.Client, tags []string) (string, error) { + var caps tailscaleclient.KeyCapabilities + caps.Devices.Create.Reusable = false + caps.Devices.Create.Preauthorized = true + caps.Devices.Create.Tags = tags + + key, err := client.Keys().CreateAuthKey(ctx, tailscaleclient.CreateKeyRequest{Capabilities: caps}) + if err != nil { + return "", fmt.Errorf("failed to create auth key: %w", err) + } + return key.Key, nil +} + +// AuthKeyFromConfigSecret returns the auth key embedded in the tailscaled config file stored in secret, or nil if +// none is set. secret is expected to be a Secret produced by NewConfigSecret. The Data map may contain multiple +// versioned config files (cap-.hujson); the first one to parse successfully and yield a non-empty AuthKey wins. +func AuthKeyFromConfigSecret(secret *corev1.Secret) *string { + for _, body := range secret.Data { + var conf ipn.ConfigVAlpha + if err := json.Unmarshal(body, &conf); err != nil { + continue + } + if conf.AuthKey != nil && *conf.AuthKey != "" { + return conf.AuthKey + } + } + return nil +} diff --git a/k8s-operator/reconciler/tailscaled/proxyclass.go b/k8s-operator/reconciler/tailscaled/proxyclass.go new file mode 100644 index 000000000..cecb93efb --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/proxyclass.go @@ -0,0 +1,115 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +package tailscaled + +import ( + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + + tsapi "tailscale.com/k8s-operator/apis/v1alpha1" +) + +// ApplyProxyClass overlays the settings in pc onto ss. It's the generic slice of ProxyClass application used by +// any reconciler that produces a tailscaled StatefulSet (peer relay, connector, proxy group, etc.). +func ApplyProxyClass(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass, managedLabels, managedAnnotations []string) *appsv1.StatefulSet { + if pc == nil || ss == nil || pc.Spec.StatefulSet == nil { + return ss + } + + if wantsLabels := pc.Spec.StatefulSet.Labels.Parse(); len(wantsLabels) > 0 { + ss.ObjectMeta.Labels = mergeProtected(ss.ObjectMeta.Labels, wantsLabels, managedLabels) + } + + if wantsAnnots := pc.Spec.StatefulSet.Annotations; len(wantsAnnots) > 0 { + ss.ObjectMeta.Annotations = mergeProtected(ss.ObjectMeta.Annotations, wantsAnnots, managedAnnotations) + } + + if pc.Spec.StatefulSet.Pod == nil { + return ss + } + wantsPod := pc.Spec.StatefulSet.Pod + + if wantsPodLabels := wantsPod.Labels.Parse(); len(wantsPodLabels) > 0 { + ss.Spec.Template.ObjectMeta.Labels = mergeProtected(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, managedLabels) + } + + if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 { + ss.Spec.Template.ObjectMeta.Annotations = mergeProtected(ss.Spec.Template.ObjectMeta.Annotations, wantsPodAnnots, managedAnnotations) + } + + ss.Spec.Template.Spec.SecurityContext = wantsPod.SecurityContext + ss.Spec.Template.Spec.ImagePullSecrets = wantsPod.ImagePullSecrets + ss.Spec.Template.Spec.NodeName = wantsPod.NodeName + ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector + ss.Spec.Template.Spec.Affinity = wantsPod.Affinity + ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations + ss.Spec.Template.Spec.PriorityClassName = wantsPod.PriorityClassName + ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints + + if wantsPod.DNSPolicy != nil { + ss.Spec.Template.Spec.DNSPolicy = *wantsPod.DNSPolicy + } + + if wantsPod.DNSConfig != nil { + ss.Spec.Template.Spec.DNSConfig = wantsPod.DNSConfig + } + + if wantsPod.TailscaleContainer != nil { + for i := range ss.Spec.Template.Spec.Containers { + c := &ss.Spec.Template.Spec.Containers[i] + if c.Name != containerName { + continue + } + + applyContainerOverlay(c, wantsPod.TailscaleContainer) + break + } + } + + return ss +} + +func mergeProtected(current, custom map[string]string, protected []string) map[string]string { + if custom == nil { + custom = make(map[string]string) + } + for k, v := range current { + if slices.Contains(protected, k) { + custom[k] = v + } + } + return custom +} + +func applyContainerOverlay(c *corev1.Container, overlay *tsapi.Container) { + if overlay.SecurityContext != nil { + c.SecurityContext = overlay.SecurityContext + } + + if len(overlay.Resources.Requests) > 0 { + c.Resources.Requests = overlay.Resources.Requests + } + + if len(overlay.Resources.Limits) > 0 { + c.Resources.Limits = overlay.Resources.Limits + } + + for _, e := range overlay.Env { + // Env vars added by ProxyClass are appended; Kubernetes uses the last entry for a duplicate name, so this + // lets the user override anything we set (e.g. TS_USERSPACE) without us having to know the full list. + c.Env = append(c.Env, corev1.EnvVar{Name: string(e.Name), Value: e.Value}) + } + + if overlay.Image != "" { + c.Image = overlay.Image + } + + if overlay.ImagePullPolicy != "" { + c.ImagePullPolicy = overlay.ImagePullPolicy + } +} diff --git a/k8s-operator/reconciler/tailscaled/statefulset.go b/k8s-operator/reconciler/tailscaled/statefulset.go new file mode 100644 index 000000000..d1b8d0032 --- /dev/null +++ b/k8s-operator/reconciler/tailscaled/statefulset.go @@ -0,0 +1,223 @@ +// Copyright (c) Tailscale Inc & contributors +// SPDX-License-Identifier: BSD-3-Clause + +//go:build !plan9 + +// Package tailscaled provides shared building blocks for operator reconcilers that manage StatefulSets running +// tailscaled pods (peer relays, connectors, proxy groups, etc). Callers describe the workload via StatefulSetOptions +// / ConfigSecretOptions and this package returns fully-populated *appsv1.StatefulSet and *corev1.Secret objects +// wired up the same way across the codebase: config-file-driven tailscaled started from a per-replica Secret +// mounted at /etc/tsconfig/. +package tailscaled + +import ( + "encoding/json" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "tailscale.com/ipn" + tsoperator "tailscale.com/k8s-operator" + "tailscale.com/kube/kubetypes" + "tailscale.com/tailcfg" +) + +const ( + // ConfigVolumeMountPath is the base directory tailscaled reads config files from. Each pod's config lives at + // //cap-.hujson. + ConfigVolumeMountPath = "/etc/tsconfig" + + // ConfigDirEnvVar is the env var containerboot reads to find the config file directory. It is templated with + // $(POD_NAME) so each replica picks its own directory at runtime. + ConfigDirEnvVar = "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR" + + // containerName is the single container inside each pod that runs tailscaled. + containerName = "tailscaled" +) + +// StatefulSetOptions describes a StatefulSet of tailscaled pods. The zero value is not valid , Name, Namespace, +// Image, Labels, and ConfigSecretNameFunc must be set. +type StatefulSetOptions struct { + // Name is the StatefulSet's metadata name; pods will be named -. + Name string + + // Namespace is the namespace the StatefulSet lives in. + Namespace string + + // Labels are applied to the StatefulSet, its pod template, and used as the label selector. Callers must + // include enough labels to uniquely identify the workload , typically at least tailscale.com/parent-resource + // and tailscale.com/parent-resource-type. + Labels map[string]string + + // Image is the tailscale container image used for every pod. + Image string + + // Replicas is the desired number of pods. + Replicas int32 + + // ServiceAccountName is the ServiceAccount used by every pod. Must have get/create/patch/update permission on + // the per-pod state Secret named after each pod (containerboot's TS_KUBE_SECRET). Defaults to "default" when + // unset, which is unlikely to have the needed RBAC. + ServiceAccountName string + + // ConfigSecretNameFunc returns the name of the config Secret containing tailscaled config for the given + // replica ordinal. Its output is used to build a per-replica volume and mount into the pod at + // /-. + ConfigSecretNameFunc func(idx int32) string +} + +// NewStatefulSet returns a *appsv1.StatefulSet configured to run tailscaled from per-replica config Secrets. +// The caller is responsible for setting resource requests/limits, ProxyClass overrides, etc. after the fact. +func NewStatefulSet(opts StatefulSetOptions) *appsv1.StatefulSet { + volumes := make([]corev1.Volume, 0, opts.Replicas) + mounts := make([]corev1.VolumeMount, 0, opts.Replicas) + for i := int32(0); i < opts.Replicas; i++ { + volName := fmt.Sprintf("tailscaledconfig-%d", i) + volumes = append(volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: opts.ConfigSecretNameFunc(i)}, + }, + }) + mounts = append(mounts, corev1.VolumeMount{ + Name: volName, + ReadOnly: true, + MountPath: fmt.Sprintf("%s/%s-%d", ConfigVolumeMountPath, opts.Name, i), + }) + } + + return &appsv1.StatefulSet{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "apps/v1", + Kind: "StatefulSet", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.Name, + Namespace: opts.Namespace, + Labels: opts.Labels, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &opts.Replicas, + ServiceName: opts.Name, + Selector: &metav1.LabelSelector{MatchLabels: opts.Labels}, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: opts.Labels}, + Spec: corev1.PodSpec{ + ServiceAccountName: opts.ServiceAccountName, + Volumes: volumes, + Containers: []corev1.Container{{ + Name: containerName, + Image: opts.Image, + VolumeMounts: mounts, + Env: []corev1.EnvVar{ + { + Name: "POD_NAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}, + }, + }, + { + // containerboot picks up the config file matching its own capability version from + // this directory. + Name: ConfigDirEnvVar, + Value: fmt.Sprintf("%s/$(POD_NAME)", ConfigVolumeMountPath), + }, + { + // tailscaled persists device/machine keys in this Secret so a pod restart doesn't + // force reauth. Naming it after the pod gives each replica its own state. + Name: "TS_KUBE_SECRET", + Value: "$(POD_NAME)", + }, + }, + }}, + }, + }, + }, + } +} + +// ConfigSecretOptions describes a single-replica tailscaled config Secret. Name, Namespace, and Config must be +// set. If CapVersion is 0, tailcfg.CurrentCapabilityVersion is used. +type ConfigSecretOptions struct { + Name string + Namespace string + Labels map[string]string + CapVersion tailcfg.CapabilityVersion + Config ipn.ConfigVAlpha +} + +// NewConfigSecret marshals opts.Config into JSON and returns a *corev1.Secret with the file keyed by +// tsoperator.TailscaledConfigFileName(opts.CapVersion). The tailscale.com/secret-type=config label is stamped on +// automatically alongside any caller-provided labels. +func NewConfigSecret(opts ConfigSecretOptions) (*corev1.Secret, error) { + cap := opts.CapVersion + if cap == 0 { + cap = tailcfg.CurrentCapabilityVersion + } + + body, err := json.Marshal(opts.Config) + if err != nil { + return nil, fmt.Errorf("failed to marshal tailscaled config: %w", err) + } + + labels := make(map[string]string, len(opts.Labels)+1) + for k, v := range opts.Labels { + labels[k] = v + } + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeConfig + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.Name, + Namespace: opts.Namespace, + Labels: labels, + }, + Data: map[string][]byte{ + tsoperator.TailscaledConfigFileName(cap): body, + }, + }, nil +} + +// StateSecretOptions describes a per-pod tailscaled state Secret. Name must match the pod name (the value +// containerboot reads from TS_KUBE_SECRET) so that tailscaled can locate it at runtime. +type StateSecretOptions struct { + Name string + Namespace string + Labels map[string]string +} + +// NewStateSecret returns an empty *corev1.Secret to be pre-created for tailscaled's kube state store. Pre-creating it +// (rather than letting containerboot create it on first run) lets callers stamp ownership labels so cleanup can select +// state Secrets by label rather than by pod-name convention. The tailscale.com/secret-type=state label is stamped on +// automatically alongside any caller-provided labels; tailscaled populates the Data on first run. +func NewStateSecret(opts StateSecretOptions) *corev1.Secret { + labels := make(map[string]string, len(opts.Labels)+1) + for k, v := range opts.Labels { + labels[k] = v + } + labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeState + + return &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: opts.Name, + Namespace: opts.Namespace, + Labels: labels, + }, + } +} + +// DeviceIDFromStateSecret returns the tailnet device ID that tailscaled recorded in secret, or "" if none. secret +// should be a state Secret populated by containerboot; the device ID is the value stored under kubetypes.KeyDeviceID. +func DeviceIDFromStateSecret(secret *corev1.Secret) string { + return string(secret.Data[kubetypes.KeyDeviceID]) +} diff --git a/kube/kubetypes/types.go b/kube/kubetypes/types.go index 02ad336e4..b99980ef4 100644 --- a/kube/kubetypes/types.go +++ b/kube/kubetypes/types.go @@ -34,6 +34,7 @@ const ( MetricProxyGroupIngressCount = "k8s_proxygroup_ingress_resources" MetricProxyGroupAPIServerCount = "k8s_proxygroup_kube_apiserver_resources" MetricTailnetCount = "k8s_tailnet_resources" + MetricPeerRelayCount = "k8s_peerrelay_resources" // Keys that containerboot writes to state file that can be used to determine its state. // fields set in Tailscale state Secret. These are mostly used by the Tailscale Kubernetes operator to determine