cmd/k8s-operator,k8s-operator: Kubernetes Peer Relays (#20495)

This commit contains the Kubernetes implementation of peer relays via the new `PeerRelay` CRD. It's a mega branch consisting of the commits of other PRs gone into this work:

1. https://github.com/tailscale/tailscale/pull/20211
2. https://github.com/tailscale/tailscale/pull/20329
3. https://github.com/tailscale/tailscale/pull/20423
4. https://github.com/tailscale/tailscale/pull/20503

An instance of the `PeerRelay` CRD deploys a `StatefulSet` of containerboot instances configured to advertise themselves as peer relays using the IP addresses configured via `LoadBalancer` services on each cloud provider (with some AWS specifics as it's less automatic than its competing cloud providers). 

Per replica, a `LoadBalancer` type `Service` resource is provisioned and its IP address is used to configure the respective relay.

This has been tested with success in AWS, GCP & Azure and provides additional modification to `Service` resources via the CRD for any other kinds of deployment environments. It also contains some work that may appear to be duplication of what already exists within `cmd/k8s-operator` so we can start building an appropriate migration path for `Connector`, `ProxyGroup` etc into respective `k8s-operator/reconciler/*` packages.

Closes https://github.com/tailscale/corp/issues/34524

k8s-operator/apis/v1alpha1/register.go

@@ -71,6 +71,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&TailnetList{},
 		&ProxyGroupPolicy{},
 		&ProxyGroupPolicyList{},
+		&PeerRelay{},
+		&PeerRelayList{},
 	)
 
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

k8s-operator/apis/v1alpha1/types_peerrelay.go

@@ -0,0 +1,168 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Code comments on these types should be treated as user facing documentation-
+// they will appear on the PeerRelay CRD i.e. if someone runs kubectl explain peerrelay.
+
+var PeerRelayKind = "PeerRelay"
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=pr
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "PeerRelayReady")].reason`,description="Status of the deployed PeerRelay resources."
+// +kubebuilder:printcolumn:name="Endpoints",type="string",JSONPath=`.status.endpoints[*].address`,description="Public addresses the peer relay replicas are reachable on."
+
+type PeerRelay struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitzero"`
+
+	// Spec describes the desired state of the PeerRelay.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
+	Spec PeerRelaySpec `json:"spec"`
+
+	// Status describes the status of the PeerRelay. This is set
+	// and managed by the Tailscale operator.
+	// +optional
+	Status PeerRelayStatus `json:"status"`
+}
+
+// +kubebuilder:object:root=true
+
+type PeerRelayList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []PeerRelay `json:"items"`
+}
+
+// +kubebuilder:validation:XValidation:rule="!has(self.aws) || !has(self.aws.elasticIPs) || self.aws.elasticIPs.size() >= self.replicas",message="spec.aws.elasticIPs must contain at least one entry per replica"
+type PeerRelaySpec struct {
+	// Tags that the Tailscale node will be tagged with.
+	// Defaults to [tag:k8s].
+	// To autoapprove the device defined by a PeerRelay,
+	// you can configure Tailscale ACLs to give these tags the necessary
+	// permissions.
+	// See https://tailscale.com/kb/1337/acl-syntax#autoapprovers.
+	// If you specify custom tags here, you must also make the operator an owner of these tags.
+	// See  https://tailscale.com/kb/1236/kubernetes-operator/#setting-up-the-kubernetes-operator.
+	// Tags cannot be changed once a PeerRelay node has been created.
+	// Tag values must be in form ^tag:[a-zA-Z][a-zA-Z0-9-]*$.
+	// +optional
+	Tags Tags `json:"tags,omitempty"`
+
+	// HostnamePrefix specifies the hostname prefix for each
+	// replica. Each device will have the integer number
+	// from its StatefulSet pod appended to this prefix to form the full hostname.
+	// HostnamePrefix can contain lower case letters, numbers and dashes, it
+	// must not start with a dash and must be between 1 and 62 characters long.
+	// +optional
+	HostnamePrefix HostnamePrefix `json:"hostnamePrefix,omitzero"`
+
+	// ProxyClass is the name of the ProxyClass custom resource that
+	// contains configuration options that should be applied to the
+	// resources created for this PeerRelay. If unset, the operator will
+	// create resources with the default configuration.
+	// +optional
+	ProxyClass string `json:"proxyClass,omitempty"`
+
+	// Replicas specifies how many devices to create. Set this to enable
+	// high availability for peer relays.
+	// https://tailscale.com/kb/1115/high-availability. Defaults to 1.
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	// +kubebuilder:default=1
+	Replicas *int32 `json:"replicas,omitzero"`
+
+	// Tailnet specifies the tailnet this PeerRelay should join. If blank, the default tailnet is used. When set, this
+	// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
+	// +optional
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PeerRelay tailnet is immutable"
+	Tailnet string `json:"tailnet,omitempty"`
+
+	// Service contains configuration values to modify the LoadBalancer service used to expose the peer relay.
+	// +optional
+	Service *PeerRelayService `json:"service,omitzero"`
+
+	// AWS contains configuration for pinning each replica to a specific AWS Elastic IP and subnet. Only meaningful
+	// when running on EKS with the AWS Load Balancer Controller. When set, the per-replica values override any
+	// aws-load-balancer-eip-allocations or aws-load-balancer-subnets values supplied via spec.service.annotations.
+	// +optional
+	AWS *PeerRelayAWS `json:"aws,omitzero"`
+}
+
+type PeerRelayService struct {
+	// Annotations to apply to the LoadBalancer service. Any annotations that conflict with those used by known
+	// cloud providers to ensure IP addresses rather than DNS names are ignored.
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// PeerRelayAWS contains AWS-specific configuration for a PeerRelay.
+type PeerRelayAWS struct {
+	// ElasticIPs pins each replica to a specific AWS EIP allocation and subnet. Only meaningful when Network Load
+	// Balancers are provisioned by the AWS Load Balancer Controller. ElasticIPs supplies one allocation-subnet pair
+	// per replica: replica N uses ElasticIPs[N]. The list must be at least as long as spec.replicas so every replica
+	// has a distinct EIP; extra entries are permitted so that scale-up doesn't immediately trip validation.
+	//
+	// When set, the reconciler stamps
+	// service.beta.kubernetes.io/aws-load-balancer-eip-allocations and
+	// service.beta.kubernetes.io/aws-load-balancer-subnets on each per-replica Service, overriding any values in
+	// spec.service.annotations.
+	// +listType=atomic
+	// +kubebuilder:validation:MinItems=1
+	ElasticIPs []PeerRelayAWSElasticIP `json:"elasticIPs"`
+}
+
+// PeerRelayAWSElasticIP pairs an EIP allocation with the subnet in the same AZ.
+type PeerRelayAWSElasticIP struct {
+	// AllocationID is the AWS EIP allocation ID (e.g. eipalloc-0123abcd) whose public IP this replica is reachable
+	// on. Stamped as service.beta.kubernetes.io/aws-load-balancer-eip-allocations on the replica's Service.
+	// +kubebuilder:validation:Pattern=`^eipalloc-[0-9a-f]+$`
+	AllocationID string `json:"allocationID"`
+
+	// SubnetID is the AWS subnet in the same availability zone as AllocationID (e.g. subnet-0123abcd). Stamped as
+	// service.beta.kubernetes.io/aws-load-balancer-subnets on the replica's Service so the NLB is provisioned in
+	// the same AZ as the EIP.
+	// +kubebuilder:validation:Pattern=`^subnet-[0-9a-f]+$`
+	SubnetID string `json:"subnetID"`
+}
+
+type PeerRelayStatus struct {
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions"`
+
+	// Endpoints lists the public address:port pairs each peer relay replica is reachable on. There is one entry
+	// per replica whose LoadBalancer Service has been assigned a public address; entries appear as the underlying
+	// cloud provisions each Service.
+	// +listType=map
+	// +listMapKey=replica
+	// +optional
+	Endpoints []PeerRelayEndpoint `json:"endpoints,omitempty"`
+}
+
+type PeerRelayEndpoint struct {
+	// Replica is the zero-based index of the peer relay replica this endpoint targets.
+	Replica int32 `json:"replica"`
+
+	// Address is the public IP or hostname the cloud has allocated for this replica's LoadBalancer Service.
+	// Peers reach this relay by connecting to Address:Port over UDP.
+	Address string `json:"address"`
+
+	// Port is the UDP port the peer relay listens on.
+	Port int32 `json:"port"`
+}
+
+// PeerRelayReady is set to True if the PeerRelay is available for use by operator workloads.
+const PeerRelayReady ConditionType = `PeerRelayReady`

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -550,6 +550,199 @@ func (in *NodePortConfig) DeepCopy() *NodePortConfig {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelay) DeepCopyInto(out *PeerRelay) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelay.
+func (in *PeerRelay) DeepCopy() *PeerRelay {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelay)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *PeerRelay) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelayAWS) DeepCopyInto(out *PeerRelayAWS) {
+	*out = *in
+	if in.ElasticIPs != nil {
+		in, out := &in.ElasticIPs, &out.ElasticIPs
+		*out = make([]PeerRelayAWSElasticIP, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayAWS.
+func (in *PeerRelayAWS) DeepCopy() *PeerRelayAWS {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelayAWS)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelayAWSElasticIP) DeepCopyInto(out *PeerRelayAWSElasticIP) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayAWSElasticIP.
+func (in *PeerRelayAWSElasticIP) DeepCopy() *PeerRelayAWSElasticIP {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelayAWSElasticIP)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelayEndpoint) DeepCopyInto(out *PeerRelayEndpoint) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayEndpoint.
+func (in *PeerRelayEndpoint) DeepCopy() *PeerRelayEndpoint {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelayEndpoint)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelayList) DeepCopyInto(out *PeerRelayList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]PeerRelay, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayList.
+func (in *PeerRelayList) DeepCopy() *PeerRelayList {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelayList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *PeerRelayList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelayService) DeepCopyInto(out *PeerRelayService) {
+	*out = *in
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayService.
+func (in *PeerRelayService) DeepCopy() *PeerRelayService {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelayService)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelaySpec) DeepCopyInto(out *PeerRelaySpec) {
+	*out = *in
+	if in.Tags != nil {
+		in, out := &in.Tags, &out.Tags
+		*out = make(Tags, len(*in))
+		copy(*out, *in)
+	}
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Service != nil {
+		in, out := &in.Service, &out.Service
+		*out = new(PeerRelayService)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.AWS != nil {
+		in, out := &in.AWS, &out.AWS
+		*out = new(PeerRelayAWS)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelaySpec.
+func (in *PeerRelaySpec) DeepCopy() *PeerRelaySpec {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelaySpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PeerRelayStatus) DeepCopyInto(out *PeerRelayStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.Endpoints != nil {
+		in, out := &in.Endpoints, &out.Endpoints
+		*out = make([]PeerRelayEndpoint, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PeerRelayStatus.
+func (in *PeerRelayStatus) DeepCopy() *PeerRelayStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(PeerRelayStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Pod) DeepCopyInto(out *Pod) {
 	*out = *in