cmd/k8s-operator,k8s-operator: Allow the use of multiple tailnets (#18344)

This commit contains  the implementation of multi-tailnet support within the Kubernetes Operator

Each of our custom resources now expose the `spec.tailnet` field. This field is a string that must match the name of an existing `Tailnet` resource. A `Tailnet` resource looks like this:

```yaml
apiVersion: tailscale.com/v1alpha1
kind: Tailnet
metadata:
  name: example  # This is the name that must be referenced by other resources
spec:
  credentials:
    secretName: example-oauth
```

Each `Tailnet` references a `Secret` resource that contains a set of oauth credentials. This secret must be created in the same namespace as the operator:

```yaml
apiVersion: v1
kind: Secret
metadata:
  name: example-oauth # This is the name that's referenced by the Tailnet resource.
  namespace: tailscale
stringData:
  client_id: "client-id"
  client_secret: "client-secret"
```

When created, the operator performs a basic check that the oauth client has access to all required scopes. This is done using read actions on devices, keys & services. While this doesn't capture a missing "write" permission, it catches completely missing permissions. Once this check passes, the `Tailnet` moves into a ready state and can be referenced. Attempting to use a `Tailnet` in a non-ready state will stall the deployment of `Connector`s, `ProxyGroup`s and `Recorder`s until the `Tailnet` becomes ready.

The `spec.tailnet` field informs the operator that a `Connector`, `ProxyGroup`, or `Recorder` must be given an auth key generated using the specified oauth client. For backwards compatibility, the set of credentials the operator is configured with are considered the default. That is, where `spec.tailnet` is not set, the resource will be deployed in the same tailnet as the operator. 

Updates https://github.com/tailscale/corp/issues/34561

k8s-operator/apis/v1alpha1/register.go

@@ -67,6 +67,8 @@ func addKnownTypes(scheme *runtime.Scheme) error {
 		&RecorderList{},
 		&ProxyGroup{},
 		&ProxyGroupList{},
+		&Tailnet{},
+		&TailnetList{},
 	)
 
 	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

k8s-operator/apis/v1alpha1/types_connector.go

@@ -133,6 +133,12 @@ type ConnectorSpec struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=0
 	Replicas *int32 `json:"replicas,omitempty"`
+
+	// Tailnet specifies the tailnet this Connector should join. If blank, the default tailnet is used. When set, this
+	// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
+	// +optional
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Connector tailnet is immutable"
+	Tailnet string `json:"tailnet,omitempty"`
 }
 
 // SubnetRouter defines subnet routes that should be exposed to tailnet via a

k8s-operator/apis/v1alpha1/types_proxygroup.go

@@ -97,6 +97,12 @@ type ProxyGroupSpec struct {
 	// ProxyGroup type. This field is only used when Type is set to "kube-apiserver".
 	// +optional
 	KubeAPIServer *KubeAPIServerConfig `json:"kubeAPIServer,omitempty"`
+
+	// Tailnet specifies the tailnet this ProxyGroup should join. If blank, the default tailnet is used. When set, this
+	// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
+	// +optional
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="ProxyGroup tailnet is immutable"
+	Tailnet string `json:"tailnet,omitempty"`
 }
 
 type ProxyGroupStatus struct {

k8s-operator/apis/v1alpha1/types_recorder.go

@@ -81,6 +81,12 @@ type RecorderSpec struct {
 	// +optional
 	// +kubebuilder:validation:Minimum=0
 	Replicas *int32 `json:"replicas,omitzero"`
+
+	// Tailnet specifies the tailnet this Recorder should join. If blank, the default tailnet is used. When set, this
+	// name must match that of a valid Tailnet resource. This field is immutable and cannot be changed once set.
+	// +optional
+	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Recorder tailnet is immutable"
+	Tailnet string `json:"tailnet,omitempty"`
 }
 
 type RecorderStatefulSet struct {

k8s-operator/apis/v1alpha1/types_tailnet.go

@@ -0,0 +1,69 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !plan9
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// Code comments on these types should be treated as user facing documentation-
+// they will appear on the Tailnet CRD i.e. if someone runs kubectl explain tailnet.
+
+var TailnetKind = "Tailnet"
+
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=tn
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+// +kubebuilder:printcolumn:name="Status",type="string",JSONPath=`.status.conditions[?(@.type == "TailnetReady")].reason`,description="Status of the deployed Tailnet resources."
+
+type Tailnet struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitzero"`
+
+	// Spec describes the desired state of the Tailnet.
+	// More info:
+	// https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status
+	Spec TailnetSpec `json:"spec"`
+
+	// Status describes the status of the Tailnet. This is set
+	// and managed by the Tailscale operator.
+	// +optional
+	Status TailnetStatus `json:"status"`
+}
+
+// +kubebuilder:object:root=true
+
+type TailnetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata"`
+
+	Items []Tailnet `json:"items"`
+}
+
+type TailnetSpec struct {
+	// URL of the control plane to be used by all resources managed by the operator using this Tailnet.
+	// +optional
+	LoginURL string `json:"loginUrl,omitempty"`
+	// Denotes the location of the OAuth credentials to use for authenticating with this Tailnet.
+	Credentials TailnetCredentials `json:"credentials"`
+}
+
+type TailnetCredentials struct {
+	// The name of the secret containing the OAuth credentials. This secret must contain two fields "client_id" and
+	// "client_secret".
+	SecretName string `json:"secretName"`
+}
+
+type TailnetStatus struct {
+	// +listType=map
+	// +listMapKey=type
+	// +optional
+	Conditions []metav1.Condition `json:"conditions"`
+}
+
+// TailnetReady is set to True if the Tailnet is available for use by operator workloads.
+const TailnetReady ConditionType = `TailnetReady`

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -1365,6 +1365,48 @@ func (in Tags) DeepCopy() Tags {
 	return *out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Tailnet) DeepCopyInto(out *Tailnet) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	out.Spec = in.Spec
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tailnet.
+func (in *Tailnet) DeepCopy() *Tailnet {
+	if in == nil {
+		return nil
+	}
+	out := new(Tailnet)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *Tailnet) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TailnetCredentials) DeepCopyInto(out *TailnetCredentials) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetCredentials.
+func (in *TailnetCredentials) DeepCopy() *TailnetCredentials {
+	if in == nil {
+		return nil
+	}
+	out := new(TailnetCredentials)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TailnetDevice) DeepCopyInto(out *TailnetDevice) {
 	*out = *in
@@ -1390,6 +1432,76 @@ func (in *TailnetDevice) DeepCopy() *TailnetDevice {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TailnetList) DeepCopyInto(out *TailnetList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]Tailnet, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetList.
+func (in *TailnetList) DeepCopy() *TailnetList {
+	if in == nil {
+		return nil
+	}
+	out := new(TailnetList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *TailnetList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TailnetSpec) DeepCopyInto(out *TailnetSpec) {
+	*out = *in
+	out.Credentials = in.Credentials
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetSpec.
+func (in *TailnetSpec) DeepCopy() *TailnetSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(TailnetSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TailnetStatus) DeepCopyInto(out *TailnetStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]v1.Condition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TailnetStatus.
+func (in *TailnetStatus) DeepCopy() *TailnetStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(TailnetStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TailscaleConfig) DeepCopyInto(out *TailscaleConfig) {
 	*out = *in