cmd/k8s-operator: add multi replica support for recorders (#17864)

This commit adds the `spec.replicas` field to the `Recorder` custom
resource that allows for a highly available deployment of `tsrecorder`
within a kubernetes cluster.

Many changes were required here as the code hard-coded the assumption
of a single replica. This has required a few loops, similar to what we
do for the `Connector` resource to create auth and state secrets. It
was also required to add a check to remove dangling state and auth
secrets should the recorder be scaled down.

Updates: https://github.com/tailscale/tailscale/issues/17965

Signed-off-by: David Bond <davidsbond93@gmail.com>

k8s-operator/apis/v1alpha1/types_recorder.go

@@ -44,6 +44,8 @@ type RecorderList struct {
 	Items []Recorder `json:"items"`
 }
 
+// RecorderSpec describes a tsrecorder instance to be deployed in the cluster
+// +kubebuilder:validation:XValidation:rule="!(self.replicas > 1 && (!has(self.storage) || !has(self.storage.s3)))",message="S3 storage must be used when deploying multiple Recorder replicas"
 type RecorderSpec struct {
 	// Configuration parameters for the Recorder's StatefulSet. The operator
 	// deploys a StatefulSet for each Recorder resource.
@@ -74,6 +76,11 @@ type RecorderSpec struct {
 	// lifetime of a specific pod.
 	// +optional
 	Storage Storage `json:"storage,omitempty"`
+
+	// Replicas specifies how many instances of tsrecorder to run. Defaults to 1.
+	// +optional
+	// +kubebuilder:validation:Minimum=0
+	Replicas *int32 `json:"replicas,omitzero"`
 }
 
 type RecorderStatefulSet struct {

k8s-operator/apis/v1alpha1/zz_generated.deepcopy.go

@@ -1068,6 +1068,11 @@ func (in *RecorderSpec) DeepCopyInto(out *RecorderSpec) {
 		copy(*out, *in)
 	}
 	in.Storage.DeepCopyInto(&out.Storage)
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RecorderSpec.