cmd/k8s-operator,k8s-operator: Kubernetes Peer Relays (#20495)
This commit contains the Kubernetes implementation of peer relays via the new `PeerRelay` CRD. It's a mega branch consisting of the commits of other PRs gone into this work: 1. https://github.com/tailscale/tailscale/pull/20211 2. https://github.com/tailscale/tailscale/pull/20329 3. https://github.com/tailscale/tailscale/pull/20423 4. https://github.com/tailscale/tailscale/pull/20503 An instance of the `PeerRelay` CRD deploys a `StatefulSet` of containerboot instances configured to advertise themselves as peer relays using the IP addresses configured via `LoadBalancer` services on each cloud provider (with some AWS specifics as it's less automatic than its competing cloud providers). Per replica, a `LoadBalancer` type `Service` resource is provisioned and its IP address is used to configure the respective relay. This has been tested with success in AWS, GCP & Azure and provides additional modification to `Service` resources via the CRD for any other kinds of deployment environments. It also contains some work that may appear to be duplication of what already exists within `cmd/k8s-operator` so we can start building an appropriate migration path for `Connector`, `ProxyGroup` etc into respective `k8s-operator/reconciler/*` packages. Closes https://github.com/tailscale/corp/issues/34524
This commit is contained in:
David Bond
@@ -0,0 +1,56 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package tailscaled
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
tailscaleclient "tailscale.com/client/tailscale/v2"
|
||||
|
||||
"tailscale.com/ipn"
|
||||
"tailscale.com/k8s-operator/tsclient"
|
||||
)
|
||||
|
||||
// ClientProvider returns a Tailscale API client for the given tailnet name. A blank name should return the
|
||||
// operator's default client.
|
||||
type ClientProvider interface {
|
||||
For(tailnet string) (tsclient.Client, error)
|
||||
}
|
||||
|
||||
// NewAuthKey mints a single-use, preauthorized tailnet auth key with the given tags. The key is intended for one
|
||||
// tailscaled pod to consume on first startup; callers should not persist or share it.
|
||||
func NewAuthKey(ctx context.Context, client tsclient.Client, tags []string) (string, error) {
|
||||
var caps tailscaleclient.KeyCapabilities
|
||||
caps.Devices.Create.Reusable = false
|
||||
caps.Devices.Create.Preauthorized = true
|
||||
caps.Devices.Create.Tags = tags
|
||||
|
||||
key, err := client.Keys().CreateAuthKey(ctx, tailscaleclient.CreateKeyRequest{Capabilities: caps})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to create auth key: %w", err)
|
||||
}
|
||||
return key.Key, nil
|
||||
}
|
||||
|
||||
// AuthKeyFromConfigSecret returns the auth key embedded in the tailscaled config file stored in secret, or nil if
|
||||
// none is set. secret is expected to be a Secret produced by NewConfigSecret. The Data map may contain multiple
|
||||
// versioned config files (cap-<n>.hujson); the first one to parse successfully and yield a non-empty AuthKey wins.
|
||||
func AuthKeyFromConfigSecret(secret *corev1.Secret) *string {
|
||||
for _, body := range secret.Data {
|
||||
var conf ipn.ConfigVAlpha
|
||||
if err := json.Unmarshal(body, &conf); err != nil {
|
||||
continue
|
||||
}
|
||||
if conf.AuthKey != nil && *conf.AuthKey != "" {
|
||||
return conf.AuthKey
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
package tailscaled
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
|
||||
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
|
||||
)
|
||||
|
||||
// ApplyProxyClass overlays the settings in pc onto ss. It's the generic slice of ProxyClass application used by
|
||||
// any reconciler that produces a tailscaled StatefulSet (peer relay, connector, proxy group, etc.).
|
||||
func ApplyProxyClass(ss *appsv1.StatefulSet, pc *tsapi.ProxyClass, managedLabels, managedAnnotations []string) *appsv1.StatefulSet {
|
||||
if pc == nil || ss == nil || pc.Spec.StatefulSet == nil {
|
||||
return ss
|
||||
}
|
||||
|
||||
if wantsLabels := pc.Spec.StatefulSet.Labels.Parse(); len(wantsLabels) > 0 {
|
||||
ss.ObjectMeta.Labels = mergeProtected(ss.ObjectMeta.Labels, wantsLabels, managedLabels)
|
||||
}
|
||||
|
||||
if wantsAnnots := pc.Spec.StatefulSet.Annotations; len(wantsAnnots) > 0 {
|
||||
ss.ObjectMeta.Annotations = mergeProtected(ss.ObjectMeta.Annotations, wantsAnnots, managedAnnotations)
|
||||
}
|
||||
|
||||
if pc.Spec.StatefulSet.Pod == nil {
|
||||
return ss
|
||||
}
|
||||
wantsPod := pc.Spec.StatefulSet.Pod
|
||||
|
||||
if wantsPodLabels := wantsPod.Labels.Parse(); len(wantsPodLabels) > 0 {
|
||||
ss.Spec.Template.ObjectMeta.Labels = mergeProtected(ss.Spec.Template.ObjectMeta.Labels, wantsPodLabels, managedLabels)
|
||||
}
|
||||
|
||||
if wantsPodAnnots := wantsPod.Annotations; len(wantsPodAnnots) > 0 {
|
||||
ss.Spec.Template.ObjectMeta.Annotations = mergeProtected(ss.Spec.Template.ObjectMeta.Annotations, wantsPodAnnots, managedAnnotations)
|
||||
}
|
||||
|
||||
ss.Spec.Template.Spec.SecurityContext = wantsPod.SecurityContext
|
||||
ss.Spec.Template.Spec.ImagePullSecrets = wantsPod.ImagePullSecrets
|
||||
ss.Spec.Template.Spec.NodeName = wantsPod.NodeName
|
||||
ss.Spec.Template.Spec.NodeSelector = wantsPod.NodeSelector
|
||||
ss.Spec.Template.Spec.Affinity = wantsPod.Affinity
|
||||
ss.Spec.Template.Spec.Tolerations = wantsPod.Tolerations
|
||||
ss.Spec.Template.Spec.PriorityClassName = wantsPod.PriorityClassName
|
||||
ss.Spec.Template.Spec.TopologySpreadConstraints = wantsPod.TopologySpreadConstraints
|
||||
|
||||
if wantsPod.DNSPolicy != nil {
|
||||
ss.Spec.Template.Spec.DNSPolicy = *wantsPod.DNSPolicy
|
||||
}
|
||||
|
||||
if wantsPod.DNSConfig != nil {
|
||||
ss.Spec.Template.Spec.DNSConfig = wantsPod.DNSConfig
|
||||
}
|
||||
|
||||
if wantsPod.TailscaleContainer != nil {
|
||||
for i := range ss.Spec.Template.Spec.Containers {
|
||||
c := &ss.Spec.Template.Spec.Containers[i]
|
||||
if c.Name != containerName {
|
||||
continue
|
||||
}
|
||||
|
||||
applyContainerOverlay(c, wantsPod.TailscaleContainer)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return ss
|
||||
}
|
||||
|
||||
func mergeProtected(current, custom map[string]string, protected []string) map[string]string {
|
||||
if custom == nil {
|
||||
custom = make(map[string]string)
|
||||
}
|
||||
for k, v := range current {
|
||||
if slices.Contains(protected, k) {
|
||||
custom[k] = v
|
||||
}
|
||||
}
|
||||
return custom
|
||||
}
|
||||
|
||||
func applyContainerOverlay(c *corev1.Container, overlay *tsapi.Container) {
|
||||
if overlay.SecurityContext != nil {
|
||||
c.SecurityContext = overlay.SecurityContext
|
||||
}
|
||||
|
||||
if len(overlay.Resources.Requests) > 0 {
|
||||
c.Resources.Requests = overlay.Resources.Requests
|
||||
}
|
||||
|
||||
if len(overlay.Resources.Limits) > 0 {
|
||||
c.Resources.Limits = overlay.Resources.Limits
|
||||
}
|
||||
|
||||
for _, e := range overlay.Env {
|
||||
// Env vars added by ProxyClass are appended; Kubernetes uses the last entry for a duplicate name, so this
|
||||
// lets the user override anything we set (e.g. TS_USERSPACE) without us having to know the full list.
|
||||
c.Env = append(c.Env, corev1.EnvVar{Name: string(e.Name), Value: e.Value})
|
||||
}
|
||||
|
||||
if overlay.Image != "" {
|
||||
c.Image = overlay.Image
|
||||
}
|
||||
|
||||
if overlay.ImagePullPolicy != "" {
|
||||
c.ImagePullPolicy = overlay.ImagePullPolicy
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
// Copyright (c) Tailscale Inc & contributors
|
||||
// SPDX-License-Identifier: BSD-3-Clause
|
||||
|
||||
//go:build !plan9
|
||||
|
||||
// Package tailscaled provides shared building blocks for operator reconcilers that manage StatefulSets running
|
||||
// tailscaled pods (peer relays, connectors, proxy groups, etc). Callers describe the workload via StatefulSetOptions
|
||||
// / ConfigSecretOptions and this package returns fully-populated *appsv1.StatefulSet and *corev1.Secret objects
|
||||
// wired up the same way across the codebase: config-file-driven tailscaled started from a per-replica Secret
|
||||
// mounted at /etc/tsconfig/<pod-name>.
|
||||
package tailscaled
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
appsv1 "k8s.io/api/apps/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
|
||||
"tailscale.com/ipn"
|
||||
tsoperator "tailscale.com/k8s-operator"
|
||||
"tailscale.com/kube/kubetypes"
|
||||
"tailscale.com/tailcfg"
|
||||
)
|
||||
|
||||
const (
|
||||
// ConfigVolumeMountPath is the base directory tailscaled reads config files from. Each pod's config lives at
|
||||
// <ConfigVolumeMountPath>/<POD_NAME>/cap-<version>.hujson.
|
||||
ConfigVolumeMountPath = "/etc/tsconfig"
|
||||
|
||||
// ConfigDirEnvVar is the env var containerboot reads to find the config file directory. It is templated with
|
||||
// $(POD_NAME) so each replica picks its own directory at runtime.
|
||||
ConfigDirEnvVar = "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR"
|
||||
|
||||
// containerName is the single container inside each pod that runs tailscaled.
|
||||
containerName = "tailscaled"
|
||||
)
|
||||
|
||||
// StatefulSetOptions describes a StatefulSet of tailscaled pods. The zero value is not valid , Name, Namespace,
|
||||
// Image, Labels, and ConfigSecretNameFunc must be set.
|
||||
type StatefulSetOptions struct {
|
||||
// Name is the StatefulSet's metadata name; pods will be named <Name>-<ordinal>.
|
||||
Name string
|
||||
|
||||
// Namespace is the namespace the StatefulSet lives in.
|
||||
Namespace string
|
||||
|
||||
// Labels are applied to the StatefulSet, its pod template, and used as the label selector. Callers must
|
||||
// include enough labels to uniquely identify the workload , typically at least tailscale.com/parent-resource
|
||||
// and tailscale.com/parent-resource-type.
|
||||
Labels map[string]string
|
||||
|
||||
// Image is the tailscale container image used for every pod.
|
||||
Image string
|
||||
|
||||
// Replicas is the desired number of pods.
|
||||
Replicas int32
|
||||
|
||||
// ServiceAccountName is the ServiceAccount used by every pod. Must have get/create/patch/update permission on
|
||||
// the per-pod state Secret named after each pod (containerboot's TS_KUBE_SECRET). Defaults to "default" when
|
||||
// unset, which is unlikely to have the needed RBAC.
|
||||
ServiceAccountName string
|
||||
|
||||
// ConfigSecretNameFunc returns the name of the config Secret containing tailscaled config for the given
|
||||
// replica ordinal. Its output is used to build a per-replica volume and mount into the pod at
|
||||
// <ConfigVolumeMountPath>/<Name>-<ordinal>.
|
||||
ConfigSecretNameFunc func(idx int32) string
|
||||
}
|
||||
|
||||
// NewStatefulSet returns a *appsv1.StatefulSet configured to run tailscaled from per-replica config Secrets.
|
||||
// The caller is responsible for setting resource requests/limits, ProxyClass overrides, etc. after the fact.
|
||||
func NewStatefulSet(opts StatefulSetOptions) *appsv1.StatefulSet {
|
||||
volumes := make([]corev1.Volume, 0, opts.Replicas)
|
||||
mounts := make([]corev1.VolumeMount, 0, opts.Replicas)
|
||||
for i := int32(0); i < opts.Replicas; i++ {
|
||||
volName := fmt.Sprintf("tailscaledconfig-%d", i)
|
||||
volumes = append(volumes, corev1.Volume{
|
||||
Name: volName,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{SecretName: opts.ConfigSecretNameFunc(i)},
|
||||
},
|
||||
})
|
||||
mounts = append(mounts, corev1.VolumeMount{
|
||||
Name: volName,
|
||||
ReadOnly: true,
|
||||
MountPath: fmt.Sprintf("%s/%s-%d", ConfigVolumeMountPath, opts.Name, i),
|
||||
})
|
||||
}
|
||||
|
||||
return &appsv1.StatefulSet{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "apps/v1",
|
||||
Kind: "StatefulSet",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: opts.Name,
|
||||
Namespace: opts.Namespace,
|
||||
Labels: opts.Labels,
|
||||
},
|
||||
Spec: appsv1.StatefulSetSpec{
|
||||
Replicas: &opts.Replicas,
|
||||
ServiceName: opts.Name,
|
||||
Selector: &metav1.LabelSelector{MatchLabels: opts.Labels},
|
||||
Template: corev1.PodTemplateSpec{
|
||||
ObjectMeta: metav1.ObjectMeta{Labels: opts.Labels},
|
||||
Spec: corev1.PodSpec{
|
||||
ServiceAccountName: opts.ServiceAccountName,
|
||||
Volumes: volumes,
|
||||
Containers: []corev1.Container{{
|
||||
Name: containerName,
|
||||
Image: opts.Image,
|
||||
VolumeMounts: mounts,
|
||||
Env: []corev1.EnvVar{
|
||||
{
|
||||
Name: "POD_NAME",
|
||||
ValueFrom: &corev1.EnvVarSource{
|
||||
FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"},
|
||||
},
|
||||
},
|
||||
{
|
||||
// containerboot picks up the config file matching its own capability version from
|
||||
// this directory.
|
||||
Name: ConfigDirEnvVar,
|
||||
Value: fmt.Sprintf("%s/$(POD_NAME)", ConfigVolumeMountPath),
|
||||
},
|
||||
{
|
||||
// tailscaled persists device/machine keys in this Secret so a pod restart doesn't
|
||||
// force reauth. Naming it after the pod gives each replica its own state.
|
||||
Name: "TS_KUBE_SECRET",
|
||||
Value: "$(POD_NAME)",
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// ConfigSecretOptions describes a single-replica tailscaled config Secret. Name, Namespace, and Config must be
|
||||
// set. If CapVersion is 0, tailcfg.CurrentCapabilityVersion is used.
|
||||
type ConfigSecretOptions struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Labels map[string]string
|
||||
CapVersion tailcfg.CapabilityVersion
|
||||
Config ipn.ConfigVAlpha
|
||||
}
|
||||
|
||||
// NewConfigSecret marshals opts.Config into JSON and returns a *corev1.Secret with the file keyed by
|
||||
// tsoperator.TailscaledConfigFileName(opts.CapVersion). The tailscale.com/secret-type=config label is stamped on
|
||||
// automatically alongside any caller-provided labels.
|
||||
func NewConfigSecret(opts ConfigSecretOptions) (*corev1.Secret, error) {
|
||||
cap := opts.CapVersion
|
||||
if cap == 0 {
|
||||
cap = tailcfg.CurrentCapabilityVersion
|
||||
}
|
||||
|
||||
body, err := json.Marshal(opts.Config)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to marshal tailscaled config: %w", err)
|
||||
}
|
||||
|
||||
labels := make(map[string]string, len(opts.Labels)+1)
|
||||
for k, v := range opts.Labels {
|
||||
labels[k] = v
|
||||
}
|
||||
labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeConfig
|
||||
|
||||
return &corev1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "Secret",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: opts.Name,
|
||||
Namespace: opts.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
tsoperator.TailscaledConfigFileName(cap): body,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// StateSecretOptions describes a per-pod tailscaled state Secret. Name must match the pod name (the value
|
||||
// containerboot reads from TS_KUBE_SECRET) so that tailscaled can locate it at runtime.
|
||||
type StateSecretOptions struct {
|
||||
Name string
|
||||
Namespace string
|
||||
Labels map[string]string
|
||||
}
|
||||
|
||||
// NewStateSecret returns an empty *corev1.Secret to be pre-created for tailscaled's kube state store. Pre-creating it
|
||||
// (rather than letting containerboot create it on first run) lets callers stamp ownership labels so cleanup can select
|
||||
// state Secrets by label rather than by pod-name convention. The tailscale.com/secret-type=state label is stamped on
|
||||
// automatically alongside any caller-provided labels; tailscaled populates the Data on first run.
|
||||
func NewStateSecret(opts StateSecretOptions) *corev1.Secret {
|
||||
labels := make(map[string]string, len(opts.Labels)+1)
|
||||
for k, v := range opts.Labels {
|
||||
labels[k] = v
|
||||
}
|
||||
labels[kubetypes.LabelSecretType] = kubetypes.LabelSecretTypeState
|
||||
|
||||
return &corev1.Secret{
|
||||
TypeMeta: metav1.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "Secret",
|
||||
},
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: opts.Name,
|
||||
Namespace: opts.Namespace,
|
||||
Labels: labels,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// DeviceIDFromStateSecret returns the tailnet device ID that tailscaled recorded in secret, or "" if none. secret
|
||||
// should be a state Secret populated by containerboot; the device ID is the value stored under kubetypes.KeyDeviceID.
|
||||
func DeviceIDFromStateSecret(secret *corev1.Secret) string {
|
||||
return string(secret.Data[kubetypes.KeyDeviceID])
|
||||
}
|
||||
Reference in New Issue
Block a user