webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/k8s-operator: allow specifying replicas for connectors (#16721)

Browse Source 
This commit adds a `replicas` field to the `Connector` custom resource that
allows users to specify the number of desired replicas deployed for their
connectors.

This allows users to deploy exit nodes, subnet routers and app connectors
in a highly available fashion.

Fixes #14020

Signed-off-by: David Bond <davidsbond93@gmail.com>
This commit is contained in:
David Bond
2025-09-02 13:10:03 +01:00
committed by GitHub
parent d05e6dc09e
commit 12ad630128
13 changed files with 665 additions and 202 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/k8s-operator/connector.go
+31 -10
View File
@@ -25,7 +25,6 @@ import (
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
@@ -176,6 +175,7 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
if cn.Spec.Hostname != "" {
hostname = string(cn.Spec.Hostname)
}
crl := childResourceLabels(cn.Name, a.tsnamespace, "connector")
proxyClass := cn.Spec.ProxyClass
@@ -188,10 +188,17 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
}
}
var replicas int32 = 1
if cn.Spec.Replicas != nil {
replicas = *cn.Spec.Replicas
}
sts := &tailscaleSTSConfig{
Replicas: replicas,
ParentResourceName: cn.Name,
ParentResourceUID: string(cn.UID),
Hostname: hostname,
HostnamePrefix: string(cn.Spec.HostnamePrefix),
ChildResourceLabels: crl,
Tags: cn.Spec.Tags.Stringify(),
Connector: &connector{
@@ -219,16 +226,19 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
} else {
a.exitNodes.Remove(cn.UID)
}
if cn.Spec.SubnetRouter != nil {
a.subnetRouters.Add(cn.GetUID())
} else {
a.subnetRouters.Remove(cn.GetUID())
}
if cn.Spec.AppConnector != nil {
a.appConnectors.Add(cn.GetUID())
} else {
a.appConnectors.Remove(cn.GetUID())
}
a.mu.Unlock()
gaugeConnectorSubnetRouterResources.Set(int64(a.subnetRouters.Len()))
gaugeConnectorExitNodeResources.Set(int64(a.exitNodes.Len()))
@@ -244,21 +254,23 @@ func (a *ConnectorReconciler) maybeProvisionConnector(ctx context.Context, logge
return err
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
if err != nil {
return err
}
if dev == nil || dev.hostname == "" {
logger.Debugf("no Tailscale hostname known yet, waiting for Connector Pod to finish auth")
// No hostname yet. Wait for the connector pod to auth.
cn.Status.TailnetIPs = nil
cn.Status.Hostname = ""
return nil
cn.Status.Devices = make([]tsapi.ConnectorDevice, len(devices))
for i, dev := range devices {
cn.Status.Devices[i] = tsapi.ConnectorDevice{
Hostname: dev.hostname,
TailnetIPs: dev.ips,
}
}
cn.Status.TailnetIPs = dev.ips
cn.Status.Hostname = dev.hostname
if len(cn.Status.Devices) > 0 {
cn.Status.Hostname = cn.Status.Devices[0].Hostname
cn.Status.TailnetIPs = cn.Status.Devices[0].TailnetIPs
}
return nil
}
@@ -302,6 +314,15 @@ func (a *ConnectorReconciler) validate(cn *tsapi.Connector) error {
if (cn.Spec.SubnetRouter != nil || cn.Spec.ExitNode) && cn.Spec.AppConnector != nil {
return errors.New("invalid spec: a Connector that is configured as an app connector must not be also configured as a subnet router or exit node")
}
// These two checks should be caught by the Connector schema validation.
if cn.Spec.Replicas != nil && *cn.Spec.Replicas > 1 && cn.Spec.Hostname != "" {
return errors.New("invalid spec: a Connector that is configured with multiple replicas cannot specify a hostname. Instead, use a hostnamePrefix")
}
if cn.Spec.HostnamePrefix != "" && cn.Spec.Hostname != "" {
return errors.New("invalid spec: a Connect cannot use both a hostname and hostname prefix")
}
if cn.Spec.AppConnector != nil {
return validateAppConnector(cn.Spec.AppConnector)
}
cmd/k8s-operator/connector_test.go
+115 -6
View File
@@ -7,6 +7,8 @@ package main
import (
"context"
"strconv"
"strings"
"testing"
"time"
@@ -20,6 +22,7 @@ import (
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
"tailscale.com/tstest"
"tailscale.com/types/ptr"
"tailscale.com/util/mak"
)
@@ -36,6 +39,7 @@ func TestConnector(t *testing.T) {
APIVersion: "tailscale.com/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
@@ -55,7 +59,8 @@ func TestConnector(t *testing.T) {
cl := tstest.NewClock(tstest.ClockOpts{})
cr := &ConnectorReconciler{
Client: fc,
Client: fc,
recorder: record.NewFakeRecorder(10),
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
@@ -78,6 +83,7 @@ func TestConnector(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
app: kubetypes.AppConnector,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -94,6 +100,10 @@ func TestConnector(t *testing.T) {
cn.Status.IsExitNode = cn.Spec.ExitNode
cn.Status.SubnetRoutes = cn.Spec.SubnetRouter.AdvertiseRoutes.Stringify()
cn.Status.Hostname = hostname
cn.Status.Devices = []tsapi.ConnectorDevice{{
Hostname: hostname,
TailnetIPs: []string{"127.0.0.1", "::1"},
}}
cn.Status.TailnetIPs = []string{"127.0.0.1", "::1"}
expectEqual(t, fc, cn, func(o *tsapi.Connector) {
o.Status.Conditions = nil
@@ -156,6 +166,7 @@ func TestConnector(t *testing.T) {
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
@@ -174,6 +185,7 @@ func TestConnector(t *testing.T) {
subnetRoutes: "10.40.0.0/14",
hostname: "test-connector",
app: kubetypes.AppConnector,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -217,9 +229,11 @@ func TestConnectorWithProxyClass(t *testing.T) {
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
SubnetRouter: &tsapi.SubnetRouter{
AdvertiseRoutes: []tsapi.Route{"10.40.0.0/14"},
},
ExitNode: true,
},
}
@@ -260,6 +274,7 @@ func TestConnectorWithProxyClass(t *testing.T) {
isExitNode: true,
subnetRoutes: "10.40.0.0/14",
app: kubetypes.AppConnector,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -311,6 +326,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](1),
AppConnector: &tsapi.AppConnector{},
},
}
@@ -340,7 +356,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
recorder: fr,
}
// 1. Connector with app connnector is created and becomes ready
// 1. Connector with app connector is created and becomes ready
expectReconciled(t, cr, "", "test")
fullName, shortName := findGenName(t, fc, "", "test", "connector")
opts := configOpts{
@@ -350,6 +366,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
hostname: "test-connector",
app: kubetypes.AppConnector,
isAppConnector: true,
replicas: cn.Spec.Replicas,
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
expectEqual(t, fc, expectedSTS(t, fc, opts), removeResourceReqs)
@@ -357,6 +374,7 @@ func TestConnectorWithAppConnector(t *testing.T) {
cn.ObjectMeta.Finalizers = append(cn.ObjectMeta.Finalizers, "tailscale.com/finalizer")
cn.Status.IsAppConnector = true
cn.Status.Devices = []tsapi.ConnectorDevice{}
cn.Status.Conditions = []metav1.Condition{{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
@@ -368,9 +386,9 @@ func TestConnectorWithAppConnector(t *testing.T) {
// 2. Connector with invalid app connector routes has status set to invalid
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
conn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"}
})
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("1.2.3.4/5")}
cn.Spec.AppConnector.Routes = tsapi.Routes{"1.2.3.4/5"}
expectReconciled(t, cr, "", "test")
cn.Status.Conditions = []metav1.Condition{{
Type: string(tsapi.ConnectorReady),
@@ -383,9 +401,9 @@ func TestConnectorWithAppConnector(t *testing.T) {
// 3. Connector with valid app connnector routes becomes ready
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
conn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"}
})
cn.Spec.AppConnector.Routes = tsapi.Routes{tsapi.Route("10.88.2.21/32")}
cn.Spec.AppConnector.Routes = tsapi.Routes{"10.88.2.21/32"}
cn.Status.Conditions = []metav1.Condition{{
Type: string(tsapi.ConnectorReady),
Status: metav1.ConditionTrue,
@@ -395,3 +413,94 @@ func TestConnectorWithAppConnector(t *testing.T) {
}}
expectReconciled(t, cr, "", "test")
}
func TestConnectorWithMultipleReplicas(t *testing.T) {
cn := &tsapi.Connector{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
UID: types.UID("1234-UID"),
},
TypeMeta: metav1.TypeMeta{
Kind: tsapi.ConnectorKind,
APIVersion: "tailscale.io/v1alpha1",
},
Spec: tsapi.ConnectorSpec{
Replicas: ptr.To[int32](3),
AppConnector: &tsapi.AppConnector{},
HostnamePrefix: "test-connector",
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(cn).
WithStatusSubresource(cn).
Build()
ft := &fakeTSClient{}
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
cl := tstest.NewClock(tstest.ClockOpts{})
fr := record.NewFakeRecorder(1)
cr := &ConnectorReconciler{
Client: fc,
clock: cl,
ssr: &tailscaleSTSReconciler{
Client: fc,
tsClient: ft,
defaultTags: []string{"tag:k8s"},
operatorNamespace: "operator-ns",
proxyImage: "tailscale/tailscale",
},
logger: zl.Sugar(),
recorder: fr,
}
// 1. Ensure that our connector resource is reconciled.
expectReconciled(t, cr, "", "test")
// 2. Ensure we have a number of secrets matching the number of replicas.
names := findGenNames(t, fc, "", "test", "connector")
if int32(len(names)) != *cn.Spec.Replicas {
t.Fatalf("expected %d secrets, got %d", *cn.Spec.Replicas, len(names))
}
// 3. Ensure each device has the correct hostname prefix and ordinal suffix.
for i, name := range names {
expected := expectedSecret(t, fc, configOpts{
secretName: name,
hostname: string(cn.Spec.HostnamePrefix) + "-" + strconv.Itoa(i),
isAppConnector: true,
parentType: "connector",
namespace: cr.tsnamespace,
})
expectEqual(t, fc, expected)
}
// 4. Ensure the generated stateful set has the matching number of replicas
shortName := strings.TrimSuffix(names[0], "-0")
var sts appsv1.StatefulSet
if err = fc.Get(t.Context(), types.NamespacedName{Namespace: "operator-ns", Name: shortName}, &sts); err != nil {
t.Fatalf("failed to get StatefulSet %q: %v", shortName, err)
}
if sts.Spec.Replicas == nil {
t.Fatalf("actual StatefulSet %q does not have replicas set", shortName)
}
if *sts.Spec.Replicas != *cn.Spec.Replicas {
t.Fatalf("expected %d replicas, got %d", *cn.Spec.Replicas, *sts.Spec.Replicas)
}
// 5. We'll scale the connector down by 1 replica and make sure its secret is cleaned up
mustUpdate[tsapi.Connector](t, fc, "", "test", func(conn *tsapi.Connector) {
conn.Spec.Replicas = ptr.To[int32](2)
})
expectReconciled(t, cr, "", "test")
names = findGenNames(t, fc, "", "test", "connector")
if len(names) != 2 {
t.Fatalf("expected 2 secrets, got %d", len(names))
}
}
cmd/k8s-operator/deploy/crds/tailscale.com_connectors.yaml
+45 -2
View File
@@ -115,9 +115,19 @@ spec:
Connector node. If unset, hostname defaults to <connector
name>-connector. Hostname can contain lower case letters, numbers and
dashes, it must not start or end with a dash and must be between 2
and 63 characters long.
and 63 characters long. This field should only be used when creating a connector
with an unspecified number of replicas, or a single replica.
type: string
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
hostnamePrefix:
description: |-
HostnamePrefix specifies the hostname prefix for each
replica. Each device will have the integer number
from its StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long.
type: string
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
proxyClass:
description: |-
ProxyClass is the name of the ProxyClass custom resource that
@@ -125,6 +135,14 @@ spec:
resources created for this Connector. If unset, the operator will
create resources with the default configuration.
type: string
replicas:
description: |-
Replicas specifies how many devices to create. Set this to enable
high availability for app connectors, subnet routers, or exit nodes.
https://tailscale.com/kb/1115/high-availability. Defaults to 1.
type: integer
format: int32
minimum: 0
subnetRouter:
description: |-
SubnetRouter defines subnet routes that the Connector device should
@@ -168,6 +186,10 @@ spec:
message: A Connector needs to have at least one of exit node, subnet router or app connector configured.
- rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
- rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)'
message: The hostname field cannot be specified when replicas is greater than 1.
- rule: '!(has(self.hostname) && has(self.hostnamePrefix))'
message: The hostname and hostnamePrefix fields are mutually exclusive.
status:
description: |-
ConnectorStatus describes the status of the Connector. This is set
@@ -235,11 +257,32 @@ spec:
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
devices:
description: Devices contains information on each device managed by the Connector resource.
type: array
items:
type: object
properties:
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector replica.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
type: string
tailnetIPs:
description: |-
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
assigned to the Connector replica.
type: array
items:
type: string
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector node.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
node. When using multiple replicas, this field will be populated with the
first replica's hostname. Use the Hostnames field for the full list
of hostnames.
type: string
isAppConnector:
description: IsAppConnector is set to true if the Connector acts as an app connector.
cmd/k8s-operator/deploy/manifests/operator.yaml
+45 -2
View File
@@ -140,9 +140,19 @@ spec:
Connector node. If unset, hostname defaults to <connector
name>-connector. Hostname can contain lower case letters, numbers and
dashes, it must not start or end with a dash and must be between 2
and 63 characters long.
and 63 characters long. This field should only be used when creating a connector
with an unspecified number of replicas, or a single replica.
pattern: ^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$
type: string
hostnamePrefix:
description: |-
HostnamePrefix specifies the hostname prefix for each
replica. Each device will have the integer number
from its StatefulSet pod appended to this prefix to form the full hostname.
HostnamePrefix can contain lower case letters, numbers and dashes, it
must not start with a dash and must be between 1 and 62 characters long.
pattern: ^[a-z0-9][a-z0-9-]{0,61}$
type: string
proxyClass:
description: |-
ProxyClass is the name of the ProxyClass custom resource that
@@ -150,6 +160,14 @@ spec:
resources created for this Connector. If unset, the operator will
create resources with the default configuration.
type: string
replicas:
description: |-
Replicas specifies how many devices to create. Set this to enable
high availability for app connectors, subnet routers, or exit nodes.
https://tailscale.com/kb/1115/high-availability. Defaults to 1.
format: int32
minimum: 0
type: integer
subnetRouter:
description: |-
SubnetRouter defines subnet routes that the Connector device should
@@ -194,6 +212,10 @@ spec:
rule: has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true) || has(self.appConnector)
- message: The appConnector field is mutually exclusive with exitNode and subnetRouter fields.
rule: '!((has(self.subnetRouter) || (has(self.exitNode) && self.exitNode == true)) && has(self.appConnector))'
- message: The hostname field cannot be specified when replicas is greater than 1.
rule: '!(has(self.hostname) && has(self.replicas) && self.replicas > 1)'
- message: The hostname and hostnamePrefix fields are mutually exclusive.
rule: '!(has(self.hostname) && has(self.hostnamePrefix))'
status:
description: |-
ConnectorStatus describes the status of the Connector. This is set
@@ -260,11 +282,32 @@ spec:
x-kubernetes-list-map-keys:
- type
x-kubernetes-list-type: map
devices:
description: Devices contains information on each device managed by the Connector resource.
items:
properties:
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector replica.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
type: string
tailnetIPs:
description: |-
TailnetIPs is the set of tailnet IP addresses (both IPv4 and IPv6)
assigned to the Connector replica.
items:
type: string
type: array
type: object
type: array
hostname:
description: |-
Hostname is the fully qualified domain name of the Connector node.
If MagicDNS is enabled in your tailnet, it is the MagicDNS name of the
node.
node. When using multiple replicas, this field will be populated with the
first replica's hostname. Use the Hostnames field for the full list
of hostnames.
type: string
isAppConnector:
description: IsAppConnector is set to true if the Connector acts as an app connector.
cmd/k8s-operator/ingress.go
+15 -16
View File
@@ -212,6 +212,7 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
hostname := hostnameForIngress(ing)
sts := &tailscaleSTSConfig{
Replicas: 1,
Hostname: hostname,
ParentResourceName: ing.Name,
ParentResourceUID: string(ing.UID),
@@ -227,27 +228,23 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
sts.ForwardClusterTrafficViaL7IngressProxy = true
}
if _, err := a.ssr.Provision(ctx, logger, sts); err != nil {
if _, err = a.ssr.Provision(ctx, logger, sts); err != nil {
return fmt.Errorf("failed to provision: %w", err)
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
if err != nil {
return fmt.Errorf("failed to retrieve Ingress HTTPS endpoint status: %w", err)
}
if dev == nil || dev.ingressDNSName == "" {
logger.Debugf("no Ingress DNS name known yet, waiting for proxy Pod initialize and start serving Ingress")
// No hostname yet. Wait for the proxy pod to auth.
ing.Status.LoadBalancer.Ingress = nil
if err := a.Status().Update(ctx, ing); err != nil {
return fmt.Errorf("failed to update ingress status: %w", err)
}
return nil
}
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
ing.Status.LoadBalancer.Ingress = []networkingv1.IngressLoadBalancerIngress{
{
ing.Status.LoadBalancer.Ingress = nil
for _, dev := range devices {
if dev.ingressDNSName == "" {
continue
}
logger.Debugf("setting Ingress hostname to %q", dev.ingressDNSName)
ing.Status.LoadBalancer.Ingress = append(ing.Status.LoadBalancer.Ingress, networkingv1.IngressLoadBalancerIngress{
Hostname: dev.ingressDNSName,
Ports: []networkingv1.IngressPortStatus{
{
@@ -255,11 +252,13 @@ func (a *IngressReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
Port: 443,
},
},
},
})
}
if err := a.Status().Update(ctx, ing); err != nil {
if err = a.Status().Update(ctx, ing); err != nil {
return fmt.Errorf("failed to update ingress status: %w", err)
}
return nil
}
cmd/k8s-operator/ingress_test.go
+2 -1
View File
@@ -57,6 +57,7 @@ func TestTailscaleIngress(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "ingress")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -766,7 +767,7 @@ func ingress() *networkingv1.Ingress {
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
UID: "1234-UID",
},
Spec: networkingv1.IngressSpec{
IngressClassName: ptr.To("tailscale"),
cmd/k8s-operator/operator_test.go
+13
View File
@@ -122,6 +122,7 @@ func TestLoadBalancerClass(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -260,6 +261,7 @@ func TestTailnetTargetFQDNAnnotation(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -372,6 +374,7 @@ func TestTailnetTargetIPAnnotation(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -623,6 +626,7 @@ func TestAnnotations(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -729,6 +733,7 @@ func TestAnnotationIntoLB(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -859,6 +864,7 @@ func TestLBIntoAnnotation(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -999,6 +1005,7 @@ func TestCustomHostname(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1111,6 +1118,7 @@ func TestCustomPriorityClassName(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1359,6 +1367,7 @@ func TestProxyClassForService(t *testing.T) {
expectReconciled(t, sr, "default", "test")
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1454,6 +1463,7 @@ func TestDefaultLoadBalancer(t *testing.T) {
expectEqual(t, fc, expectedHeadlessService(shortName, "svc"))
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1509,6 +1519,7 @@ func TestProxyFirewallMode(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
o := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
@@ -1800,6 +1811,7 @@ func Test_authKeyRemoval(t *testing.T) {
hostname: "default-test",
clusterTargetIP: "10.20.30.40",
app: kubetypes.AppIngressProxy,
replicas: ptr.To[int32](1),
}
expectEqual(t, fc, expectedSecret(t, fc, opts))
@@ -1867,6 +1879,7 @@ func Test_externalNameService(t *testing.T) {
fullName, shortName := findGenName(t, fc, "default", "test", "svc")
opts := configOpts{
replicas: ptr.To[int32](1),
stsName: shortName,
secretName: fullName,
namespace: "default",
cmd/k8s-operator/sts.go
+229 -133
View File
@@ -13,6 +13,7 @@ import (
"fmt"
"net/http"
"os"
"path"
"slices"
"strconv"
"strings"
@@ -20,6 +21,7 @@ import (
"go.uber.org/zap"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apiequality "k8s.io/apimachinery/pkg/api/equality"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -114,6 +116,7 @@ var (
)
type tailscaleSTSConfig struct {
Replicas int32
ParentResourceName string
ParentResourceUID string
ChildResourceLabels map[string]string
@@ -144,6 +147,10 @@ type tailscaleSTSConfig struct {
// LoginServer denotes the URL of the control plane that should be used by the proxy.
LoginServer string
// HostnamePrefix specifies the desired prefix for the device's hostname. The hostname will be suffixed with the
// ordinal number generated by the StatefulSet.
HostnamePrefix string
}
type connector struct {
@@ -205,11 +212,12 @@ func (a *tailscaleSTSReconciler) Provision(ctx context.Context, logger *zap.Suga
}
sts.ProxyClass = proxyClass
secretName, _, err := a.createOrGetSecret(ctx, logger, sts, hsvc)
secretNames, err := a.provisionSecrets(ctx, logger, sts, hsvc)
if err != nil {
return nil, fmt.Errorf("failed to create or get API key secret: %w", err)
}
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretName)
_, err = a.reconcileSTS(ctx, logger, sts, hsvc, secretNames)
if err != nil {
return nil, fmt.Errorf("failed to reconcile statefulset: %w", err)
}
@@ -239,6 +247,7 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
if err != nil {
return false, fmt.Errorf("getting statefulset: %w", err)
}
if sts != nil {
if !sts.GetDeletionTimestamp().IsZero() {
// Deletion in progress, check again later. We'll get another
@@ -246,29 +255,39 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
logger.Debugf("waiting for statefulset %s/%s deletion", sts.GetNamespace(), sts.GetName())
return false, nil
}
err := a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, client.InNamespace(a.operatorNamespace), client.MatchingLabels(labels), client.PropagationPolicy(metav1.DeletePropagationForeground))
if err != nil {
options := []client.DeleteAllOfOption{
client.InNamespace(a.operatorNamespace),
client.MatchingLabels(labels),
client.PropagationPolicy(metav1.DeletePropagationForeground),
}
if err = a.DeleteAllOf(ctx, &appsv1.StatefulSet{}, options...); err != nil {
return false, fmt.Errorf("deleting statefulset: %w", err)
}
logger.Debugf("started deletion of statefulset %s/%s", sts.GetNamespace(), sts.GetName())
return false, nil
}
dev, err := a.DeviceInfo(ctx, labels, logger)
devices, err := a.DeviceInfo(ctx, labels, logger)
if err != nil {
return false, fmt.Errorf("getting device info: %w", err)
}
if dev != nil && dev.id != "" {
logger.Debugf("deleting device %s from control", string(dev.id))
if err := a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
for _, dev := range devices {
if dev.id != "" {
logger.Debugf("deleting device %s from control", string(dev.id))
if err = a.tsClient.DeleteDevice(ctx, string(dev.id)); err != nil {
errResp := &tailscale.ErrResponse{}
if ok := errors.As(err, errResp); ok && errResp.Status == http.StatusNotFound {
logger.Debugf("device %s not found, likely because it has already been deleted from control", string(dev.id))
} else {
return false, fmt.Errorf("deleting device: %w", err)
}
} else {
return false, fmt.Errorf("deleting device: %w", err)
logger.Debugf("device %s deleted from control", string(dev.id))
}
} else {
logger.Debugf("device %s deleted from control", string(dev.id))
}
}
@@ -286,9 +305,10 @@ func (a *tailscaleSTSReconciler) Cleanup(ctx context.Context, logger *zap.Sugare
tsNamespace: a.operatorNamespace,
proxyType: typ,
}
if err := maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
if err = maybeCleanupMetricsResources(ctx, mo, a.Client); err != nil {
return false, fmt.Errorf("error cleaning up metrics resources: %w", err)
}
return true, nil
}
@@ -339,91 +359,139 @@ func (a *tailscaleSTSReconciler) reconcileHeadlessService(ctx context.Context, l
return createOrUpdate(ctx, a.Client, a.operatorNamespace, hsvc, func(svc *corev1.Service) { svc.Spec = hsvc.Spec })
}
func (a *tailscaleSTSReconciler) createOrGetSecret(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) (secretName string, configs tailscaledConfigs, _ error) {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
// Hardcode a -0 suffix so that in future, if we support
// multiple StatefulSet replicas, we can provision -N for
// those.
Name: hsvc.Name + "-0",
Namespace: a.operatorNamespace,
Labels: stsC.ChildResourceLabels,
},
}
var orig *corev1.Secret // unmodified copy of secret
if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
orig = secret.DeepCopy()
} else if !apierrors.IsNotFound(err) {
return "", nil, err
}
func (a *tailscaleSTSReconciler) provisionSecrets(ctx context.Context, logger *zap.SugaredLogger, stsC *tailscaleSTSConfig, hsvc *corev1.Service) ([]string, error) {
secretNames := make([]string, stsC.Replicas)
var authKey string
if orig == nil {
// Initially it contains only tailscaled config, but when the
// proxy starts, it will also store there the state, certs and
// ACME account key.
sts, err := getSingleObject[appsv1.StatefulSet](ctx, a.Client, a.operatorNamespace, stsC.ChildResourceLabels)
// Start by ensuring we have Secrets for the desired number of replicas. This will handle both creating and scaling
// up a StatefulSet.
for i := range stsC.Replicas {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", hsvc.Name, i),
Namespace: a.operatorNamespace,
Labels: stsC.ChildResourceLabels,
},
}
// If we only have a single replica, use the hostname verbatim. Otherwise, use the hostname prefix and add
// an ordinal suffix.
hostname := stsC.Hostname
if stsC.HostnamePrefix != "" {
hostname = fmt.Sprintf("%s-%d", stsC.HostnamePrefix, i)
}
secretNames[i] = secret.Name
var orig *corev1.Secret // unmodified copy of secret
if err := a.Get(ctx, client.ObjectKeyFromObject(secret), secret); err == nil {
logger.Debugf("secret %s/%s already exists", secret.GetNamespace(), secret.GetName())
orig = secret.DeepCopy()
} else if !apierrors.IsNotFound(err) {
return nil, err
}
var (
authKey string
err error
)
if orig == nil {
// Create API Key secret which is going to be used by the statefulset
// to authenticate with Tailscale.
logger.Debugf("creating authkey for new tailscale proxy")
tags := stsC.Tags
if len(tags) == 0 {
tags = a.defaultTags
}
authKey, err = newAuthKey(ctx, a.tsClient, tags)
if err != nil {
return nil, err
}
}
configs, err := tailscaledConfig(stsC, authKey, orig, hostname)
if err != nil {
return "", nil, err
return nil, fmt.Errorf("error creating tailscaled config: %w", err)
}
if sts != nil {
// StatefulSet exists, so we have already created the secret.
// If the secret is missing, they should delete the StatefulSet.
logger.Errorf("Tailscale proxy secret doesn't exist, but the corresponding StatefulSet %s/%s already does. Something is wrong, please delete the StatefulSet.", sts.GetNamespace(), sts.GetName())
return "", nil, nil
latest := tailcfg.CapabilityVersion(-1)
var latestConfig ipn.ConfigVAlpha
for key, val := range configs {
fn := tsoperator.TailscaledConfigFileName(key)
b, err := json.Marshal(val)
if err != nil {
return nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
}
mak.Set(&secret.StringData, fn, string(b))
if key > latest {
latest = key
latestConfig = val
}
}
// Create API Key secret which is going to be used by the statefulset
// to authenticate with Tailscale.
logger.Debugf("creating authkey for new tailscale proxy")
tags := stsC.Tags
if len(tags) == 0 {
tags = a.defaultTags
if stsC.ServeConfig != nil {
j, err := json.Marshal(stsC.ServeConfig)
if err != nil {
return nil, err
}
mak.Set(&secret.StringData, "serve-config", string(j))
}
authKey, err = newAuthKey(ctx, a.tsClient, tags)
if err != nil {
return "", nil, err
}
}
configs, err := tailscaledConfig(stsC, authKey, orig)
if err != nil {
return "", nil, fmt.Errorf("error creating tailscaled config: %w", err)
}
latest := tailcfg.CapabilityVersion(-1)
var latestConfig ipn.ConfigVAlpha
for key, val := range configs {
fn := tsoperator.TailscaledConfigFileName(key)
b, err := json.Marshal(val)
if err != nil {
return "", nil, fmt.Errorf("error marshalling tailscaled config: %w", err)
}
mak.Set(&secret.StringData, fn, string(b))
if key > latest {
latest = key
latestConfig = val
if orig != nil && !apiequality.Semantic.DeepEqual(latest, orig) {
logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err = a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
return nil, err
}
} else {
logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err = a.Create(ctx, secret); err != nil {
return nil, err
}
}
}
if stsC.ServeConfig != nil {
j, err := json.Marshal(stsC.ServeConfig)
if err != nil {
return "", nil, err
}
mak.Set(&secret.StringData, "serve-config", string(j))
// Next, we check if we have additional secrets and remove them and their associated device. This happens when we
// scale an StatefulSet down.
var secrets corev1.SecretList
if err := a.List(ctx, &secrets, client.InNamespace(a.operatorNamespace), client.MatchingLabels(stsC.ChildResourceLabels)); err != nil {
return nil, err
}
if orig != nil {
logger.Debugf("patching the existing proxy Secret with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err := a.Patch(ctx, secret, client.MergeFrom(orig)); err != nil {
return "", nil, err
for _, secret := range secrets.Items {
var ordinal int32
if _, err := fmt.Sscanf(secret.Name, hsvc.Name+"-%d", &ordinal); err != nil {
return nil, err
}
} else {
logger.Debugf("creating a new Secret for the proxy with tailscaled config %s", sanitizeConfigBytes(latestConfig))
if err := a.Create(ctx, secret); err != nil {
return "", nil, err
if ordinal < stsC.Replicas {
continue
}
dev, err := deviceInfo(&secret, "", logger)
if err != nil {
return nil, err
}
if dev != nil && dev.id != "" {
var errResp *tailscale.ErrResponse
err = a.tsClient.DeleteDevice(ctx, string(dev.id))
switch {
case errors.As(err, &errResp) && errResp.Status == http.StatusNotFound:
// This device has possibly already been deleted in the admin console. So we can ignore this
// and move on to removing the secret.
case err != nil:
return nil, err
}
}
if err = a.Delete(ctx, &secret); err != nil {
return nil, err
}
}
return secret.Name, configs, nil
return secretNames, nil
}
// sanitizeConfigBytes returns ipn.ConfigVAlpha in string form with redacted
@@ -443,22 +511,38 @@ func sanitizeConfigBytes(c ipn.ConfigVAlpha) string {
// It retrieves info from a Kubernetes Secret labeled with the provided labels. Capver is cross-validated against the
// Pod to ensure that it is the currently running Pod that set the capver. If the Pod or the Secret does not exist, the
// returned capver is -1. Either of device ID, hostname and IPs can be empty string if not found in the Secret.
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) (dev *device, err error) {
sec, err := getSingleObject[corev1.Secret](ctx, a.Client, a.operatorNamespace, childLabels)
if err != nil {
return dev, err
func (a *tailscaleSTSReconciler) DeviceInfo(ctx context.Context, childLabels map[string]string, logger *zap.SugaredLogger) ([]*device, error) {
var secrets corev1.SecretList
if err := a.List(ctx, &secrets, client.InNamespace(a.operatorNamespace), client.MatchingLabels(childLabels)); err != nil {
return nil, err
}
if sec == nil {
return dev, nil
devices := make([]*device, 0)
for _, sec := range secrets.Items {
podUID := ""
pod := new(corev1.Pod)
err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod)
switch {
case apierrors.IsNotFound(err):
// If the Pod is not found, we won't have its UID. We can still get the device information but the
// capability version will be unknown.
case err != nil:
return nil, err
default:
podUID = string(pod.ObjectMeta.UID)
}
info, err := deviceInfo(&sec, podUID, logger)
if err != nil {
return nil, err
}
if info != nil {
devices = append(devices, info)
}
}
podUID := ""
pod := new(corev1.Pod)
if err := a.Get(ctx, types.NamespacedName{Namespace: sec.Namespace, Name: sec.Name}, pod); err != nil && !apierrors.IsNotFound(err) {
return dev, err
} else if err == nil {
podUID = string(pod.ObjectMeta.UID)
}
return deviceInfo(sec, podUID, logger)
return devices, nil
}
// device contains tailscale state of a proxy device as gathered from its tailscale state Secret.
@@ -534,7 +618,7 @@ var proxyYaml []byte
//go:embed deploy/manifests/userspace-proxy.yaml
var userspaceProxyYaml []byte
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecret string) (*appsv1.StatefulSet, error) {
func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.SugaredLogger, sts *tailscaleSTSConfig, headlessSvc *corev1.Service, proxySecrets []string) (*appsv1.StatefulSet, error) {
ss := new(appsv1.StatefulSet)
if sts.ServeConfig != nil && sts.ForwardClusterTrafficViaL7IngressProxy != true { // If forwarding cluster traffic via is required we need non-userspace + NET_ADMIN + forwarding
if err := yaml.Unmarshal(userspaceProxyYaml, &ss); err != nil {
@@ -573,18 +657,22 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
pod.Labels[key] = val // sync StatefulSet labels to Pod to make it easier for users to select the Pod
}
if sts.Replicas > 0 {
ss.Spec.Replicas = ptr.To(sts.Replicas)
}
// Generic containerboot configuration options.
container.Env = append(container.Env,
corev1.EnvVar{
Name: "TS_KUBE_SECRET",
Value: proxySecret,
Value: "$(POD_NAME)",
},
corev1.EnvVar{
// New style is in the form of cap-<capability-version>.hujson.
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
Value: "/etc/tsconfig",
Value: "/etc/tsconfig/$(POD_NAME)",
},
)
if sts.ForwardClusterTrafficViaL7IngressProxy {
container.Env = append(container.Env, corev1.EnvVar{
Name: "EXPERIMENTAL_ALLOW_PROXYING_CLUSTER_TRAFFIC_VIA_INGRESS",
@@ -592,20 +680,23 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
})
}
configVolume := corev1.Volume{
Name: "tailscaledconfig",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: proxySecret,
for i, secret := range proxySecrets {
configVolume := corev1.Volume{
Name: "tailscaledconfig-" + strconv.Itoa(i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret,
},
},
},
}
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: fmt.Sprintf("tailscaledconfig-%d", i),
ReadOnly: true,
MountPath: path.Join("/etc/tsconfig/", secret),
})
}
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, configVolume)
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "tailscaledconfig",
ReadOnly: true,
MountPath: "/etc/tsconfig",
})
if a.tsFirewallMode != "" {
container.Env = append(container.Env, corev1.EnvVar{
@@ -643,22 +734,27 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
} else if sts.ServeConfig != nil {
container.Env = append(container.Env, corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
Value: "/etc/tailscaled/serve-config",
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
})
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "serve-config",
ReadOnly: true,
MountPath: "/etc/tailscaled",
})
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "serve-config",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: proxySecret,
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
for i, secret := range proxySecrets {
container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
Name: "serve-config-" + strconv.Itoa(i),
ReadOnly: true,
MountPath: path.Join("/etc/tailscaled", secret),
})
pod.Spec.Volumes = append(ss.Spec.Template.Spec.Volumes, corev1.Volume{
Name: "serve-config-" + strconv.Itoa(i),
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret,
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
},
},
},
})
})
}
}
app, err := appInfoForProxy(sts)
@@ -918,13 +1014,13 @@ func isMainContainer(c *corev1.Container) bool {
// tailscaledConfig takes a proxy config, a newly generated auth key if generated and a Secret with the previous proxy
// state and auth key and returns tailscaled config files for currently supported proxy versions.
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret) (tailscaledConfigs, error) {
func tailscaledConfig(stsC *tailscaleSTSConfig, newAuthkey string, oldSecret *corev1.Secret, hostname string) (tailscaledConfigs, error) {
conf := &ipn.ConfigVAlpha{
Version: "alpha0",
AcceptDNS: "false",
AcceptRoutes: "false", // AcceptRoutes defaults to true
Locked: "false",
Hostname: &stsC.Hostname,
Hostname: &hostname,
NoStatefulFiltering: "true", // Explicitly enforce default value, see #14216
AppConnector: &ipn.AppConnectorPrefs{Advertise: false},
}
cmd/k8s-operator/svc.go
+12 -8
View File
@@ -23,7 +23,6 @@ import (
"k8s.io/client-go/tools/record"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/reconcile"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/kubetypes"
@@ -265,6 +264,7 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
}
sts := &tailscaleSTSConfig{
Replicas: 1,
ParentResourceName: svc.Name,
ParentResourceUID: string(svc.UID),
Hostname: nameForService(svc),
@@ -332,11 +332,12 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
dev, err := a.ssr.DeviceInfo(ctx, crl, logger)
devices, err := a.ssr.DeviceInfo(ctx, crl, logger)
if err != nil {
return fmt.Errorf("failed to get device ID: %w", err)
}
if dev == nil || dev.hostname == "" {
if len(devices) == 0 || devices[0].hostname == "" {
msg := "no Tailscale hostname known yet, waiting for proxy pod to finish auth"
logger.Debug(msg)
// No hostname yet. Wait for the proxy pod to auth.
@@ -345,26 +346,29 @@ func (a *ServiceReconciler) maybeProvision(ctx context.Context, logger *zap.Suga
return nil
}
dev := devices[0]
logger.Debugf("setting Service LoadBalancer status to %q, %s", dev.hostname, strings.Join(dev.ips, ", "))
ingress := []corev1.LoadBalancerIngress{
{Hostname: dev.hostname},
}
svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{
Hostname: dev.hostname,
})
clusterIPAddr, err := netip.ParseAddr(svc.Spec.ClusterIP)
if err != nil {
msg := fmt.Sprintf("failed to parse cluster IP: %v", err)
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionFalse, reasonProxyFailed, msg, a.clock, logger)
return errors.New(msg)
}
for _, ip := range dev.ips {
addr, err := netip.ParseAddr(ip)
if err != nil {
continue
}
if addr.Is4() == clusterIPAddr.Is4() { // only add addresses of the same family
ingress = append(ingress, corev1.LoadBalancerIngress{IP: ip})
svc.Status.LoadBalancer.Ingress = append(svc.Status.LoadBalancer.Ingress, corev1.LoadBalancerIngress{IP: ip})
}
}
svc.Status.LoadBalancer.Ingress = ingress
tsoperator.SetServiceCondition(svc, tsapi.ProxyReady, metav1.ConditionTrue, reasonProxyCreated, reasonProxyCreated, a.clock, logger)
return nil
}
cmd/k8s-operator/testutils_test.go
+64 -20
View File
@@ -11,6 +11,7 @@ import (
"fmt"
"net/http"
"net/netip"
"path"
"reflect"
"strings"
"sync"
@@ -69,9 +70,9 @@ type configOpts struct {
shouldRemoveAuthKey bool
secretExtraData map[string][]byte
resourceVersion string
enableMetrics bool
serviceMonitorLabels tsapi.Labels
replicas *int32
enableMetrics bool
serviceMonitorLabels tsapi.Labels
}
func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.StatefulSet {
@@ -88,8 +89,8 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
},
SecurityContext: &corev1.SecurityContext{
Privileged: ptr.To(true),
@@ -106,7 +107,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
var volumes []corev1.Volume
volumes = []corev1.Volume{
{
Name: "tailscaledconfig",
Name: "tailscaledconfig-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
@@ -115,9 +116,9 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
},
}
tsContainer.VolumeMounts = []corev1.VolumeMount{{
Name: "tailscaledconfig",
Name: "tailscaledconfig-0",
ReadOnly: true,
MountPath: "/etc/tsconfig",
MountPath: "/etc/tsconfig/" + opts.secretName,
}}
if opts.firewallMode != "" {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
@@ -154,10 +155,21 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
if opts.serveConfig != nil {
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_SERVE_CONFIG",
Value: "/etc/tailscaled/serve-config",
Value: "/etc/tailscaled/$(POD_NAME)/serve-config",
})
volumes = append(volumes, corev1.Volume{Name: "serve-config", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}})
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"})
volumes = append(volumes, corev1.Volume{
Name: "serve-config-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
Items: []corev1.KeyToPath{{
Key: "serve-config",
Path: "serve-config",
}},
},
},
})
tsContainer.VolumeMounts = append(tsContainer.VolumeMounts, corev1.VolumeMount{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)})
}
tsContainer.Env = append(tsContainer.Env, corev1.EnvVar{
Name: "TS_INTERNAL_APP",
@@ -202,7 +214,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
},
},
Spec: appsv1.StatefulSetSpec{
Replicas: ptr.To[int32](1),
Replicas: opts.replicas,
Selector: &metav1.LabelSelector{
MatchLabels: map[string]string{"app": "1234-UID"},
},
@@ -266,15 +278,15 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
{Name: "POD_IP", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "status.podIP"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_NAME", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.name"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
{Name: "TS_KUBE_SECRET", Value: opts.secretName},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig"},
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/serve-config"},
{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"},
{Name: "TS_INTERNAL_APP", Value: opts.app},
},
ImagePullPolicy: "Always",
VolumeMounts: []corev1.VolumeMount{
{Name: "tailscaledconfig", ReadOnly: true, MountPath: "/etc/tsconfig"},
{Name: "serve-config", ReadOnly: true, MountPath: "/etc/tailscaled"},
{Name: "tailscaledconfig-0", ReadOnly: true, MountPath: path.Join("/etc/tsconfig", opts.secretName)},
{Name: "serve-config-0", ReadOnly: true, MountPath: path.Join("/etc/tailscaled", opts.secretName)},
},
}
if opts.enableMetrics {
@@ -302,16 +314,22 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
}
volumes := []corev1.Volume{
{
Name: "tailscaledconfig",
Name: "tailscaledconfig-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
},
},
},
{Name: "serve-config",
{
Name: "serve-config-0",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{SecretName: opts.secretName, Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}}}}},
Secret: &corev1.SecretVolumeSource{
SecretName: opts.secretName,
Items: []corev1.KeyToPath{{Key: "serve-config", Path: "serve-config"}},
},
},
},
}
ss := &appsv1.StatefulSet{
TypeMeta: metav1.TypeMeta{
@@ -592,6 +610,32 @@ func findGenName(t *testing.T, client client.Client, ns, name, typ string) (full
return s.GetName(), strings.TrimSuffix(s.GetName(), "-0")
}
func findGenNames(t *testing.T, cl client.Client, ns, name, typ string) []string {
t.Helper()
labels := map[string]string{
kubetypes.LabelManaged: "true",
LabelParentName: name,
LabelParentNamespace: ns,
LabelParentType: typ,
}
var list corev1.SecretList
if err := cl.List(t.Context(), &list, client.InNamespace(ns), client.MatchingLabels(labels)); err != nil {
t.Fatalf("finding secrets for %q: %v", name, err)
}
if len(list.Items) == 0 {
t.Fatalf("no secrets found for %q %s %+#v", name, ns, labels)
}
names := make([]string, len(list.Items))
for i, secret := range list.Items {
names[i] = secret.GetName()
}
return names
}
func mustCreate(t *testing.T, client client.Client, obj client.Object) {
t.Helper()
if err := client.Create(context.Background(), obj); err != nil {