webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/{containerboot,k8s-operator}: reissue auth keys for broken proxies (#16450)

Browse Source 
Adds logic for containerboot to signal that it can't auth, so the
operator can reissue a new auth key. This only applies when running with
a config file and with a kube state store.

If the operator sees reissue_authkey in a state Secret, it will create a
new auth key iff the config has no auth key or its auth key matches the
value of reissue_authkey from the state Secret. This is to ensure we
don't reissue auth keys in a tight loop if the proxy is slow to start or
failing for some other reason. The reissue logic also uses a burstable
rate limiter to ensure there's no way a terminally misconfigured
or buggy operator can automatically generate new auth keys in a tight loop.

Additional implementation details (ChaosInTheCRD):

- Added `ipn.NotifyInitialHealthState` to ipn watcher, to ensure that
  `n.Health` is populated when notify's are returned.
- on auth failure, containerboot:
  - Disconnects from control server
  - Sets reissue_authkey marker in state Secret with the failing key
  - Polls config file for new auth key (10 minute timeout)
  - Restarts after receiving new key to apply it

- modified operator's reissue logic slightly:
  - Deletes old device from tailnet before creating new key
  - Rate limiting: 1 key per 30s with initial burst equal to replica count
  - In-flight tracking (authKeyReissuing map) prevents duplicate API calls
    across reconcile loops

Updates #14080

Change-Id: I6982f8e741932a6891f2f48a2936f7f6a455317f


(cherry picked from commit 969927c47c3d4de05e90f5b26a6d8d931c5ceed4)

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
Co-authored-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
Tom Proctor
2026-03-11 10:25:57 +00:00
committed by GitHub
parent 7a43e41a27
commit 95a135ead1
11 changed files with 875 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/k8s-operator/proxygroup_test.go
+254 -43
View File
@@ -6,15 +6,19 @@
package main
import (
"context"
"encoding/json"
"fmt"
"net/netip"
"reflect"
"slices"
"strings"
"testing"
"time"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
"golang.org/x/time/rate"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
@@ -28,7 +32,6 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"tailscale.com/client/tailscale"
"tailscale.com/ipn"
kube "tailscale.com/k8s-operator"
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/k8s-proxy/conf"
@@ -637,10 +640,12 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) {
tsFirewallMode: "auto",
defaultProxyClass: "default-pc",
Client: fc,
tsClient: tsClient,
recorder: fr,
clock: cl,
Client: fc,
tsClient: tsClient,
recorder: fr,
clock: cl,
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
for i, r := range tt.reconciles {
@@ -780,11 +785,13 @@ func TestProxyGroupWithStaticEndpoints(t *testing.T) {
tsFirewallMode: "auto",
defaultProxyClass: "default-pc",
Client: fc,
tsClient: tsClient,
recorder: fr,
log: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"),
clock: cl,
Client: fc,
tsClient: tsClient,
recorder: fr,
log: zl.Sugar().With("TestName", tt.name).With("Reconcile", "cleanup"),
clock: cl,
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
if err := fc.Delete(t.Context(), pg); err != nil {
@@ -841,12 +848,15 @@ func TestProxyGroup(t *testing.T) {
tsFirewallMode: "auto",
defaultProxyClass: "default-pc",
Client: fc,
tsClient: tsClient,
recorder: fr,
log: zl.Sugar(),
clock: cl,
Client: fc,
tsClient: tsClient,
recorder: fr,
log: zl.Sugar(),
clock: cl,
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
crd := &apiextensionsv1.CustomResourceDefinition{ObjectMeta: metav1.ObjectMeta{Name: serviceMonitorCRD}}
opts := configOpts{
proxyType: "proxygroup",
@@ -863,7 +873,7 @@ func TestProxyGroup(t *testing.T) {
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupReady, metav1.ConditionFalse, reasonProxyGroupCreating, "the ProxyGroup's ProxyClass \"default-pc\" is not yet in a ready state, waiting...", 1, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, false, pc)
if kube.ProxyGroupAvailable(pg) {
if tsoperator.ProxyGroupAvailable(pg) {
t.Fatal("expected ProxyGroup to not be available")
}
})
@@ -891,7 +901,7 @@ func TestProxyGroup(t *testing.T) {
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionFalse, reasonProxyGroupCreating, "0/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
if kube.ProxyGroupAvailable(pg) {
if tsoperator.ProxyGroupAvailable(pg) {
t.Fatal("expected ProxyGroup to not be available")
}
if expected := 1; reconciler.egressProxyGroups.Len() != expected {
@@ -935,7 +945,7 @@ func TestProxyGroup(t *testing.T) {
tsoperator.SetProxyGroupCondition(pg, tsapi.ProxyGroupAvailable, metav1.ConditionTrue, reasonProxyGroupAvailable, "2/2 ProxyGroup pods running", 0, cl, zl.Sugar())
expectEqual(t, fc, pg)
expectProxyGroupResources(t, fc, pg, true, pc)
if !kube.ProxyGroupAvailable(pg) {
if !tsoperator.ProxyGroupAvailable(pg) {
t.Fatal("expected ProxyGroup to be available")
}
})
@@ -1045,12 +1055,14 @@ func TestProxyGroupTypes(t *testing.T) {
zl, _ := zap.NewDevelopment()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zl.Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zl.Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
t.Run("egress_type", func(t *testing.T) {
@@ -1285,12 +1297,14 @@ func TestKubeAPIServerStatusConditionFlow(t *testing.T) {
WithStatusSubresource(pg).
Build()
r := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
expectReconciled(t, r, "", pg.Name)
@@ -1338,12 +1352,14 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
Build()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
pg := &tsapi.ProxyGroup{
@@ -1367,7 +1383,7 @@ func TestKubeAPIServerType_DoesNotOverwriteServicesConfig(t *testing.T) {
cfg := conf.VersionedConfig{
Version: "v1alpha1",
ConfigV1Alpha1: &conf.ConfigV1Alpha1{
AuthKey: new("secret-authkey"),
AuthKey: new("new-authkey"),
State: new(fmt.Sprintf("kube:%s", pgPodName(pg.Name, 0))),
App: new(kubetypes.AppProxyGroupKubeAPIServer),
LogLevel: new("debug"),
@@ -1423,12 +1439,14 @@ func TestIngressAdvertiseServicesConfigPreserved(t *testing.T) {
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zap.Must(zap.NewDevelopment()).Sugar(),
tsClient: &fakeTSClient{},
clock: tstest.NewClock(tstest.ClockOpts{}),
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
existingServices := []string{"svc1", "svc2"}
@@ -1653,6 +1671,197 @@ func TestValidateProxyGroup(t *testing.T) {
}
}
func TestProxyGroupGetAuthKey(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Finalizers: []string{"tailscale.com/finalizer"},
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeEgress,
Replicas: new(int32(1)),
},
}
tsClient := &fakeTSClient{}
// Variables to reference in test cases.
existingAuthKey := new("existing-auth-key")
newAuthKey := new("new-authkey")
configWith := func(authKey *string) map[string][]byte {
value := []byte("{}")
if authKey != nil {
value = fmt.Appendf(nil, `{"AuthKey": "%s"}`, *authKey)
}
return map[string][]byte{
tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): value,
}
}
initTest := func() (*ProxyGroupReconciler, client.WithWatch) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg).
WithStatusSubresource(pg).
Build()
zl, _ := zap.NewDevelopment()
fr := record.NewFakeRecorder(1)
cl := tstest.NewClock(tstest.ClockOpts{})
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
defaultTags: []string{"tag:test-tag"},
tsFirewallMode: "auto",
Client: fc,
tsClient: tsClient,
recorder: fr,
log: zl.Sugar(),
clock: cl,
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
reconciler.ensureStateAddedForProxyGroup(pg)
return reconciler, fc
}
// Config Secret: exists or not, has key or not.
// State Secret: has device ID or not, requested reissue or not.
for name, tc := range map[string]struct {
configData map[string][]byte
stateData map[string][]byte
expectedAuthKey *string
expectReissue bool
}{
"no_secrets_needs_new": {
expectedAuthKey: newAuthKey, // New ProxyGroup or manually cleared Pod.
},
"no_config_secret_state_authed_ok": {
stateData: map[string][]byte{
kubetypes.KeyDeviceID: []byte("nodeid-0"),
},
expectedAuthKey: newAuthKey, // Always create an auth key if we're creating the config Secret.
},
"config_secret_without_key_state_authed_with_reissue_needs_new": {
configData: configWith(nil),
stateData: map[string][]byte{
kubetypes.KeyDeviceID: []byte("nodeid-0"),
kubetypes.KeyReissueAuthkey: []byte(""),
},
expectedAuthKey: newAuthKey,
expectReissue: true, // Device is authed but reissue was requested.
},
"config_secret_with_key_state_with_reissue_stale_ok": {
configData: configWith(existingAuthKey),
stateData: map[string][]byte{
kubetypes.KeyReissueAuthkey: []byte("some-older-authkey"),
},
expectedAuthKey: existingAuthKey, // Config's auth key is different from the one marked for reissue.
},
"config_secret_with_key_state_with_reissue_existing_key_needs_new": {
configData: configWith(existingAuthKey),
stateData: map[string][]byte{
kubetypes.KeyDeviceID: []byte("nodeid-0"),
kubetypes.KeyReissueAuthkey: []byte(*existingAuthKey),
},
expectedAuthKey: newAuthKey,
expectReissue: true, // Current config's auth key is marked for reissue.
},
"config_secret_without_key_no_state_ok": {
configData: configWith(nil),
expectedAuthKey: nil, // Proxy will set reissue_authkey and then next reconcile will reissue.
},
"config_secret_without_key_state_authed_ok": {
configData: configWith(nil),
stateData: map[string][]byte{
kubetypes.KeyDeviceID: []byte("nodeid-0"),
},
expectedAuthKey: nil, // Device is already authed.
},
"config_secret_with_key_state_authed_ok": {
configData: configWith(existingAuthKey),
stateData: map[string][]byte{
kubetypes.KeyDeviceID: []byte("nodeid-0"),
},
expectedAuthKey: nil, // Auth key getting removed because device is authed.
},
"config_secret_with_key_no_state_keeps_existing": {
configData: configWith(existingAuthKey),
expectedAuthKey: existingAuthKey, // No state, waiting for containerboot to try the auth key.
},
} {
t.Run(name, func(t *testing.T) {
tsClient.deleted = tsClient.deleted[:0] // Reset deleted devices for each test case.
reconciler, fc := initTest()
var cfgSecret *corev1.Secret
if tc.configData != nil {
cfgSecret = &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgConfigSecretName(pg.Name, 0),
Namespace: tsNamespace,
},
Data: tc.configData,
}
}
if tc.stateData != nil {
mustCreate(t, fc, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: pgStateSecretName(pg.Name, 0),
Namespace: tsNamespace,
},
Data: tc.stateData,
})
}
authKey, err := reconciler.getAuthKey(t.Context(), tsClient, pg, cfgSecret, 0, reconciler.log.With("TestName", t.Name()))
if err != nil {
t.Fatalf("unexpected error getting auth key: %v", err)
}
if !reflect.DeepEqual(authKey, tc.expectedAuthKey) {
deref := func(s *string) string {
if s == nil {
return "<nil>"
}
return *s
}
t.Errorf("expected auth key %v, got %v", deref(tc.expectedAuthKey), deref(authKey))
}
// Use the device deletion as a proxy for the fact the new auth key
// was due to a reissue.
switch {
case tc.expectReissue && len(tsClient.deleted) != 1:
t.Errorf("expected 1 deleted device, got %v", tsClient.deleted)
case !tc.expectReissue && len(tsClient.deleted) != 0:
t.Errorf("expected no deleted devices, got %v", tsClient.deleted)
}
if tc.expectReissue {
// Trigger the rate limit in a tight loop. Up to 100 iterations
// to allow for CI that is extremely slow, but should happen on
// first try for any reasonable machine.
stateSecretName := pgStateSecretName(pg.Name, 0)
for range 100 {
//NOTE: (ChaosInTheCRD) we added some protection here to avoid
// trying to reissue when already reissung. This overrides it.
reconciler.mu.Lock()
reconciler.authKeyReissuing[stateSecretName] = false
reconciler.mu.Unlock()
_, err := reconciler.getAuthKey(context.Background(), tsClient, pg, cfgSecret, 0,
reconciler.log.With("TestName", t.Name()))
if err != nil {
if !strings.Contains(err.Error(), "rate limit exceeded") {
t.Fatalf("unexpected error getting auth key: %v", err)
}
return // Expected rate limit error.
}
}
t.Fatal("expected rate limit error, but got none")
}
})
}
}
func proxyClassesForLEStagingTest() (*tsapi.ProxyClass, *tsapi.ProxyClass, *tsapi.ProxyClass) {
pcLEStaging := &tsapi.ProxyClass{
ObjectMeta: metav1.ObjectMeta{
@@ -1903,6 +2112,8 @@ func TestProxyGroupLetsEncryptStaging(t *testing.T) {
tsClient: &fakeTSClient{},
log: zl.Sugar(),
clock: cl,
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
}
expectReconciled(t, reconciler, "", pg.Name)