Files
tailscale/ipn/store/kubestore/store_kube_test.go
T
chaosinthecrd 97a75c837d cmd/k8s-operator,ipn/store/kubestore,kube/kubetypes: share ACME account key per tailnet
Introduce a per-tailnet shared ACME account key so that all ingress
ProxyGroup replicas on a tailnet present the same account identity to
Let's Encrypt. This lets renewals claim the ARI "replaces" exemption
from the 50-certs-per-week rate limit, surviving Pod restarts,
ProxyGroup recreation, and cluster migrations.

The operator provisions a "tailscale-acme-accounts" Secret in its
namespace, guarded by a finalizer and a deletion warning event, and
watched so it is recreated promptly if removed. Proxies migrate any
pre-existing per-pod key into the shared Secret on first boot, adopt
the shared key on subsequent boots, and restore it on cert writes if
the Secret was recreated empty. Certs are stamped with the fingerprint
of the issuing account so renewals skip the "replaces" claim when the
account doesn't match.

Opt-in per-ProxyGroup via the tailscale.com/share-acme-account
annotation, or operator-wide via OPERATOR_SHARED_ACME_ACCOUNT_KEY.

Updates #18251
Updates #20288

Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
2026-07-27 12:06:41 +01:00

1289 lines
37 KiB
Go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package kubestore
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"tailscale.com/envknob"
"tailscale.com/ipn"
"tailscale.com/ipn/store/mem"
"tailscale.com/kube/kubeapi"
"tailscale.com/kube/kubeclient"
"tailscale.com/kube/kubetypes"
)
func TestKubernetesPodMigrationWithTPMAttestationKey(t *testing.T) {
stateWithAttestationKey := `{
"Config": {
"NodeID": "nSTABLE123456",
"AttestationKey": {
"tpmPrivate": "c2Vuc2l0aXZlLXRwbS1kYXRhLXRoYXQtb25seS13b3Jrcy1vbi1vcmlnaW5hbC1ub2Rl",
"tpmPublic": "cHVibGljLXRwbS1kYXRhLWZvci1hdHRlc3RhdGlvbi1rZXk="
}
}
}`
secretData := map[string][]byte{
"profile-abc123": []byte(stateWithAttestationKey),
"_current-profile": []byte("profile-abc123"),
}
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{Data: secretData}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secretData = p.Value.(map[string][]byte)
}
}
return nil
},
}
store := &Store{
client: client,
canPatch: true,
secretName: "ts-state",
memory: mem.Store{},
logf: t.Logf,
}
if err := store.loadState(); err != nil {
t.Fatalf("loadState failed: %v", err)
}
// Verify we can read the state from the store
stateBytes, err := store.ReadState("profile-abc123")
if err != nil {
t.Fatalf("ReadState failed: %v", err)
}
// The state should be readable as JSON
var state map[string]json.RawMessage
if err := json.Unmarshal(stateBytes, &state); err != nil {
t.Fatalf("failed to unmarshal state: %v", err)
}
// Verify the Config field exists
configRaw, ok := state["Config"]
if !ok {
t.Fatal("Config field not found in state")
}
// Parse the Config to verify fields are preserved
var config map[string]json.RawMessage
if err := json.Unmarshal(configRaw, &config); err != nil {
t.Fatalf("failed to unmarshal Config: %v", err)
}
// The AttestationKey should be stripped by the kubestore
if _, hasAttestation := config["AttestationKey"]; hasAttestation {
t.Error("AttestationKey should be stripped from state loaded by kubestore")
}
// Verify other fields are preserved
var nodeID string
if err := json.Unmarshal(config["NodeID"], &nodeID); err != nil {
t.Fatalf("failed to unmarshal NodeID: %v", err)
}
if nodeID != "nSTABLE123456" {
t.Errorf("NodeID mismatch: got %q, want %q", nodeID, "nSTABLE123456")
}
}
func TestWriteState(t *testing.T) {
tests := []struct {
name string
initial map[string][]byte
key ipn.StateKey
value []byte
wantData map[string][]byte
allowPatch bool
}{
{
name: "basic_write",
initial: map[string][]byte{
"existing": []byte("old"),
},
key: "foo",
value: []byte("bar"),
wantData: map[string][]byte{
"existing": []byte("old"),
"foo": []byte("bar"),
},
allowPatch: true,
},
{
name: "update_existing",
initial: map[string][]byte{
"foo": []byte("old"),
},
key: "foo",
value: []byte("new"),
wantData: map[string][]byte{
"foo": []byte("new"),
},
allowPatch: true,
},
{
name: "create_new_secret",
key: "foo",
value: []byte("bar"),
wantData: map[string][]byte{
"foo": []byte("bar"),
},
allowPatch: true,
},
{
name: "patch_denied",
initial: map[string][]byte{
"foo": []byte("old"),
},
key: "foo",
value: []byte("new"),
wantData: map[string][]byte{
"foo": []byte("new"),
},
allowPatch: false,
},
{
name: "sanitize_key",
initial: map[string][]byte{
"clean-key": []byte("old"),
},
key: "dirty@key",
value: []byte("new"),
wantData: map[string][]byte{
"clean-key": []byte("old"),
"dirty_key": []byte("new"),
},
allowPatch: true,
},
{
name: "delete_with_patch",
initial: map[string][]byte{
"foo": []byte("bar"),
"baz": []byte("quux"),
},
key: "foo",
value: nil,
wantData: map[string][]byte{
"baz": []byte("quux"),
},
allowPatch: true,
},
{
name: "delete_with_update",
initial: map[string][]byte{
"foo": []byte("bar"),
"baz": []byte("quux"),
},
key: "foo",
value: nil,
wantData: map[string][]byte{
"baz": []byte("quux"),
},
allowPatch: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
secret := tt.initial // track current state
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if secret == nil {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return tt.allowPatch, true, nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
secret = s.Data
return nil
},
UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
secret = s.Data
return nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
if !tt.allowPatch {
return &kubeapi.Status{Reason: "Forbidden"}
}
if secret == nil {
secret = make(map[string][]byte)
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
secret[key] = p.Value.([]byte)
} else if p.Op == "remove" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
delete(secret, key)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: tt.allowPatch,
secretName: "ts-state",
memory: mem.Store{},
}
err := s.WriteState(tt.key, tt.value)
if err != nil {
t.Errorf("WriteState() error = %v", err)
return
}
// Verify secret data
if diff := cmp.Diff(secret, tt.wantData); diff != "" {
t.Errorf("secret data mismatch (-got +want):\n%s", diff)
}
// Verify memory store was updated
got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(string(tt.key))))
if tt.value == nil {
if err != ipn.ErrStateNotExist {
t.Errorf("reading deleted key from memory store: got err %v, want ErrStateNotExist", err)
}
} else {
if err != nil {
t.Errorf("reading from memory store: %v", err)
}
if !cmp.Equal(got, tt.value) {
t.Errorf("memory store key %q = %v, want %v", tt.key, got, tt.value)
}
}
})
}
}
func TestWriteTLSCertAndKey(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
testCert = "fake-cert"
testKey = "fake-key"
)
tests := []struct {
name string
initial map[string][]byte // pre-existing cert and key
certShareMode string
allowPatch bool // whether client can patch the Secret
wantSecretName string // name of the Secret where cert and key should be written
wantSecretData map[string][]byte
wantMemoryStore map[ipn.StateKey][]byte
}{
{
name: "basic_write",
initial: map[string][]byte{
"existing": []byte("old"),
},
allowPatch: true,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"existing": []byte("old"),
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_mode_write",
certShareMode: "rw",
allowPatch: true,
wantSecretName: "my-app.tailnetxyz.ts.net",
wantSecretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
},
{
name: "cert_share_mode_write_update_existing",
initial: map[string][]byte{
"tls.crt": []byte("old-cert"),
"tls.key": []byte("old-key"),
},
certShareMode: "rw",
allowPatch: true,
wantSecretName: "my-app.tailnetxyz.ts.net",
wantSecretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
},
{
name: "update_existing",
initial: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte("old-cert"),
"my-app.tailnetxyz.ts.net.key": []byte("old-key"),
},
certShareMode: "",
allowPatch: true,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "patch_denied",
certShareMode: "",
allowPatch: false,
wantSecretName: "ts-state",
wantSecretData: map[string][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Set POD_NAME for testing selectors
envknob.Setenv("POD_NAME", "ingress-proxies-1")
defer envknob.Setenv("POD_NAME", "")
secret := tt.initial // track current state
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if secret == nil {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return tt.allowPatch, true, nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
if s.Name != tt.wantSecretName {
t.Errorf("CreateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
}
secret = s.Data
return nil
},
UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
if s.Name != tt.wantSecretName {
t.Errorf("UpdateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName)
}
secret = s.Data
return nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
if !tt.allowPatch {
return &kubeapi.Status{Reason: "Forbidden"}
}
if name != tt.wantSecretName {
t.Errorf("JSONPatchResource called with wrong name, got %q, want %q", name, tt.wantSecretName)
}
if secret == nil {
secret = make(map[string][]byte)
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
secret[key] = p.Value.([]byte)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: tt.allowPatch,
secretName: tt.wantSecretName,
certShareMode: tt.certShareMode,
memory: mem.Store{},
}
err := s.WriteTLSCertAndKey(testDomain, []byte(testCert), []byte(testKey))
if err != nil {
t.Errorf("WriteTLSCertAndKey() error = '%v'", err)
return
}
// Verify secret data
if diff := cmp.Diff(secret, tt.wantSecretData); diff != "" {
t.Errorf("secret data mismatch (-got +want):\n%s", diff)
}
// Verify memory store was updated
for key, want := range tt.wantMemoryStore {
got, err := s.memory.ReadState(key)
if err != nil {
t.Errorf("reading from memory store: %v", err)
continue
}
if !cmp.Equal(got, want) {
t.Errorf("memory store key %q = %v, want %v", key, got, want)
}
}
})
}
}
func TestReadTLSCertAndKey(t *testing.T) {
const (
testDomain = "my-app.tailnetxyz.ts.net"
testCert = "fake-cert"
testKey = "fake-key"
)
tests := []struct {
name string
memoryStore map[ipn.StateKey][]byte // pre-existing memory store state
certShareMode string
domain string
secretData map[string][]byte // data to return from mock GetSecret
secretGetErr error // error to return from mock GetSecret
wantCert []byte
wantKey []byte
wantErr error
// what should end up in memory store after the store is created
wantMemoryStore map[ipn.StateKey][]byte
}{
{
name: "found_in_memory",
memoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
domain: testDomain,
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "not_found_in_memory",
domain: testDomain,
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_found_in_secret",
certShareMode: "ro",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_rw_mode_found_in_secret",
certShareMode: "rw",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": []byte(testCert),
"tls.key": []byte(testKey),
},
wantCert: []byte(testCert),
wantKey: []byte(testKey),
},
{
name: "cert_share_ro_mode_found_in_memory",
certShareMode: "ro",
memoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
domain: testDomain,
wantCert: []byte(testCert),
wantKey: []byte(testKey),
wantMemoryStore: map[ipn.StateKey][]byte{
"my-app.tailnetxyz.ts.net.crt": []byte(testCert),
"my-app.tailnetxyz.ts.net.key": []byte(testKey),
},
},
{
name: "cert_share_ro_mode_not_found",
certShareMode: "ro",
domain: testDomain,
secretGetErr: &kubeapi.Status{Code: 404},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_forbidden",
certShareMode: "ro",
domain: testDomain,
secretGetErr: &kubeapi.Status{Code: 403},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_empty_cert_in_secret",
certShareMode: "ro",
domain: testDomain,
secretData: map[string][]byte{
"tls.crt": {},
"tls.key": []byte(testKey),
},
wantErr: ipn.ErrStateNotExist,
},
{
name: "cert_share_ro_mode_kube_api_error",
certShareMode: "ro",
domain: testDomain,
secretGetErr: fmt.Errorf("api error"),
wantErr: fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
return &kubeapi.Secret{Data: tt.secretData}, nil
},
}
s := &Store{
client: client,
secretName: "ts-state",
certShareMode: tt.certShareMode,
memory: mem.Store{},
}
// Initialize memory store
for k, v := range tt.memoryStore {
s.memory.WriteState(k, v)
}
gotCert, gotKey, err := s.ReadTLSCertAndKey(tt.domain)
if tt.wantErr != nil {
if err == nil {
t.Errorf("ReadTLSCertAndKey() error = nil, want error containing %v", tt.wantErr)
return
}
if !strings.Contains(err.Error(), tt.wantErr.Error()) {
t.Errorf("ReadTLSCertAndKey() error = %v, want error containing %v", err, tt.wantErr)
}
return
}
if err != nil {
t.Errorf("ReadTLSCertAndKey() unexpected error: %v", err)
return
}
if !bytes.Equal(gotCert, tt.wantCert) {
t.Errorf("ReadTLSCertAndKey() gotCert = %v, want %v", gotCert, tt.wantCert)
}
if !bytes.Equal(gotKey, tt.wantKey) {
t.Errorf("ReadTLSCertAndKey() gotKey = %v, want %v", gotKey, tt.wantKey)
}
// Verify memory store contents after operation
if tt.wantMemoryStore != nil {
for key, want := range tt.wantMemoryStore {
got, err := s.memory.ReadState(key)
if err != nil {
t.Errorf("reading from memory store: %v", err)
continue
}
if !bytes.Equal(got, want) {
t.Errorf("memory store key %q = %v, want %v", key, got, want)
}
}
}
})
}
}
func TestNewWithClient(t *testing.T) {
const (
secretName = "ts-state"
testCert = "fake-cert"
testKey = "fake-key"
)
certSecretsLabels := map[string]string{
"tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts,
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "ingress-proxies",
}
// Helper function to create Secret objects for testing
makeSecret := func(name string, labels map[string]string, certSuffix string) kubeapi.Secret {
return kubeapi.Secret{
ObjectMeta: kubeapi.ObjectMeta{
Name: name,
Labels: labels,
},
Data: map[string][]byte{
"tls.crt": []byte(testCert + certSuffix),
"tls.key": []byte(testKey + certSuffix),
},
}
}
tests := []struct {
name string
stateSecretContents map[string][]byte // data in state Secret
TLSSecrets []kubeapi.Secret // list of TLS cert Secrets
certMode string
secretGetErr error // error to return from GetSecret
secretsListErr error // error to return from ListSecrets
wantMemoryStoreContents map[ipn.StateKey][]byte
wantErr error
}{
{
name: "empty_state_secret",
stateSecretContents: map[string][]byte{},
wantMemoryStoreContents: map[ipn.StateKey][]byte{},
},
{
name: "state_secret_not_found",
secretGetErr: &kubeapi.Status{Code: 404},
wantMemoryStoreContents: map[ipn.StateKey][]byte{},
},
{
name: "state_secret_get_error",
secretGetErr: fmt.Errorf("some error"),
wantErr: fmt.Errorf("error loading state from kube Secret: some error"),
},
{
name: "load_existing_state",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"baz": []byte("qux"),
},
},
{
name: "load_select_certs_in_read_only_mode",
certMode: "ro",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
makeSecret("some-other-secret", nil, "3"),
makeSecret("app3.other-proxies.ts.net", map[string]string{
"tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts,
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "some-other-proxygroup",
}, "4"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
"app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"),
"app1.tailnetxyz.ts.net.key": []byte(testKey + "1"),
"app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"),
"app2.tailnetxyz.ts.net.key": []byte(testKey + "2"),
},
},
{
name: "do_not_load_certs_in_read_write_mode",
certMode: "rw",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"),
makeSecret("some-other-secret", nil, "3"),
makeSecret("app3.other-proxies.ts.net", map[string]string{
"tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts,
"tailscale.com/managed": "true",
"tailscale.com/proxy-group": "some-other-proxygroup",
}, "4"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
{
name: "list_cert_secrets_fails",
certMode: "ro",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
secretsListErr: fmt.Errorf("list error"),
// The error is logged but not returned, and state is still loaded
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
{
name: "cert_secrets_not_loaded_when_not_in_share_mode",
certMode: "",
stateSecretContents: map[string][]byte{
"foo": []byte("bar"),
},
TLSSecrets: []kubeapi.Secret{
makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"),
},
wantMemoryStoreContents: map[ipn.StateKey][]byte{
"foo": []byte("bar"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
envknob.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
t.Setenv("POD_NAME", "ingress-proxies-1")
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.secretGetErr != nil {
return nil, tt.secretGetErr
}
if name == secretName {
return &kubeapi.Secret{Data: tt.stateSecretContents}, nil
}
return nil, &kubeapi.Status{Code: 404}
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
if tt.secretsListErr != nil {
return nil, tt.secretsListErr
}
var matchingSecrets []kubeapi.Secret
for _, secret := range tt.TLSSecrets {
matches := true
for k, v := range selector {
if secret.Labels[k] != v {
matches = false
break
}
}
if matches {
matchingSecrets = append(matchingSecrets, secret)
}
}
return &kubeapi.SecretList{Items: matchingSecrets}, nil
},
}
s, err := newWithClient(t.Logf, client, secretName)
if tt.wantErr != nil {
if err == nil {
t.Errorf("NewWithClient() error = nil, want error containing %v", tt.wantErr)
return
}
if !strings.Contains(err.Error(), tt.wantErr.Error()) {
t.Errorf("NewWithClient() error = %v, want error containing %v", err, tt.wantErr)
}
return
}
if err != nil {
t.Errorf("NewWithClient() unexpected error: %v", err)
return
}
// Verify memory store contents
gotJSON, err := s.memory.ExportToJSON()
if err != nil {
t.Errorf("ExportToJSON failed: %v", err)
return
}
var got map[ipn.StateKey][]byte
if err := json.Unmarshal(gotJSON, &got); err != nil {
t.Errorf("failed to unmarshal memory store JSON: %v", err)
return
}
want := tt.wantMemoryStoreContents
if want == nil {
want = map[ipn.StateKey][]byte{}
}
if diff := cmp.Diff(got, want); diff != "" {
t.Errorf("memory store contents mismatch (-got +want):\n%s", diff)
}
})
}
}
func TestSharedACMEAccountKey(t *testing.T) {
const (
stateSecretName = "ingress-proxies-0"
sharedSecretName = kubetypes.ACMEAccountsSecretName
sharedField = "my-tailnet" + kubetypes.ACMEAccountKeySuffix
)
existingKey := []byte("-----BEGIN PRIVATE KEY-----\nexisting\n-----END PRIVATE KEY-----")
freshKey := []byte("-----BEGIN PRIVATE KEY-----\nfresh\n-----END PRIVATE KEY-----")
tests := []struct {
name string
certMode string
stateSecret map[string][]byte
sharedSecret map[string][]byte // nil = Secret does not exist
envSecretName string
envField string
writeAfterInit []byte // if non-nil, call WriteState(acmeAccountStateKey, …) after init
wantMemoryACME []byte
wantSharedSecret map[string][]byte
wantStateSecret map[string][]byte // optional: when set, asserts state Secret was not touched on the ACME path
wantPreAdopted []byte // expected s.preAdoptedLocalKey (sha256 of pre-adoption local key)
}{
{
name: "adopts_shared_key_when_present",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
},
{
name: "shared_overrides_per_pod_key",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: freshKey, // would be stale
},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
wantPreAdopted: sha256Sum(freshKey),
},
{
name: "adopting_matching_local_leaves_preadopted_nil",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: existingKey, // matches shared
},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
},
{
name: "migrates_per_pod_key_when_shared_field_empty",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: existingKey,
},
sharedSecret: map[string][]byte{}, // exists but no field for this tailnet
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
},
{
name: "no_op_when_no_keys_anywhere",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{},
wantMemoryACME: nil,
wantSharedSecret: map[string][]byte{},
},
{
name: "ro_mode_ignores_env_vars",
certMode: "ro",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: freshKey,
},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: freshKey, // per-pod copy stays; shared Secret never consulted
},
{
name: "write_routes_to_shared_secret",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{},
writeAfterInit: freshKey,
wantMemoryACME: freshKey,
wantSharedSecret: map[string][]byte{sharedField: freshKey},
wantStateSecret: map[string][]byte{}, // state Secret untouched by the ACME write
},
{
name: "write_in_ro_mode_goes_to_state_secret",
certMode: "ro",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{},
writeAfterInit: freshKey,
wantMemoryACME: freshKey,
wantSharedSecret: map[string][]byte{}, // env vars ignored in ro mode
wantStateSecret: map[string][]byte{acmeAccountStateKey: freshKey},
},
{
name: "no_env_vars_routes_to_state_secret",
certMode: "rw",
envSecretName: "", // shared account disabled
envField: "",
stateSecret: map[string][]byte{
acmeAccountStateKey: existingKey,
},
sharedSecret: nil, // does not exist; should not be touched
wantMemoryACME: existingKey,
wantStateSecret: map[string][]byte{acmeAccountStateKey: existingKey},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
t.Setenv("TS_ACME_ACCOUNT_SECRET_NAME", tt.envSecretName)
t.Setenv("TS_ACME_ACCOUNT_FIELD", tt.envField)
t.Setenv("POD_NAME", stateSecretName)
stateData := cloneMap(tt.stateSecret)
sharedData := cloneMap(tt.sharedSecret)
sharedExists := tt.sharedSecret != nil
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
switch name {
case stateSecretName:
return &kubeapi.Secret{Data: stateData}, nil
case sharedSecretName:
if !sharedExists {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: sharedData}, nil
}
return nil, &kubeapi.Status{Code: 404}
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
var target *map[string][]byte
switch name {
case stateSecretName:
target = &stateData
case sharedSecretName:
target = &sharedData
sharedExists = true
default:
t.Errorf("unexpected patch target Secret %q", name)
return nil
}
if *target == nil {
*target = map[string][]byte{}
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
*target = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
(*target)[key] = p.Value.([]byte)
}
}
return nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
switch s.Name {
case stateSecretName:
stateData = s.Data
case sharedSecretName:
sharedData = s.Data
sharedExists = true
default:
t.Errorf("unexpected create target Secret %q", s.Name)
}
return nil
},
ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
// Used by ro-mode TLS Secret preload; irrelevant to this test.
return &kubeapi.SecretList{}, nil
},
}
s, err := newWithClient(t.Logf, client, stateSecretName)
if err != nil {
t.Fatalf("newWithClient: %v", err)
}
if tt.writeAfterInit != nil {
if err := s.WriteState(ipn.StateKey(acmeAccountStateKey), tt.writeAfterInit); err != nil {
t.Fatalf("WriteState(ACME key): %v", err)
}
}
gotMemACME, err := s.memory.ReadState(ipn.StateKey(acmeAccountStateKey))
if err != nil && tt.wantMemoryACME != nil {
t.Errorf("memory ReadState(ACME key): %v", err)
}
if !bytes.Equal(gotMemACME, tt.wantMemoryACME) {
t.Errorf("memory ACME key = %q, want %q", gotMemACME, tt.wantMemoryACME)
}
if tt.wantSharedSecret != nil {
if diff := cmp.Diff(sharedData, tt.wantSharedSecret); diff != "" {
t.Errorf("shared Secret contents mismatch (-got +want):\n%s", diff)
}
}
if tt.wantStateSecret != nil {
if diff := cmp.Diff(stateData, tt.wantStateSecret); diff != "" {
t.Errorf("state Secret contents mismatch (-got +want):\n%s", diff)
}
}
if !bytes.Equal(s.preAdoptedLocalKey, tt.wantPreAdopted) {
t.Errorf("preAdoptedLocalKey = %x, want %x", s.preAdoptedLocalKey, tt.wantPreAdopted)
}
})
}
}
func cloneMap(m map[string][]byte) map[string][]byte {
if m == nil {
return nil
}
out := make(map[string][]byte, len(m))
for k, v := range m {
out[k] = append([]byte(nil), v...)
}
return out
}
func sha256Sum(b []byte) []byte {
sum := sha256.Sum256(b)
return sum[:]
}
func TestShouldUseARIReplacesForRenewal(t *testing.T) {
const domain = "app.tailnetxyz.ts.net"
acmeKey := []byte("-----BEGIN PRIVATE KEY-----\ncurrent\n-----END PRIVATE KEY-----")
otherKey := []byte("-----BEGIN PRIVATE KEY-----\nother\n-----END PRIVATE KEY-----")
curFP := sha256Sum(acmeKey)
otherFP := sha256Sum(otherKey)
tests := []struct {
name string
certShareMode string
acmeInMemory []byte // per-pod ACME key present in memory
preAdopted []byte // sha256 of pre-adoption local key (foreign-key path)
certSecret map[string][]byte
certGetErr error
want bool
wantErr bool
}{
{
name: "non_rw_mode_returns_true",
certShareMode: "",
want: true,
},
{
name: "ro_mode_returns_true",
certShareMode: "ro",
want: true,
},
{
name: "no_acme_key_returns_true",
certShareMode: "rw",
want: true,
},
{
name: "cert_not_found_returns_true",
certShareMode: "rw",
acmeInMemory: acmeKey,
certGetErr: &kubeapi.Status{Code: 404},
want: true,
},
{
name: "cert_get_error_returns_true_with_err",
certShareMode: "rw",
acmeInMemory: acmeKey,
certGetErr: fmt.Errorf("api down"),
want: true,
wantErr: true,
},
{
name: "fingerprint_matches",
certShareMode: "rw",
acmeInMemory: acmeKey,
certSecret: map[string][]byte{keyACMEAcctFP: curFP},
want: true,
},
{
name: "fingerprint_differs",
certShareMode: "rw",
acmeInMemory: acmeKey,
certSecret: map[string][]byte{keyACMEAcctFP: otherFP},
want: false,
},
{
name: "legacy_cert_no_preadopted_returns_true",
certShareMode: "rw",
acmeInMemory: acmeKey,
certSecret: map[string][]byte{}, // no fingerprint field
want: true,
},
{
name: "legacy_cert_with_preadopted_returns_false",
certShareMode: "rw",
acmeInMemory: acmeKey,
preAdopted: sha256Sum(otherKey),
certSecret: map[string][]byte{}, // no fingerprint field
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.certGetErr != nil {
return nil, tt.certGetErr
}
return &kubeapi.Secret{Data: tt.certSecret}, nil
},
}
s := &Store{
client: client,
certShareMode: tt.certShareMode,
memory: mem.Store{},
preAdoptedLocalKey: tt.preAdopted,
logf: t.Logf,
}
if len(tt.acmeInMemory) > 0 {
s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), tt.acmeInMemory)
}
got, err := s.ShouldUseARIReplacesForRenewal(domain)
if (err != nil) != tt.wantErr {
t.Errorf("err = %v, wantErr = %v", err, tt.wantErr)
}
if got != tt.want {
t.Errorf("got %v, want %v", got, tt.want)
}
})
}
}
func TestWriteTLSCertAndKeyStampsFingerprint(t *testing.T) {
const domain = "app.tailnetxyz.ts.net"
acmeKey := []byte("-----BEGIN PRIVATE KEY-----\naccount\n-----END PRIVATE KEY-----")
wantFP := sha256Sum(acmeKey)
tests := []struct {
name string
certMode string
acmeInMemory []byte
wantFP []byte // expected value of keyACMEAcctFP field; nil means field must be absent
}{
{
name: "rw_mode_with_acme_key_stamps_fingerprint",
certMode: "rw",
acmeInMemory: acmeKey,
wantFP: wantFP,
},
{
name: "rw_mode_no_acme_key_no_stamp",
certMode: "rw",
},
{
name: "non_share_mode_no_stamp",
certMode: "",
acmeInMemory: acmeKey,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
secret := map[string][]byte{}
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
secret[strings.TrimPrefix(p.Path, "/data/")] = p.Value.([]byte)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: true,
secretName: "ts-state",
certShareMode: tt.certMode,
memory: mem.Store{},
logf: t.Logf,
}
if len(tt.acmeInMemory) > 0 {
s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), tt.acmeInMemory)
}
if err := s.WriteTLSCertAndKey(domain, []byte("cert"), []byte("key")); err != nil {
t.Fatalf("WriteTLSCertAndKey: %v", err)
}
gotFP, present := secret[keyACMEAcctFP]
if tt.wantFP == nil {
if present {
t.Errorf("unexpected fingerprint stamped: %x", gotFP)
}
return
}
if !bytes.Equal(gotFP, tt.wantFP) {
t.Errorf("fingerprint = %x, want %x", gotFP, tt.wantFP)
}
})
}
}