// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause package kubestore import ( "bytes" "context" "crypto/sha256" "encoding/json" "fmt" "strings" "testing" "github.com/google/go-cmp/cmp" "tailscale.com/envknob" "tailscale.com/ipn" "tailscale.com/ipn/store/mem" "tailscale.com/kube/kubeapi" "tailscale.com/kube/kubeclient" "tailscale.com/kube/kubetypes" ) func TestKubernetesPodMigrationWithTPMAttestationKey(t *testing.T) { stateWithAttestationKey := `{ "Config": { "NodeID": "nSTABLE123456", "AttestationKey": { "tpmPrivate": "c2Vuc2l0aXZlLXRwbS1kYXRhLXRoYXQtb25seS13b3Jrcy1vbi1vcmlnaW5hbC1ub2Rl", "tpmPublic": "cHVibGljLXRwbS1kYXRhLWZvci1hdHRlc3RhdGlvbi1rZXk=" } } }` secretData := map[string][]byte{ "profile-abc123": []byte(stateWithAttestationKey), "_current-profile": []byte("profile-abc123"), } client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { return &kubeapi.Secret{Data: secretData}, nil }, CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) { return true, true, nil }, JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error { for _, p := range patches { if p.Op == "add" && p.Path == "/data" { secretData = p.Value.(map[string][]byte) } } return nil }, } store := &Store{ client: client, canPatch: true, secretName: "ts-state", memory: mem.Store{}, logf: t.Logf, } if err := store.loadState(); err != nil { t.Fatalf("loadState failed: %v", err) } // Verify we can read the state from the store stateBytes, err := store.ReadState("profile-abc123") if err != nil { t.Fatalf("ReadState failed: %v", err) } // The state should be readable as JSON var state map[string]json.RawMessage if err := json.Unmarshal(stateBytes, &state); err != nil { t.Fatalf("failed to unmarshal state: %v", err) } // Verify the Config field exists configRaw, ok := state["Config"] if !ok { t.Fatal("Config field not found in state") } // Parse the Config to verify fields are preserved var config map[string]json.RawMessage if err := json.Unmarshal(configRaw, &config); err != nil { t.Fatalf("failed to unmarshal Config: %v", err) } // The AttestationKey should be stripped by the kubestore if _, hasAttestation := config["AttestationKey"]; hasAttestation { t.Error("AttestationKey should be stripped from state loaded by kubestore") } // Verify other fields are preserved var nodeID string if err := json.Unmarshal(config["NodeID"], &nodeID); err != nil { t.Fatalf("failed to unmarshal NodeID: %v", err) } if nodeID != "nSTABLE123456" { t.Errorf("NodeID mismatch: got %q, want %q", nodeID, "nSTABLE123456") } } func TestWriteState(t *testing.T) { tests := []struct { name string initial map[string][]byte key ipn.StateKey value []byte wantData map[string][]byte allowPatch bool }{ { name: "basic_write", initial: map[string][]byte{ "existing": []byte("old"), }, key: "foo", value: []byte("bar"), wantData: map[string][]byte{ "existing": []byte("old"), "foo": []byte("bar"), }, allowPatch: true, }, { name: "update_existing", initial: map[string][]byte{ "foo": []byte("old"), }, key: "foo", value: []byte("new"), wantData: map[string][]byte{ "foo": []byte("new"), }, allowPatch: true, }, { name: "create_new_secret", key: "foo", value: []byte("bar"), wantData: map[string][]byte{ "foo": []byte("bar"), }, allowPatch: true, }, { name: "patch_denied", initial: map[string][]byte{ "foo": []byte("old"), }, key: "foo", value: []byte("new"), wantData: map[string][]byte{ "foo": []byte("new"), }, allowPatch: false, }, { name: "sanitize_key", initial: map[string][]byte{ "clean-key": []byte("old"), }, key: "dirty@key", value: []byte("new"), wantData: map[string][]byte{ "clean-key": []byte("old"), "dirty_key": []byte("new"), }, allowPatch: true, }, { name: "delete_with_patch", initial: map[string][]byte{ "foo": []byte("bar"), "baz": []byte("quux"), }, key: "foo", value: nil, wantData: map[string][]byte{ "baz": []byte("quux"), }, allowPatch: true, }, { name: "delete_with_update", initial: map[string][]byte{ "foo": []byte("bar"), "baz": []byte("quux"), }, key: "foo", value: nil, wantData: map[string][]byte{ "baz": []byte("quux"), }, allowPatch: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { secret := tt.initial // track current state client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { if secret == nil { return nil, &kubeapi.Status{Code: 404} } return &kubeapi.Secret{Data: secret}, nil }, CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) { return tt.allowPatch, true, nil }, CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error { secret = s.Data return nil }, UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error { secret = s.Data return nil }, JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error { if !tt.allowPatch { return &kubeapi.Status{Reason: "Forbidden"} } if secret == nil { secret = make(map[string][]byte) } for _, p := range patches { if p.Op == "add" && p.Path == "/data" { secret = p.Value.(map[string][]byte) } else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") { key := strings.TrimPrefix(p.Path, "/data/") secret[key] = p.Value.([]byte) } else if p.Op == "remove" && strings.HasPrefix(p.Path, "/data/") { key := strings.TrimPrefix(p.Path, "/data/") delete(secret, key) } } return nil }, } s := &Store{ client: client, canPatch: tt.allowPatch, secretName: "ts-state", memory: mem.Store{}, } err := s.WriteState(tt.key, tt.value) if err != nil { t.Errorf("WriteState() error = %v", err) return } // Verify secret data if diff := cmp.Diff(secret, tt.wantData); diff != "" { t.Errorf("secret data mismatch (-got +want):\n%s", diff) } // Verify memory store was updated got, err := s.memory.ReadState(ipn.StateKey(sanitizeKey(string(tt.key)))) if tt.value == nil { if err != ipn.ErrStateNotExist { t.Errorf("reading deleted key from memory store: got err %v, want ErrStateNotExist", err) } } else { if err != nil { t.Errorf("reading from memory store: %v", err) } if !cmp.Equal(got, tt.value) { t.Errorf("memory store key %q = %v, want %v", tt.key, got, tt.value) } } }) } } func TestWriteTLSCertAndKey(t *testing.T) { const ( testDomain = "my-app.tailnetxyz.ts.net" testCert = "fake-cert" testKey = "fake-key" ) tests := []struct { name string initial map[string][]byte // pre-existing cert and key certShareMode string allowPatch bool // whether client can patch the Secret wantSecretName string // name of the Secret where cert and key should be written wantSecretData map[string][]byte wantMemoryStore map[ipn.StateKey][]byte }{ { name: "basic_write", initial: map[string][]byte{ "existing": []byte("old"), }, allowPatch: true, wantSecretName: "ts-state", wantSecretData: map[string][]byte{ "existing": []byte("old"), "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, wantMemoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, }, { name: "cert_share_mode_write", certShareMode: "rw", allowPatch: true, wantSecretName: "my-app.tailnetxyz.ts.net", wantSecretData: map[string][]byte{ "tls.crt": []byte(testCert), "tls.key": []byte(testKey), }, }, { name: "cert_share_mode_write_update_existing", initial: map[string][]byte{ "tls.crt": []byte("old-cert"), "tls.key": []byte("old-key"), }, certShareMode: "rw", allowPatch: true, wantSecretName: "my-app.tailnetxyz.ts.net", wantSecretData: map[string][]byte{ "tls.crt": []byte(testCert), "tls.key": []byte(testKey), }, }, { name: "update_existing", initial: map[string][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte("old-cert"), "my-app.tailnetxyz.ts.net.key": []byte("old-key"), }, certShareMode: "", allowPatch: true, wantSecretName: "ts-state", wantSecretData: map[string][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, wantMemoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, }, { name: "patch_denied", certShareMode: "", allowPatch: false, wantSecretName: "ts-state", wantSecretData: map[string][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, wantMemoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { // Set POD_NAME for testing selectors envknob.Setenv("POD_NAME", "ingress-proxies-1") defer envknob.Setenv("POD_NAME", "") secret := tt.initial // track current state client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { if secret == nil { return nil, &kubeapi.Status{Code: 404} } return &kubeapi.Secret{Data: secret}, nil }, CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) { return tt.allowPatch, true, nil }, CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error { if s.Name != tt.wantSecretName { t.Errorf("CreateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName) } secret = s.Data return nil }, UpdateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error { if s.Name != tt.wantSecretName { t.Errorf("UpdateSecret called with wrong name, got %q, want %q", s.Name, tt.wantSecretName) } secret = s.Data return nil }, JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error { if !tt.allowPatch { return &kubeapi.Status{Reason: "Forbidden"} } if name != tt.wantSecretName { t.Errorf("JSONPatchResource called with wrong name, got %q, want %q", name, tt.wantSecretName) } if secret == nil { secret = make(map[string][]byte) } for _, p := range patches { if p.Op == "add" && p.Path == "/data" { secret = p.Value.(map[string][]byte) } else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") { key := strings.TrimPrefix(p.Path, "/data/") secret[key] = p.Value.([]byte) } } return nil }, } s := &Store{ client: client, canPatch: tt.allowPatch, secretName: tt.wantSecretName, certShareMode: tt.certShareMode, memory: mem.Store{}, } err := s.WriteTLSCertAndKey(testDomain, []byte(testCert), []byte(testKey)) if err != nil { t.Errorf("WriteTLSCertAndKey() error = '%v'", err) return } // Verify secret data if diff := cmp.Diff(secret, tt.wantSecretData); diff != "" { t.Errorf("secret data mismatch (-got +want):\n%s", diff) } // Verify memory store was updated for key, want := range tt.wantMemoryStore { got, err := s.memory.ReadState(key) if err != nil { t.Errorf("reading from memory store: %v", err) continue } if !cmp.Equal(got, want) { t.Errorf("memory store key %q = %v, want %v", key, got, want) } } }) } } func TestReadTLSCertAndKey(t *testing.T) { const ( testDomain = "my-app.tailnetxyz.ts.net" testCert = "fake-cert" testKey = "fake-key" ) tests := []struct { name string memoryStore map[ipn.StateKey][]byte // pre-existing memory store state certShareMode string domain string secretData map[string][]byte // data to return from mock GetSecret secretGetErr error // error to return from mock GetSecret wantCert []byte wantKey []byte wantErr error // what should end up in memory store after the store is created wantMemoryStore map[ipn.StateKey][]byte }{ { name: "found_in_memory", memoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, domain: testDomain, wantCert: []byte(testCert), wantKey: []byte(testKey), wantMemoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, }, { name: "not_found_in_memory", domain: testDomain, wantErr: ipn.ErrStateNotExist, }, { name: "cert_share_ro_mode_found_in_secret", certShareMode: "ro", domain: testDomain, secretData: map[string][]byte{ "tls.crt": []byte(testCert), "tls.key": []byte(testKey), }, wantCert: []byte(testCert), wantKey: []byte(testKey), wantMemoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, }, { name: "cert_share_rw_mode_found_in_secret", certShareMode: "rw", domain: testDomain, secretData: map[string][]byte{ "tls.crt": []byte(testCert), "tls.key": []byte(testKey), }, wantCert: []byte(testCert), wantKey: []byte(testKey), }, { name: "cert_share_ro_mode_found_in_memory", certShareMode: "ro", memoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, domain: testDomain, wantCert: []byte(testCert), wantKey: []byte(testKey), wantMemoryStore: map[ipn.StateKey][]byte{ "my-app.tailnetxyz.ts.net.crt": []byte(testCert), "my-app.tailnetxyz.ts.net.key": []byte(testKey), }, }, { name: "cert_share_ro_mode_not_found", certShareMode: "ro", domain: testDomain, secretGetErr: &kubeapi.Status{Code: 404}, wantErr: ipn.ErrStateNotExist, }, { name: "cert_share_ro_mode_forbidden", certShareMode: "ro", domain: testDomain, secretGetErr: &kubeapi.Status{Code: 403}, wantErr: ipn.ErrStateNotExist, }, { name: "cert_share_ro_mode_empty_cert_in_secret", certShareMode: "ro", domain: testDomain, secretData: map[string][]byte{ "tls.crt": {}, "tls.key": []byte(testKey), }, wantErr: ipn.ErrStateNotExist, }, { name: "cert_share_ro_mode_kube_api_error", certShareMode: "ro", domain: testDomain, secretGetErr: fmt.Errorf("api error"), wantErr: fmt.Errorf("getting TLS Secret %q: api error", sanitizeKey(testDomain)), }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { if tt.secretGetErr != nil { return nil, tt.secretGetErr } return &kubeapi.Secret{Data: tt.secretData}, nil }, } s := &Store{ client: client, secretName: "ts-state", certShareMode: tt.certShareMode, memory: mem.Store{}, } // Initialize memory store for k, v := range tt.memoryStore { s.memory.WriteState(k, v) } gotCert, gotKey, err := s.ReadTLSCertAndKey(tt.domain) if tt.wantErr != nil { if err == nil { t.Errorf("ReadTLSCertAndKey() error = nil, want error containing %v", tt.wantErr) return } if !strings.Contains(err.Error(), tt.wantErr.Error()) { t.Errorf("ReadTLSCertAndKey() error = %v, want error containing %v", err, tt.wantErr) } return } if err != nil { t.Errorf("ReadTLSCertAndKey() unexpected error: %v", err) return } if !bytes.Equal(gotCert, tt.wantCert) { t.Errorf("ReadTLSCertAndKey() gotCert = %v, want %v", gotCert, tt.wantCert) } if !bytes.Equal(gotKey, tt.wantKey) { t.Errorf("ReadTLSCertAndKey() gotKey = %v, want %v", gotKey, tt.wantKey) } // Verify memory store contents after operation if tt.wantMemoryStore != nil { for key, want := range tt.wantMemoryStore { got, err := s.memory.ReadState(key) if err != nil { t.Errorf("reading from memory store: %v", err) continue } if !bytes.Equal(got, want) { t.Errorf("memory store key %q = %v, want %v", key, got, want) } } } }) } } func TestNewWithClient(t *testing.T) { const ( secretName = "ts-state" testCert = "fake-cert" testKey = "fake-key" ) certSecretsLabels := map[string]string{ "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "ingress-proxies", } // Helper function to create Secret objects for testing makeSecret := func(name string, labels map[string]string, certSuffix string) kubeapi.Secret { return kubeapi.Secret{ ObjectMeta: kubeapi.ObjectMeta{ Name: name, Labels: labels, }, Data: map[string][]byte{ "tls.crt": []byte(testCert + certSuffix), "tls.key": []byte(testKey + certSuffix), }, } } tests := []struct { name string stateSecretContents map[string][]byte // data in state Secret TLSSecrets []kubeapi.Secret // list of TLS cert Secrets certMode string secretGetErr error // error to return from GetSecret secretsListErr error // error to return from ListSecrets wantMemoryStoreContents map[ipn.StateKey][]byte wantErr error }{ { name: "empty_state_secret", stateSecretContents: map[string][]byte{}, wantMemoryStoreContents: map[ipn.StateKey][]byte{}, }, { name: "state_secret_not_found", secretGetErr: &kubeapi.Status{Code: 404}, wantMemoryStoreContents: map[ipn.StateKey][]byte{}, }, { name: "state_secret_get_error", secretGetErr: fmt.Errorf("some error"), wantErr: fmt.Errorf("error loading state from kube Secret: some error"), }, { name: "load_existing_state", stateSecretContents: map[string][]byte{ "foo": []byte("bar"), "baz": []byte("qux"), }, wantMemoryStoreContents: map[ipn.StateKey][]byte{ "foo": []byte("bar"), "baz": []byte("qux"), }, }, { name: "load_select_certs_in_read_only_mode", certMode: "ro", stateSecretContents: map[string][]byte{ "foo": []byte("bar"), }, TLSSecrets: []kubeapi.Secret{ makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"), makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"), makeSecret("some-other-secret", nil, "3"), makeSecret("app3.other-proxies.ts.net", map[string]string{ "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "some-other-proxygroup", }, "4"), }, wantMemoryStoreContents: map[ipn.StateKey][]byte{ "foo": []byte("bar"), "app1.tailnetxyz.ts.net.crt": []byte(testCert + "1"), "app1.tailnetxyz.ts.net.key": []byte(testKey + "1"), "app2.tailnetxyz.ts.net.crt": []byte(testCert + "2"), "app2.tailnetxyz.ts.net.key": []byte(testKey + "2"), }, }, { name: "do_not_load_certs_in_read_write_mode", certMode: "rw", stateSecretContents: map[string][]byte{ "foo": []byte("bar"), }, TLSSecrets: []kubeapi.Secret{ makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"), makeSecret("app2.tailnetxyz.ts.net", certSecretsLabels, "2"), makeSecret("some-other-secret", nil, "3"), makeSecret("app3.other-proxies.ts.net", map[string]string{ "tailscale.com/secret-type": kubetypes.LabelSecretTypeCerts, "tailscale.com/managed": "true", "tailscale.com/proxy-group": "some-other-proxygroup", }, "4"), }, wantMemoryStoreContents: map[ipn.StateKey][]byte{ "foo": []byte("bar"), }, }, { name: "list_cert_secrets_fails", certMode: "ro", stateSecretContents: map[string][]byte{ "foo": []byte("bar"), }, secretsListErr: fmt.Errorf("list error"), // The error is logged but not returned, and state is still loaded wantMemoryStoreContents: map[ipn.StateKey][]byte{ "foo": []byte("bar"), }, }, { name: "cert_secrets_not_loaded_when_not_in_share_mode", certMode: "", stateSecretContents: map[string][]byte{ "foo": []byte("bar"), }, TLSSecrets: []kubeapi.Secret{ makeSecret("app1.tailnetxyz.ts.net", certSecretsLabels, "1"), }, wantMemoryStoreContents: map[ipn.StateKey][]byte{ "foo": []byte("bar"), }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { envknob.Setenv("TS_CERT_SHARE_MODE", tt.certMode) t.Setenv("POD_NAME", "ingress-proxies-1") client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { if tt.secretGetErr != nil { return nil, tt.secretGetErr } if name == secretName { return &kubeapi.Secret{Data: tt.stateSecretContents}, nil } return nil, &kubeapi.Status{Code: 404} }, CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) { return true, true, nil }, ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) { if tt.secretsListErr != nil { return nil, tt.secretsListErr } var matchingSecrets []kubeapi.Secret for _, secret := range tt.TLSSecrets { matches := true for k, v := range selector { if secret.Labels[k] != v { matches = false break } } if matches { matchingSecrets = append(matchingSecrets, secret) } } return &kubeapi.SecretList{Items: matchingSecrets}, nil }, } s, err := newWithClient(t.Logf, client, secretName) if tt.wantErr != nil { if err == nil { t.Errorf("NewWithClient() error = nil, want error containing %v", tt.wantErr) return } if !strings.Contains(err.Error(), tt.wantErr.Error()) { t.Errorf("NewWithClient() error = %v, want error containing %v", err, tt.wantErr) } return } if err != nil { t.Errorf("NewWithClient() unexpected error: %v", err) return } // Verify memory store contents gotJSON, err := s.memory.ExportToJSON() if err != nil { t.Errorf("ExportToJSON failed: %v", err) return } var got map[ipn.StateKey][]byte if err := json.Unmarshal(gotJSON, &got); err != nil { t.Errorf("failed to unmarshal memory store JSON: %v", err) return } want := tt.wantMemoryStoreContents if want == nil { want = map[ipn.StateKey][]byte{} } if diff := cmp.Diff(got, want); diff != "" { t.Errorf("memory store contents mismatch (-got +want):\n%s", diff) } }) } } func TestSharedACMEAccountKey(t *testing.T) { const ( stateSecretName = "ingress-proxies-0" sharedSecretName = kubetypes.ACMEAccountsSecretName sharedField = "my-tailnet" + kubetypes.ACMEAccountKeySuffix ) existingKey := []byte("-----BEGIN PRIVATE KEY-----\nexisting\n-----END PRIVATE KEY-----") freshKey := []byte("-----BEGIN PRIVATE KEY-----\nfresh\n-----END PRIVATE KEY-----") tests := []struct { name string certMode string stateSecret map[string][]byte sharedSecret map[string][]byte // nil = Secret does not exist envSecretName string envField string writeAfterInit []byte // if non-nil, call WriteState(acmeAccountStateKey, …) after init wantMemoryACME []byte wantSharedSecret map[string][]byte wantStateSecret map[string][]byte // optional: when set, asserts state Secret was not touched on the ACME path wantPreAdopted []byte // expected s.preAdoptedLocalKey (sha256 of pre-adoption local key) }{ { name: "adopts_shared_key_when_present", certMode: "rw", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{}, sharedSecret: map[string][]byte{sharedField: existingKey}, wantMemoryACME: existingKey, wantSharedSecret: map[string][]byte{sharedField: existingKey}, }, { name: "shared_overrides_per_pod_key", certMode: "rw", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{ acmeAccountStateKey: freshKey, // would be stale }, sharedSecret: map[string][]byte{sharedField: existingKey}, wantMemoryACME: existingKey, wantSharedSecret: map[string][]byte{sharedField: existingKey}, wantPreAdopted: sha256Sum(freshKey), }, { name: "adopting_matching_local_leaves_preadopted_nil", certMode: "rw", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{ acmeAccountStateKey: existingKey, // matches shared }, sharedSecret: map[string][]byte{sharedField: existingKey}, wantMemoryACME: existingKey, wantSharedSecret: map[string][]byte{sharedField: existingKey}, }, { name: "migrates_per_pod_key_when_shared_field_empty", certMode: "rw", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{ acmeAccountStateKey: existingKey, }, sharedSecret: map[string][]byte{}, // exists but no field for this tailnet wantMemoryACME: existingKey, wantSharedSecret: map[string][]byte{sharedField: existingKey}, }, { name: "no_op_when_no_keys_anywhere", certMode: "rw", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{}, sharedSecret: map[string][]byte{}, wantMemoryACME: nil, wantSharedSecret: map[string][]byte{}, }, { name: "ro_mode_ignores_env_vars", certMode: "ro", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{ acmeAccountStateKey: freshKey, }, sharedSecret: map[string][]byte{sharedField: existingKey}, wantMemoryACME: freshKey, // per-pod copy stays; shared Secret never consulted }, { name: "write_routes_to_shared_secret", certMode: "rw", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{}, sharedSecret: map[string][]byte{}, writeAfterInit: freshKey, wantMemoryACME: freshKey, wantSharedSecret: map[string][]byte{sharedField: freshKey}, wantStateSecret: map[string][]byte{}, // state Secret untouched by the ACME write }, { name: "write_in_ro_mode_goes_to_state_secret", certMode: "ro", envSecretName: sharedSecretName, envField: sharedField, stateSecret: map[string][]byte{}, sharedSecret: map[string][]byte{}, writeAfterInit: freshKey, wantMemoryACME: freshKey, wantSharedSecret: map[string][]byte{}, // env vars ignored in ro mode wantStateSecret: map[string][]byte{acmeAccountStateKey: freshKey}, }, { name: "no_env_vars_routes_to_state_secret", certMode: "rw", envSecretName: "", // shared account disabled envField: "", stateSecret: map[string][]byte{ acmeAccountStateKey: existingKey, }, sharedSecret: nil, // does not exist; should not be touched wantMemoryACME: existingKey, wantStateSecret: map[string][]byte{acmeAccountStateKey: existingKey}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Setenv("TS_CERT_SHARE_MODE", tt.certMode) t.Setenv("TS_ACME_ACCOUNT_SECRET_NAME", tt.envSecretName) t.Setenv("TS_ACME_ACCOUNT_FIELD", tt.envField) t.Setenv("POD_NAME", stateSecretName) stateData := cloneMap(tt.stateSecret) sharedData := cloneMap(tt.sharedSecret) sharedExists := tt.sharedSecret != nil client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { switch name { case stateSecretName: return &kubeapi.Secret{Data: stateData}, nil case sharedSecretName: if !sharedExists { return nil, &kubeapi.Status{Code: 404} } return &kubeapi.Secret{Data: sharedData}, nil } return nil, &kubeapi.Status{Code: 404} }, CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) { return true, true, nil }, JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error { var target *map[string][]byte switch name { case stateSecretName: target = &stateData case sharedSecretName: target = &sharedData sharedExists = true default: t.Errorf("unexpected patch target Secret %q", name) return nil } if *target == nil { *target = map[string][]byte{} } for _, p := range patches { if p.Op == "add" && p.Path == "/data" { *target = p.Value.(map[string][]byte) } else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") { key := strings.TrimPrefix(p.Path, "/data/") (*target)[key] = p.Value.([]byte) } } return nil }, CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error { switch s.Name { case stateSecretName: stateData = s.Data case sharedSecretName: sharedData = s.Data sharedExists = true default: t.Errorf("unexpected create target Secret %q", s.Name) } return nil }, ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) { // Used by ro-mode TLS Secret preload; irrelevant to this test. return &kubeapi.SecretList{}, nil }, } s, err := newWithClient(t.Logf, client, stateSecretName) if err != nil { t.Fatalf("newWithClient: %v", err) } if tt.writeAfterInit != nil { if err := s.WriteState(ipn.StateKey(acmeAccountStateKey), tt.writeAfterInit); err != nil { t.Fatalf("WriteState(ACME key): %v", err) } } gotMemACME, err := s.memory.ReadState(ipn.StateKey(acmeAccountStateKey)) if err != nil && tt.wantMemoryACME != nil { t.Errorf("memory ReadState(ACME key): %v", err) } if !bytes.Equal(gotMemACME, tt.wantMemoryACME) { t.Errorf("memory ACME key = %q, want %q", gotMemACME, tt.wantMemoryACME) } if tt.wantSharedSecret != nil { if diff := cmp.Diff(sharedData, tt.wantSharedSecret); diff != "" { t.Errorf("shared Secret contents mismatch (-got +want):\n%s", diff) } } if tt.wantStateSecret != nil { if diff := cmp.Diff(stateData, tt.wantStateSecret); diff != "" { t.Errorf("state Secret contents mismatch (-got +want):\n%s", diff) } } if !bytes.Equal(s.preAdoptedLocalKey, tt.wantPreAdopted) { t.Errorf("preAdoptedLocalKey = %x, want %x", s.preAdoptedLocalKey, tt.wantPreAdopted) } }) } } func cloneMap(m map[string][]byte) map[string][]byte { if m == nil { return nil } out := make(map[string][]byte, len(m)) for k, v := range m { out[k] = append([]byte(nil), v...) } return out } func sha256Sum(b []byte) []byte { sum := sha256.Sum256(b) return sum[:] } func TestShouldUseARIReplacesForRenewal(t *testing.T) { const domain = "app.tailnetxyz.ts.net" acmeKey := []byte("-----BEGIN PRIVATE KEY-----\ncurrent\n-----END PRIVATE KEY-----") otherKey := []byte("-----BEGIN PRIVATE KEY-----\nother\n-----END PRIVATE KEY-----") curFP := sha256Sum(acmeKey) otherFP := sha256Sum(otherKey) tests := []struct { name string certShareMode string acmeInMemory []byte // per-pod ACME key present in memory preAdopted []byte // sha256 of pre-adoption local key (foreign-key path) certSecret map[string][]byte certGetErr error want bool wantErr bool }{ { name: "non_rw_mode_returns_true", certShareMode: "", want: true, }, { name: "ro_mode_returns_true", certShareMode: "ro", want: true, }, { name: "no_acme_key_returns_true", certShareMode: "rw", want: true, }, { name: "cert_not_found_returns_true", certShareMode: "rw", acmeInMemory: acmeKey, certGetErr: &kubeapi.Status{Code: 404}, want: true, }, { name: "cert_get_error_returns_true_with_err", certShareMode: "rw", acmeInMemory: acmeKey, certGetErr: fmt.Errorf("api down"), want: true, wantErr: true, }, { name: "fingerprint_matches", certShareMode: "rw", acmeInMemory: acmeKey, certSecret: map[string][]byte{keyACMEAcctFP: curFP}, want: true, }, { name: "fingerprint_differs", certShareMode: "rw", acmeInMemory: acmeKey, certSecret: map[string][]byte{keyACMEAcctFP: otherFP}, want: false, }, { name: "legacy_cert_no_preadopted_returns_true", certShareMode: "rw", acmeInMemory: acmeKey, certSecret: map[string][]byte{}, // no fingerprint field want: true, }, { name: "legacy_cert_with_preadopted_returns_false", certShareMode: "rw", acmeInMemory: acmeKey, preAdopted: sha256Sum(otherKey), certSecret: map[string][]byte{}, // no fingerprint field want: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { if tt.certGetErr != nil { return nil, tt.certGetErr } return &kubeapi.Secret{Data: tt.certSecret}, nil }, } s := &Store{ client: client, certShareMode: tt.certShareMode, memory: mem.Store{}, preAdoptedLocalKey: tt.preAdopted, logf: t.Logf, } if len(tt.acmeInMemory) > 0 { s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), tt.acmeInMemory) } got, err := s.ShouldUseARIReplacesForRenewal(domain) if (err != nil) != tt.wantErr { t.Errorf("err = %v, wantErr = %v", err, tt.wantErr) } if got != tt.want { t.Errorf("got %v, want %v", got, tt.want) } }) } } func TestWriteTLSCertAndKeyStampsFingerprint(t *testing.T) { const domain = "app.tailnetxyz.ts.net" acmeKey := []byte("-----BEGIN PRIVATE KEY-----\naccount\n-----END PRIVATE KEY-----") wantFP := sha256Sum(acmeKey) tests := []struct { name string certMode string acmeInMemory []byte wantFP []byte // expected value of keyACMEAcctFP field; nil means field must be absent }{ { name: "rw_mode_with_acme_key_stamps_fingerprint", certMode: "rw", acmeInMemory: acmeKey, wantFP: wantFP, }, { name: "rw_mode_no_acme_key_no_stamp", certMode: "rw", }, { name: "non_share_mode_no_stamp", certMode: "", acmeInMemory: acmeKey, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { secret := map[string][]byte{} client := &kubeclient.FakeClient{ GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) { return &kubeapi.Secret{Data: secret}, nil }, CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) { return true, true, nil }, JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error { for _, p := range patches { if p.Op == "add" && p.Path == "/data" { secret = p.Value.(map[string][]byte) } else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") { secret[strings.TrimPrefix(p.Path, "/data/")] = p.Value.([]byte) } } return nil }, } s := &Store{ client: client, canPatch: true, secretName: "ts-state", certShareMode: tt.certMode, memory: mem.Store{}, logf: t.Logf, } if len(tt.acmeInMemory) > 0 { s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), tt.acmeInMemory) } if err := s.WriteTLSCertAndKey(domain, []byte("cert"), []byte("key")); err != nil { t.Fatalf("WriteTLSCertAndKey: %v", err) } gotFP, present := secret[keyACMEAcctFP] if tt.wantFP == nil { if present { t.Errorf("unexpected fingerprint stamped: %x", gotFP) } return } if !bytes.Equal(gotFP, tt.wantFP) { t.Errorf("fingerprint = %x, want %x", gotFP, tt.wantFP) } }) } }