WIP: rebase fork onto upstream/main (v1.103.0) #15

Closed
codinget wants to merge 670 commits from webnet into save/webnet-2026-07-29
13 changed files with 975 additions and 34 deletions
Showing only changes of commit 97a75c837d - Show all commits
@@ -124,6 +124,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: OPERATOR_SHARED_ACME_ACCOUNT_KEY
value: {{ .Values.operatorConfig.sharedACMEAccountKey | quote }}
{{- with .Values.operatorConfig.extraEnv }}
{{- toYaml . | nindent 12 }}
{{- end }}
@@ -87,6 +87,13 @@ operatorConfig:
# - name: EXTRA_VAR2
# value: "value2"
# Default for the tailscale.com/share-acme-account annotation on new
# ProxyGroups. When true, the operator provisions a shared per-tailnet
# ACME account key Secret and configures proxies to use it, preserving
# Let's Encrypt's ARI "replaces" renewal exemption across pod restarts
# and ProxyGroup recreation. See #18251.
sharedACMEAccountKey: false
# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
ingressClass:
# Allows for customization of the ingress class name used by the operator to identify ingresses to reconcile. This does
@@ -6874,6 +6874,8 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.uid
- name: OPERATOR_SHARED_ACME_ACCOUNT_KEY
value: "false"
image: tailscale/k8s-operator:stable
imagePullPolicy: Always
name: operator
+39
View File
@@ -96,6 +96,7 @@ func main() {
tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "")
defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "")
isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
sharedACMEAccountKey = defaultBool("OPERATOR_SHARED_ACME_ACCOUNT_KEY", false)
loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/")
ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale")
operatorSAName = defaultEnv("OPERATOR_SERVICE_ACCOUNT_NAME", "operator")
@@ -170,6 +171,7 @@ func main() {
defaultProxyClass: defaultProxyClass,
loginServer: loginServer,
ingressClassName: ingressClassName,
sharedACMEAccountKey: sharedACMEAccountKey,
})
}
@@ -752,6 +754,7 @@ func runReconcilers(opts reconcilerOpts) {
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog))
saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog))
acmeSecretFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(acmeAccountsSecretHandlerForProxyGroup(mgr.GetClient(), opts.tailscaleNamespace, opts.sharedACMEAccountKey, startlog))
err = builder.ControllerManagedBy(mgr).
For(&tsapi.ProxyGroup{}).
Named("proxygroup-reconciler").
@@ -760,6 +763,9 @@ func runReconcilers(opts reconcilerOpts) {
Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup).
Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
// The shared ACME accounts Secret has no ProxyGroup owner ref, so
// watch it by name to react to its deletion/recreation.
Watches(&corev1.Secret{}, acmeSecretFilterForProxyGroup).
Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
@@ -780,6 +786,8 @@ func runReconcilers(opts reconcilerOpts) {
loginServer: opts.tsServer.ControlURL,
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
sharedACMEAccountKey: opts.sharedACMEAccountKey,
})
if err != nil {
startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)
@@ -835,6 +843,13 @@ type reconcilerOpts struct {
// ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults
// to "tailscale" but can be customised.
ingressClassName string
// sharedACMEAccountKey is the operator-wide default for the
// shared-ACME-account feature. When true, every ProxyGroup uses the
// shared per-tailnet account key unless the ProxyGroup explicitly
// opts out via tailscale.com/share-acme-account=false. When false,
// ProxyGroups opt in individually via
// tailscale.com/share-acme-account=true.
sharedACMEAccountKey bool
// operatorSAName is the name of the ServiceAccount that the operator pod runs as. It is used as the target
// ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate using
// workload identity federation.
@@ -1230,6 +1245,30 @@ func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLog
}
}
// acmeAccountsSecretHandlerForProxyGroup enqueues ProxyGroups that use the
// shared ACME account when the shared ACME accounts Secret changes. The
// Secret carries no owner reference, so the owner-based Secret watch never
// matches it.
func acmeAccountsSecretHandlerForProxyGroup(cl client.Client, tsNamespace string, sharedACMEAccountDefault bool, logger *zap.SugaredLogger) handler.MapFunc {
return func(ctx context.Context, o client.Object) []reconcile.Request {
if o.GetName() != kubetypes.ACMEAccountsSecretName || o.GetNamespace() != tsNamespace {
return nil
}
pgList := new(tsapi.ProxyGroupList)
if err := cl.List(ctx, pgList); err != nil {
logger.Debugf("error listing ProxyGroups for shared ACME accounts Secret: %v", err)
return nil
}
reqs := make([]reconcile.Request, 0, len(pgList.Items))
for _, pg := range pgList.Items {
if sharedACMEAccountEnabled(&pg, sharedACMEAccountDefault) {
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
}
}
return reqs
}
}
// serviceHandlerForIngress returns a handler for Service events for ingress
// reconciler that ensures that if the Service associated with an event is of
// interest to the reconciler, the associated Ingress(es) gets be reconciled.
+53 -2
View File
@@ -56,6 +56,7 @@ const (
reasonProxyGroupCreating = "ProxyGroupCreating"
reasonProxyGroupInvalid = "ProxyGroupInvalid"
reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable"
reasonACMEAccountsPendingDeletion = "ACMEAccountsPendingDeletion"
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
@@ -102,6 +103,14 @@ type ProxyGroupReconciler struct {
apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge
authKeyRateLimits map[string]*rate.Limiter // per-ProxyGroup rate limiters for auth key re-issuance.
authKeyReissuing map[string]bool
// sharedACMEAccountKey is the operator-wide default for the
// shared-ACME-account feature. When true, every ProxyGroup uses the
// shared per-tailnet account key unless the ProxyGroup explicitly
// opts out via tailscale.com/share-acme-account=false. When false,
// only ProxyGroups annotated with tailscale.com/share-acme-account=true
// use it.
sharedACMEAccountKey bool
}
func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
@@ -354,7 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl
}
}
role := pgRole(pg, r.tsNamespace)
role := pgRole(pg, r.tsNamespace, r.sharedACMEAccountEnabledFor(pg))
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
r.ObjectMeta.Labels = role.ObjectMeta.Labels
r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
@@ -394,13 +403,36 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl
}); err != nil {
return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
}
// Ensure the shared ACME accounts Secret exists (with finalizer)
// when this ProxyGroup opts into the feature. Proxy pods
// populate its fields on first cert issuance. See #18251.
if r.sharedACMEAccountEnabledFor(pg) {
acmeSecret := pgACMEAccountSecret(r.tsNamespace)
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, acmeSecret, func(existing *corev1.Secret) {
if !existing.DeletionTimestamp.IsZero() {
// Deletion can't be undone; warn so the account keys
// get backed up before the finalizer is removed.
msg := fmt.Sprintf("shared ACME accounts Secret %q is marked for deletion but retained by the %q finalizer. Its data remains readable until the finalizer is removed - back it up first to preserve the ACME account keys.", existing.Name, kubetypes.ACMEAccountsFinalizer)
r.recorder.Event(existing, corev1.EventTypeWarning, reasonACMEAccountsPendingDeletion, msg)
logger.Warn(msg)
return
}
existing.Labels = acmeSecret.Labels
if !slices.Contains(existing.Finalizers, kubetypes.ACMEAccountsFinalizer) {
existing.Finalizers = append(existing.Finalizers, kubetypes.ACMEAccountsFinalizer)
}
}); err != nil {
return r.notReadyErrf(pg, logger, "error provisioning shared ACME accounts Secret %q: %w", acmeSecret.Name, err)
}
}
}
defaultImage := r.tsProxyImage
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
defaultImage = r.k8sProxyImage
}
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass)
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass, r.sharedACMEAccountEnabledFor(pg))
if err != nil {
return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err)
}
@@ -1347,6 +1379,25 @@ func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason,
}, nil
}
// sharedACMEAccountEnabledFor reports whether the shared-ACME-account
// feature should be applied to pg. The per-PG
// tailscale.com/share-acme-account annotation wins when set; otherwise
// the operator's OPERATOR_SHARED_ACME_ACCOUNT_KEY setting is the default
// for every ProxyGroup.
func (r *ProxyGroupReconciler) sharedACMEAccountEnabledFor(pg *tsapi.ProxyGroup) bool {
return sharedACMEAccountEnabled(pg, r.sharedACMEAccountKey)
}
// sharedACMEAccountEnabled reports whether pg should use the shared ACME
// account, with the tailscale.com/share-acme-account annotation overriding
// the operator-wide default.
func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool {
if v, ok := pg.Annotations[AnnotationShareACMEAccount]; ok {
return v == "true"
}
return operatorDefault
}
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
err := fmt.Errorf(format, a...)
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
+65 -11
View File
@@ -64,8 +64,12 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor
}
// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
// applied over the top after.
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
// applied over the top after. shareACMEAccount, when true, injects the env
// vars that route the pod's ACME account key to the shared per-tailnet
// Secret and drops TS_DEBUG_ACME_FORCE_RENEWAL so ARI-based renewals are
// attempted; the caller is responsible for checking the operator setting
// and the PG opt-in annotation.
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass, shareACMEAccount bool) (*appsv1.StatefulSet, error) {
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
return kubeAPIServerStatefulSet(pg, namespace, image, port)
}
@@ -187,14 +191,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
Value: "/etc/tsconfig/$(POD_NAME)",
},
{
// This ensures that cert renewals can succeed if ACME account
// keys have changed since issuance. We cannot guarantee or
// validate that the account key has not changed, see
// https://github.com/tailscale/tailscale/issues/18251
Name: "TS_DEBUG_ACME_FORCE_RENEWAL",
Value: "true",
},
}
if port != nil {
@@ -252,6 +248,29 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
Value: "true",
},
)
if shareACMEAccount {
envs = append(envs,
corev1.EnvVar{
Name: "TS_ACME_ACCOUNT_SECRET_NAME",
Value: kubetypes.ACMEAccountsSecretName,
},
corev1.EnvVar{
Name: "TS_ACME_ACCOUNT_FIELD",
Value: pgACMEAccountField(pg),
},
)
} else {
// Without a shared account key we cannot guarantee that
// the account key that issued the previous cert is the
// same one attempting renewal. Force plain new-order flow
// so renewals do not silently fail on rejected ARI
// "replaces" claims. See
// https://github.com/tailscale/tailscale/issues/18251.
envs = append(envs, corev1.EnvVar{
Name: "TS_DEBUG_ACME_FORCE_RENEWAL",
Value: "true",
})
}
}
return append(c.Env, envs...)
}()
@@ -407,7 +426,7 @@ func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAcc
}
}
func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
func pgRole(pg *tsapi.ProxyGroup, namespace string, shareACMEAccount bool) *rbacv1.Role {
return &rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: pg.Name,
@@ -439,6 +458,12 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
pgPodName(pg.Name, i), // State.
)
}
// Ingress ProxyGroup write replicas need access to the
// shared ACME account Secret so they can read the
// per-tailnet account key and write it on first use.
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress && shareACMEAccount {
secrets = append(secrets, kubetypes.ACMEAccountsSecretName)
}
return secrets
}(),
},
@@ -477,6 +502,35 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
}
}
// pgACMEAccountField returns the field name used inside the shared
// tailscale-acme-accounts Secret for this ProxyGroup's tailnet. The blank
// tailnet (operator-default credentials) is represented by a reserved
// identifier so it gets a stable, unique field.
func pgACMEAccountField(pg *tsapi.ProxyGroup) string {
tn := pg.Spec.Tailnet
if tn == "" {
tn = kubetypes.ACMEAccountDefaultKey
}
return tn + kubetypes.ACMEAccountKeySuffix
}
// pgACMEAccountSecret returns the shared per-tailnet ACME account key
// Secret, keyed by tailnet inside its data. Not owned by any ProxyGroup
// so it outlives ProxyGroup deletion.
func pgACMEAccountSecret(namespace string) *corev1.Secret {
return &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: kubetypes.ACMEAccountsSecretName,
Namespace: namespace,
Labels: map[string]string{
kubetypes.LabelManaged: "true",
},
// Block accidental deletion.
Finalizers: []string{kubetypes.ACMEAccountsFinalizer},
},
}
}
// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything
// else uses a per-ProxyGroup ServiceAccount.
func pgServiceAccountName(pg *tsapi.ProxyGroup) string {
+108 -11
View File
@@ -1136,14 +1136,15 @@ func TestProxyGroupTypes(t *testing.T) {
zl, _ := zap.NewDevelopment()
reconciler := &ProxyGroupReconciler{
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zl.Sugar(),
clients: tsclient.NewProvider(&fakeTSClient{}),
clock: tstest.NewClock(tstest.ClockOpts{}),
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
tsNamespace: tsNamespace,
tsProxyImage: testProxyImage,
Client: fc,
log: zl.Sugar(),
clients: tsclient.NewProvider(&fakeTSClient{}),
clock: tstest.NewClock(tstest.ClockOpts{}),
authKeyRateLimits: make(map[string]*rate.Limiter),
authKeyReissuing: make(map[string]bool),
sharedACMEAccountKey: true,
}
t.Run("egress_type", func(t *testing.T) {
@@ -1263,6 +1264,9 @@ func TestProxyGroupTypes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress",
UID: "test-ingress-uid",
Annotations: map[string]string{
AnnotationShareACMEAccount: "true",
},
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
@@ -1283,6 +1287,44 @@ func TestProxyGroupTypes(t *testing.T) {
verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
verifyEnvVar(t, sts, "TS_ACME_ACCOUNT_SECRET_NAME", kubetypes.ACMEAccountsSecretName)
// pg.Spec.Tailnet is empty here so the default tailnet field is used.
verifyEnvVar(t, sts, "TS_ACME_ACCOUNT_FIELD", kubetypes.ACMEAccountDefaultKey+kubetypes.ACMEAccountKeySuffix)
// TS_DEBUG_ACME_FORCE_RENEWAL must NOT be set when the PG is
// opted in to the shared ACME account.
for _, e := range sts.Spec.Template.Spec.Containers[0].Env {
if e.Name == "TS_DEBUG_ACME_FORCE_RENEWAL" {
t.Errorf("TS_DEBUG_ACME_FORCE_RENEWAL must not be set on ingress ProxyGroup pods that share an ACME account")
}
}
// Verify the shared ACME accounts Secret exists and has the
// deletion finalizer (see tailscale/tailscale#18251).
acmeSecret := &corev1.Secret{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: kubetypes.ACMEAccountsSecretName}, acmeSecret); err != nil {
t.Errorf("failed to get shared ACME accounts Secret: %v", err)
}
if !slices.Contains(acmeSecret.Finalizers, kubetypes.ACMEAccountsFinalizer) {
t.Errorf("shared ACME accounts Secret missing finalizer %q (got %v)", kubetypes.ACMEAccountsFinalizer, acmeSecret.Finalizers)
}
// Verify the per-ProxyGroup Role grants access to the shared
// ACME accounts Secret (write replicas need it to read/write the
// per-tailnet account key).
role := &rbacv1.Role{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, role); err != nil {
t.Fatalf("failed to get ProxyGroup Role: %v", err)
}
var sawACMEAccess bool
for _, rule := range role.Rules {
if slices.Contains(rule.Verbs, "patch") && slices.Contains(rule.ResourceNames, kubetypes.ACMEAccountsSecretName) {
sawACMEAccess = true
break
}
}
if !sawACMEAccess {
t.Errorf("ProxyGroup Role does not grant patch access to %q", kubetypes.ACMEAccountsSecretName)
}
// Verify ConfigMap volume mount
cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
@@ -1312,6 +1354,60 @@ func TestProxyGroupTypes(t *testing.T) {
}
})
t.Run("ingress_type_shared_acme_opt_out", func(t *testing.T) {
// The reconciler has sharedACMEAccountKey=true, so ingress PGs
// default to shared. Explicit tailscale.com/share-acme-account=false
// must opt this PG out: no shared-Secret env vars, no Role
// access to the shared Secret, and TS_DEBUG_ACME_FORCE_RENEWAL
// must still be set so ARI "replaces" doesn't silently fail.
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "test-ingress-optout",
UID: "test-ingress-optout-uid",
Annotations: map[string]string{
AnnotationShareACMEAccount: "false",
},
},
Spec: tsapi.ProxyGroupSpec{
Type: tsapi.ProxyGroupTypeIngress,
Replicas: new(int32(0)),
},
}
if err := fc.Create(t.Context(), pg); err != nil {
t.Fatal(err)
}
expectReconciled(t, reconciler, "", pg.Name)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
t.Fatalf("failed to get StatefulSet: %v", err)
}
for _, e := range sts.Spec.Template.Spec.Containers[0].Env {
switch e.Name {
case "TS_ACME_ACCOUNT_SECRET_NAME", "TS_ACME_ACCOUNT_FIELD":
t.Errorf("env %q unexpectedly present on opt-out PG", e.Name)
}
}
var sawForceRenewal bool
for _, e := range sts.Spec.Template.Spec.Containers[0].Env {
if e.Name == "TS_DEBUG_ACME_FORCE_RENEWAL" {
sawForceRenewal = true
}
}
if !sawForceRenewal {
t.Errorf("TS_DEBUG_ACME_FORCE_RENEWAL must be set on opt-out PG (avoids silent ARI \"replaces\" rejection)")
}
role := &rbacv1.Role{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, role); err != nil {
t.Fatalf("failed to get ProxyGroup Role: %v", err)
}
for _, rule := range role.Rules {
if slices.Contains(rule.ResourceNames, kubetypes.ACMEAccountsSecretName) {
t.Errorf("opt-out PG Role must not grant access to %q", kubetypes.ACMEAccountsSecretName)
}
}
})
t.Run("kubernetes_api_server_type", func(t *testing.T) {
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
@@ -1331,7 +1427,7 @@ func TestProxyGroupTypes(t *testing.T) {
}
expectReconciled(t, reconciler, "", pg.Name)
verifyProxyGroupCounts(t, reconciler, 1, 2, 1)
verifyProxyGroupCounts(t, reconciler, 2, 2, 1)
sts := &appsv1.StatefulSet{}
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
@@ -2036,10 +2132,11 @@ func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string)
func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, proxyClass *tsapi.ProxyClass) {
t.Helper()
role := pgRole(pg, tsNamespace)
shareACMEAccount := pg.Annotations[AnnotationShareACMEAccount] == "true"
role := pgRole(pg, tsNamespace, shareACMEAccount)
roleBinding := pgRoleBinding(pg, tsNamespace)
serviceAccount := pgServiceAccount(pg, tsNamespace)
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass)
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass, shareACMEAccount)
if err != nil {
t.Fatal(err)
}
+6
View File
@@ -68,6 +68,12 @@ const (
AnnotationProxyGroup = "tailscale.com/proxy-group"
// AnnotationShareACMEAccount opts a single ProxyGroup into ("true")
// or out of ("false") using the shared per-tailnet ACME account key.
// When absent, OPERATOR_SHARED_ACME_ACCOUNT_KEY on the operator is
// the default. See tailscale/tailscale#18251.
AnnotationShareACMEAccount = "tailscale.com/share-acme-account"
// Annotations settable by users on ingresses.
AnnotationFunnel = "tailscale.com/funnel"
AnnotationHTTPRedirect = "tailscale.com/http-redirect"
+20 -7
View File
@@ -360,17 +360,30 @@ var getCertPEM = func(ctx context.Context, e *extension, b *ipnlocal.LocalBacken
return nil, fmt.Errorf("unexpected ACME account status %q", a.Status)
}
// If we have a previous cert, include it in the order. Assuming we're
// within the ARI renewal window this should exclude us from LE rate
// limits.
// Note that this order extension will fail renewals if the ACME account key has changed
// since the last issuance, see
// https://github.com/tailscale/tailscale/issues/18251
// If we have a previous cert, include it in the order via the ARI
// "replaces" extension so LE classifies the new cert as a renewal
// and exempts it from the per-registered-domain rate limit. Stores
// that can tell the current ACME account did not issue the previous
// cert opt out via [ARIReplacesAllower]; see #18251.
var opts []xacme.OrderOption
if previous != nil && !envknob.Bool("TS_DEBUG_ACME_FORCE_RENEWAL") {
prevCrt, err := parseCertificate(previous)
if err == nil {
opts = append(opts, xacme.WithOrderReplacesCert(prevCrt))
useReplaces := true
if a, ok := cs.(ARIReplacesAllower); ok {
if allowed, err := a.ShouldUseARIReplacesForRenewal(domain); err != nil {
// Fail open: an error means we couldn't check
// eligibility, not that the account is misaligned.
logf("acme: failed to check ARI 'replaces' eligibility for %q, defaulting to use it: %v", domain, err)
} else {
useReplaces = allowed
}
}
if useReplaces {
opts = append(opts, xacme.WithOrderReplacesCert(prevCrt))
} else {
logf("account key mismatch for previous cert; skipping ARI 'replaces' hint")
}
}
}
+29
View File
@@ -187,6 +187,25 @@ type TLSCertKeyReader interface {
ReadTLSCertAndKey(domain string) ([]byte, []byte, error)
}
// ARIReplacesAllower is optionally implemented by state stores to opt
// out of the ARI "replaces" hint on a per-domain basis at renewal time.
// When implemented and returning false, the renewal path submits a plain
// newOrder instead of claiming renewal exemption via "replaces".
//
// Used by the k8s cert-share store to prevent submitting a "replaces"
// claim that Let's Encrypt would reject because the current ACME
// account did not issue the previous cert (which can happen when a
// shared per-tailnet account key is adopted after a cert was already
// issued by a per-pod account). See #18251.
type ARIReplacesAllower interface {
// ShouldUseARIReplacesForRenewal reports whether the renewal order
// for domain should carry the ARI "replaces" hint. It is advisory:
// a non-nil error means eligibility could not be determined, not
// that the hint should be skipped, and callers fail open (use the
// hint) in that case.
ShouldUseARIReplacesForRenewal(domain string) (bool, error)
}
func (s certStateStore) Read(domain string, now time.Time) (*ipnlocal.TLSCertKeyPair, error) {
// If we're using a store that supports atomic reads, use that
if kr, ok := s.StateStore.(TLSCertKeyReader); ok {
@@ -215,6 +234,16 @@ func (s certStateStore) Read(domain string, now time.Time) (*ipnlocal.TLSCertKey
return &ipnlocal.TLSCertKeyPair{CertPEM: certPEM, KeyPEM: keyPEM, Cached: true}, nil
}
// ShouldUseARIReplacesForRenewal delegates to the underlying state store
// if it implements [ARIReplacesAllower]. Stores that don't opt in default
// to true (attempt "replaces"), preserving the pre-hook behaviour.
func (s certStateStore) ShouldUseARIReplacesForRenewal(domain string) (bool, error) {
if a, ok := s.StateStore.(ARIReplacesAllower); ok {
return a.ShouldUseARIReplacesForRenewal(domain)
}
return true, nil
}
func (s certStateStore) WriteCert(domain string, cert []byte) error {
return ipn.WriteState(s.StateStore, ipn.StateKey(domain+".crt"), cert)
}
+183 -3
View File
@@ -5,7 +5,9 @@
package kubestore
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"net"
@@ -47,6 +49,17 @@ const (
keyTLSCert = "tls.crt"
keyTLSKey = "tls.key"
// keyACMEAcctFP is the cert Secret field that records the SHA-256
// fingerprint of the PEM-encoded ACME account key that issued the
// cert. The renewal path uses this to decide whether to include the
// ARI "replaces" hint: only if the current account key matches
// (otherwise Let's Encrypt rejects the claim).
keyACMEAcctFP = "acme-account-fingerprint"
// acmeAccountStateKey is the ipn.StateStore key under which tailscaled
// stores its ACME account private key. Mirrors the acmePEMName constant
// in ipn/ipnlocal/cert.go. Duplicated here to avoid an import cycle.
acmeAccountStateKey = "acme-account.key.pem"
)
// Store is an ipn.StateStore that uses a Kubernetes Secret for persistence.
@@ -57,6 +70,17 @@ type Store struct {
certShareMode string // 'ro', 'rw', or empty
podName string
// acmeAccountsSecretName, when non-empty in "rw" cert share mode,
// routes reads and writes of acmeAccountStateKey to acmeAccountField
// inside this shared per-tailnet Secret. See #18251.
acmeAccountsSecretName string
acmeAccountField string
// preAdoptedLocalKey is the SHA-256 of the per-pod ACME account key
// that was in the local state Secret before we adopted a foreign
// shared key. Non-nil only when adoption changed the key.
preAdoptedLocalKey []byte
logf logger.Logf
// memory holds the latest tailscale state. Writes write state to a kube
@@ -106,6 +130,17 @@ func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*S
s.certShareMode = "ro"
}
// Configure shared ACME account lookup. Only meaningful for the cert
// issuer (cert share "rw") — read replicas never issue.
if s.certShareMode == "rw" {
s.acmeAccountsSecretName = os.Getenv("TS_ACME_ACCOUNT_SECRET_NAME")
s.acmeAccountField = os.Getenv("TS_ACME_ACCOUNT_FIELD")
if s.acmeAccountsSecretName != "" && s.acmeAccountField == "" {
s.logf("[unexpected] TS_ACME_ACCOUNT_SECRET_NAME set without TS_ACME_ACCOUNT_FIELD; ignoring shared ACME account configuration")
s.acmeAccountsSecretName = ""
}
}
// Load latest state from kube Secret if it already exists.
if err := s.loadState(); err != nil && err != ipn.ErrStateNotExist {
return nil, fmt.Errorf("error loading state from kube Secret: %w", err)
@@ -126,9 +161,135 @@ func newWithClient(logf logger.Logf, c kubeclient.Client, secretName string) (*S
if s.certShareMode == "ro" {
go s.runCertReload(context.Background())
}
if s.acmeAccountsSecretName != "" {
if err := s.reconcileSharedACMEAccountKey(); err != nil {
// Non-fatal: the cert loop will retry on next issuance.
s.logf("kubestore: reconciling shared ACME account key: %v", err)
}
}
return s, nil
}
// reconcileSharedACMEAccountKey aligns the in-memory ACME account key with
// the shared per-tailnet field: adopt the shared value if present, otherwise
// copy the local per-pod key up so upgrading deployments keep renewal
// continuity.
func (s *Store) reconcileSharedACMEAccountKey() error {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
sharedSecret, err := s.client.GetSecret(ctx, s.acmeAccountsSecretName)
if err != nil && !kubeclient.IsNotFoundErr(err) {
return fmt.Errorf("reading shared ACME accounts Secret %q: %w", s.acmeAccountsSecretName, err)
}
var sharedKey []byte
if sharedSecret != nil {
sharedKey = sharedSecret.Data[sanitizeKey(s.acmeAccountField)]
}
if len(sharedKey) > 0 {
// Shared field already populated for this tailnet. Adopt it. If
// our local per-pod key differs, remember its fingerprint so
// legacy certs on this pod (issued before we started stamping
// fingerprints) can be recognised as mis-aligned on renewal.
localKey, err := s.memory.ReadState(ipn.StateKey(acmeAccountStateKey))
if err == nil && len(localKey) > 0 && !bytes.Equal(localKey, sharedKey) {
sum := sha256.Sum256(localKey)
s.preAdoptedLocalKey = sum[:]
}
s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), sharedKey)
return nil
}
// Shared field is empty. If we have a per-pod key from the state
// Secret, copy it up so existing renewals stay exempt.
localKey, err := s.memory.ReadState(ipn.StateKey(acmeAccountStateKey))
if err != nil || len(localKey) == 0 {
// Nothing local either; the cert loop will generate one on first
// use and route the write through writeSharedACMEAccountKey.
return nil
}
if err := s.writeSharedACMEAccountKey(localKey); err != nil {
return fmt.Errorf("copying per-pod ACME account key to shared Secret: %w", err)
}
s.logf("kubestore: migrated per-pod ACME account key into shared Secret %q field %q", s.acmeAccountsSecretName, s.acmeAccountField)
return nil
}
// writeSharedACMEAccountKey writes key to acmeAccountField inside
// acmeAccountsSecretName, using whichever access pattern (patch or update)
// this Store has permission for.
func (s *Store) writeSharedACMEAccountKey(key []byte) error {
return s.updateSecret(map[string][]byte{s.acmeAccountField: key}, s.acmeAccountsSecretName)
}
// maybeRestoreSharedACMEAccountKey writes the in-memory ACME account key to
// the shared Secret's per-tailnet field if that field is empty or missing,
// e.g. because the Secret was deleted and recreated while this process was
// running. It never overwrites an existing shared key.
func (s *Store) maybeRestoreSharedACMEAccountKey() error {
if s.acmeAccountsSecretName == "" {
return nil
}
key, err := s.memory.ReadState(ipn.StateKey(acmeAccountStateKey))
if err != nil || len(key) == 0 {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
shared, err := s.client.GetSecret(ctx, s.acmeAccountsSecretName)
if err != nil && !kubeclient.IsNotFoundErr(err) {
return fmt.Errorf("reading shared ACME accounts Secret %q: %w", s.acmeAccountsSecretName, err)
}
if shared != nil && len(shared.Data[sanitizeKey(s.acmeAccountField)]) > 0 {
return nil
}
s.logf("kubestore: shared ACME accounts Secret %q field %q is empty; restoring account key from memory", s.acmeAccountsSecretName, s.acmeAccountField)
return s.writeSharedACMEAccountKey(key)
}
// acmeAccountKeyFingerprint returns the SHA-256 of the current in-memory
// PEM-encoded ACME account key, or (nil, false) if the key isn't set.
func acmeAccountKeyFingerprint(m *mem.Store) ([]byte, bool) {
key, err := m.ReadState(ipn.StateKey(acmeAccountStateKey))
if err != nil || len(key) == 0 {
return nil, false
}
sum := sha256.Sum256(key)
return sum[:], true
}
// ShouldUseARIReplacesForRenewal reports whether the current ACME account
// key matches the one that issued the cert for domain. See #18251.
func (s *Store) ShouldUseARIReplacesForRenewal(domain string) (bool, error) {
if s.certShareMode != "rw" {
return true, nil
}
curFP, ok := acmeAccountKeyFingerprint(&s.memory)
if !ok {
// No current account key in memory yet; nothing to compare.
return true, nil
}
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
sec, err := s.client.GetSecret(ctx, domain)
if err != nil {
if kubeclient.IsNotFoundErr(err) {
return true, nil
}
return true, fmt.Errorf("getting TLS Secret %q: %w", domain, err)
}
certFP := sec.Data[keyACMEAcctFP]
if len(certFP) == 0 {
// Legacy cert, no fingerprint stamp. Assume misaligned only if
// we adopted a foreign shared key.
if len(s.preAdoptedLocalKey) > 0 {
return false, nil
}
return true, nil
}
return bytes.Equal(certFP, curFP), nil
}
func (s *Store) SetDialer(d func(ctx context.Context, network, address string) (net.Conn, error)) {
s.client.SetDialer(d)
}
@@ -147,6 +308,12 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
s.memory.WriteState(ipn.StateKey(sanitizeKey(id)), bs)
}
}()
if s.acmeAccountsSecretName != "" && string(id) == acmeAccountStateKey {
if bs == nil {
return s.removeSecretField(s.acmeAccountField, s.acmeAccountsSecretName)
}
return s.writeSharedACMEAccountKey(bs)
}
if bs == nil {
return s.removeSecretField(string(id), s.secretName)
}
@@ -154,7 +321,9 @@ func (s *Store) WriteState(id ipn.StateKey, bs []byte) (err error) {
}
// WriteTLSCertAndKey writes a TLS cert and key to domain.crt, domain.key fields
// of a Tailscale Kubernetes node's state Secret.
// of a Tailscale Kubernetes node's state Secret. In cert-share "rw" mode it
// also stamps acme-account-fingerprint alongside the cert so the renewal path
// can tell whether the current ACME account key issued this cert.
func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error) {
if s.certShareMode == "ro" {
s.logf("[unexpected] TLS cert and key write in read-only mode")
@@ -175,6 +344,14 @@ func (s *Store) WriteTLSCertAndKey(domain string, cert, key []byte) (err error)
keyTLSCert: cert,
keyTLSKey: key,
}
if fp, ok := acmeAccountKeyFingerprint(&s.memory); ok {
data[keyACMEAcctFP] = fp
}
// The shared Secret may have been deleted and recreated empty
// while we were running; re-assert the account key if so.
if err := s.maybeRestoreSharedACMEAccountKey(); err != nil {
s.logf("kubestore: restoring shared ACME account key: %v", err)
}
}
if err := s.updateSecret(data, secretName); err != nil {
return fmt.Errorf("error writing TLS cert and key to Secret: %w", err)
@@ -489,7 +666,9 @@ func (s *Store) loadCerts(ctx context.Context, sel map[string]string) error {
// canCreateSecret returns true if this node should be allowed to create the given
// Secret in its namespace.
func (s *Store) canCreateSecret(secret string) bool {
// Only allow creating the state Secret (and not TLS Secrets).
// Only allow creating the state Secret (and not TLS Secrets). The
// shared ACME accounts Secret is precreated by the operator, so write
// replicas never need create permission for it.
return secret == s.secretName
}
@@ -498,7 +677,8 @@ func (s *Store) canCreateSecret(secret string) bool {
func (s *Store) canPatchSecret(secret string) bool {
// For backwards compatibility reasons, setups where the proxies are not
// given PATCH permissions for state Secrets are allowed. For TLS
// Secrets, we should always have PATCH permissions.
// Secrets and the shared ACME accounts Secret, we should always have
// PATCH permissions.
if secret == s.secretName {
return s.canPatch
}
+439
View File
@@ -6,6 +6,7 @@ package kubestore
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"fmt"
"strings"
@@ -847,3 +848,441 @@ func TestNewWithClient(t *testing.T) {
})
}
}
func TestSharedACMEAccountKey(t *testing.T) {
const (
stateSecretName = "ingress-proxies-0"
sharedSecretName = kubetypes.ACMEAccountsSecretName
sharedField = "my-tailnet" + kubetypes.ACMEAccountKeySuffix
)
existingKey := []byte("-----BEGIN PRIVATE KEY-----\nexisting\n-----END PRIVATE KEY-----")
freshKey := []byte("-----BEGIN PRIVATE KEY-----\nfresh\n-----END PRIVATE KEY-----")
tests := []struct {
name string
certMode string
stateSecret map[string][]byte
sharedSecret map[string][]byte // nil = Secret does not exist
envSecretName string
envField string
writeAfterInit []byte // if non-nil, call WriteState(acmeAccountStateKey, …) after init
wantMemoryACME []byte
wantSharedSecret map[string][]byte
wantStateSecret map[string][]byte // optional: when set, asserts state Secret was not touched on the ACME path
wantPreAdopted []byte // expected s.preAdoptedLocalKey (sha256 of pre-adoption local key)
}{
{
name: "adopts_shared_key_when_present",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
},
{
name: "shared_overrides_per_pod_key",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: freshKey, // would be stale
},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
wantPreAdopted: sha256Sum(freshKey),
},
{
name: "adopting_matching_local_leaves_preadopted_nil",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: existingKey, // matches shared
},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
},
{
name: "migrates_per_pod_key_when_shared_field_empty",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: existingKey,
},
sharedSecret: map[string][]byte{}, // exists but no field for this tailnet
wantMemoryACME: existingKey,
wantSharedSecret: map[string][]byte{sharedField: existingKey},
},
{
name: "no_op_when_no_keys_anywhere",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{},
wantMemoryACME: nil,
wantSharedSecret: map[string][]byte{},
},
{
name: "ro_mode_ignores_env_vars",
certMode: "ro",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{
acmeAccountStateKey: freshKey,
},
sharedSecret: map[string][]byte{sharedField: existingKey},
wantMemoryACME: freshKey, // per-pod copy stays; shared Secret never consulted
},
{
name: "write_routes_to_shared_secret",
certMode: "rw",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{},
writeAfterInit: freshKey,
wantMemoryACME: freshKey,
wantSharedSecret: map[string][]byte{sharedField: freshKey},
wantStateSecret: map[string][]byte{}, // state Secret untouched by the ACME write
},
{
name: "write_in_ro_mode_goes_to_state_secret",
certMode: "ro",
envSecretName: sharedSecretName,
envField: sharedField,
stateSecret: map[string][]byte{},
sharedSecret: map[string][]byte{},
writeAfterInit: freshKey,
wantMemoryACME: freshKey,
wantSharedSecret: map[string][]byte{}, // env vars ignored in ro mode
wantStateSecret: map[string][]byte{acmeAccountStateKey: freshKey},
},
{
name: "no_env_vars_routes_to_state_secret",
certMode: "rw",
envSecretName: "", // shared account disabled
envField: "",
stateSecret: map[string][]byte{
acmeAccountStateKey: existingKey,
},
sharedSecret: nil, // does not exist; should not be touched
wantMemoryACME: existingKey,
wantStateSecret: map[string][]byte{acmeAccountStateKey: existingKey},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Setenv("TS_CERT_SHARE_MODE", tt.certMode)
t.Setenv("TS_ACME_ACCOUNT_SECRET_NAME", tt.envSecretName)
t.Setenv("TS_ACME_ACCOUNT_FIELD", tt.envField)
t.Setenv("POD_NAME", stateSecretName)
stateData := cloneMap(tt.stateSecret)
sharedData := cloneMap(tt.sharedSecret)
sharedExists := tt.sharedSecret != nil
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
switch name {
case stateSecretName:
return &kubeapi.Secret{Data: stateData}, nil
case sharedSecretName:
if !sharedExists {
return nil, &kubeapi.Status{Code: 404}
}
return &kubeapi.Secret{Data: sharedData}, nil
}
return nil, &kubeapi.Status{Code: 404}
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
var target *map[string][]byte
switch name {
case stateSecretName:
target = &stateData
case sharedSecretName:
target = &sharedData
sharedExists = true
default:
t.Errorf("unexpected patch target Secret %q", name)
return nil
}
if *target == nil {
*target = map[string][]byte{}
}
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
*target = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
key := strings.TrimPrefix(p.Path, "/data/")
(*target)[key] = p.Value.([]byte)
}
}
return nil
},
CreateSecretImpl: func(ctx context.Context, s *kubeapi.Secret) error {
switch s.Name {
case stateSecretName:
stateData = s.Data
case sharedSecretName:
sharedData = s.Data
sharedExists = true
default:
t.Errorf("unexpected create target Secret %q", s.Name)
}
return nil
},
ListSecretsImpl: func(ctx context.Context, selector map[string]string) (*kubeapi.SecretList, error) {
// Used by ro-mode TLS Secret preload; irrelevant to this test.
return &kubeapi.SecretList{}, nil
},
}
s, err := newWithClient(t.Logf, client, stateSecretName)
if err != nil {
t.Fatalf("newWithClient: %v", err)
}
if tt.writeAfterInit != nil {
if err := s.WriteState(ipn.StateKey(acmeAccountStateKey), tt.writeAfterInit); err != nil {
t.Fatalf("WriteState(ACME key): %v", err)
}
}
gotMemACME, err := s.memory.ReadState(ipn.StateKey(acmeAccountStateKey))
if err != nil && tt.wantMemoryACME != nil {
t.Errorf("memory ReadState(ACME key): %v", err)
}
if !bytes.Equal(gotMemACME, tt.wantMemoryACME) {
t.Errorf("memory ACME key = %q, want %q", gotMemACME, tt.wantMemoryACME)
}
if tt.wantSharedSecret != nil {
if diff := cmp.Diff(sharedData, tt.wantSharedSecret); diff != "" {
t.Errorf("shared Secret contents mismatch (-got +want):\n%s", diff)
}
}
if tt.wantStateSecret != nil {
if diff := cmp.Diff(stateData, tt.wantStateSecret); diff != "" {
t.Errorf("state Secret contents mismatch (-got +want):\n%s", diff)
}
}
if !bytes.Equal(s.preAdoptedLocalKey, tt.wantPreAdopted) {
t.Errorf("preAdoptedLocalKey = %x, want %x", s.preAdoptedLocalKey, tt.wantPreAdopted)
}
})
}
}
func cloneMap(m map[string][]byte) map[string][]byte {
if m == nil {
return nil
}
out := make(map[string][]byte, len(m))
for k, v := range m {
out[k] = append([]byte(nil), v...)
}
return out
}
func sha256Sum(b []byte) []byte {
sum := sha256.Sum256(b)
return sum[:]
}
func TestShouldUseARIReplacesForRenewal(t *testing.T) {
const domain = "app.tailnetxyz.ts.net"
acmeKey := []byte("-----BEGIN PRIVATE KEY-----\ncurrent\n-----END PRIVATE KEY-----")
otherKey := []byte("-----BEGIN PRIVATE KEY-----\nother\n-----END PRIVATE KEY-----")
curFP := sha256Sum(acmeKey)
otherFP := sha256Sum(otherKey)
tests := []struct {
name string
certShareMode string
acmeInMemory []byte // per-pod ACME key present in memory
preAdopted []byte // sha256 of pre-adoption local key (foreign-key path)
certSecret map[string][]byte
certGetErr error
want bool
wantErr bool
}{
{
name: "non_rw_mode_returns_true",
certShareMode: "",
want: true,
},
{
name: "ro_mode_returns_true",
certShareMode: "ro",
want: true,
},
{
name: "no_acme_key_returns_true",
certShareMode: "rw",
want: true,
},
{
name: "cert_not_found_returns_true",
certShareMode: "rw",
acmeInMemory: acmeKey,
certGetErr: &kubeapi.Status{Code: 404},
want: true,
},
{
name: "cert_get_error_returns_true_with_err",
certShareMode: "rw",
acmeInMemory: acmeKey,
certGetErr: fmt.Errorf("api down"),
want: true,
wantErr: true,
},
{
name: "fingerprint_matches",
certShareMode: "rw",
acmeInMemory: acmeKey,
certSecret: map[string][]byte{keyACMEAcctFP: curFP},
want: true,
},
{
name: "fingerprint_differs",
certShareMode: "rw",
acmeInMemory: acmeKey,
certSecret: map[string][]byte{keyACMEAcctFP: otherFP},
want: false,
},
{
name: "legacy_cert_no_preadopted_returns_true",
certShareMode: "rw",
acmeInMemory: acmeKey,
certSecret: map[string][]byte{}, // no fingerprint field
want: true,
},
{
name: "legacy_cert_with_preadopted_returns_false",
certShareMode: "rw",
acmeInMemory: acmeKey,
preAdopted: sha256Sum(otherKey),
certSecret: map[string][]byte{}, // no fingerprint field
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
if tt.certGetErr != nil {
return nil, tt.certGetErr
}
return &kubeapi.Secret{Data: tt.certSecret}, nil
},
}
s := &Store{
client: client,
certShareMode: tt.certShareMode,
memory: mem.Store{},
preAdoptedLocalKey: tt.preAdopted,
logf: t.Logf,
}
if len(tt.acmeInMemory) > 0 {
s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), tt.acmeInMemory)
}
got, err := s.ShouldUseARIReplacesForRenewal(domain)
if (err != nil) != tt.wantErr {
t.Errorf("err = %v, wantErr = %v", err, tt.wantErr)
}
if got != tt.want {
t.Errorf("got %v, want %v", got, tt.want)
}
})
}
}
func TestWriteTLSCertAndKeyStampsFingerprint(t *testing.T) {
const domain = "app.tailnetxyz.ts.net"
acmeKey := []byte("-----BEGIN PRIVATE KEY-----\naccount\n-----END PRIVATE KEY-----")
wantFP := sha256Sum(acmeKey)
tests := []struct {
name string
certMode string
acmeInMemory []byte
wantFP []byte // expected value of keyACMEAcctFP field; nil means field must be absent
}{
{
name: "rw_mode_with_acme_key_stamps_fingerprint",
certMode: "rw",
acmeInMemory: acmeKey,
wantFP: wantFP,
},
{
name: "rw_mode_no_acme_key_no_stamp",
certMode: "rw",
},
{
name: "non_share_mode_no_stamp",
certMode: "",
acmeInMemory: acmeKey,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
secret := map[string][]byte{}
client := &kubeclient.FakeClient{
GetSecretImpl: func(ctx context.Context, name string) (*kubeapi.Secret, error) {
return &kubeapi.Secret{Data: secret}, nil
},
CheckSecretPermissionsImpl: func(ctx context.Context, name string) (bool, bool, error) {
return true, true, nil
},
JSONPatchResourceImpl: func(ctx context.Context, name, resourceType string, patches []kubeclient.JSONPatch) error {
for _, p := range patches {
if p.Op == "add" && p.Path == "/data" {
secret = p.Value.(map[string][]byte)
} else if p.Op == "add" && strings.HasPrefix(p.Path, "/data/") {
secret[strings.TrimPrefix(p.Path, "/data/")] = p.Value.([]byte)
}
}
return nil
},
}
s := &Store{
client: client,
canPatch: true,
secretName: "ts-state",
certShareMode: tt.certMode,
memory: mem.Store{},
logf: t.Logf,
}
if len(tt.acmeInMemory) > 0 {
s.memory.WriteState(ipn.StateKey(acmeAccountStateKey), tt.acmeInMemory)
}
if err := s.WriteTLSCertAndKey(domain, []byte("cert"), []byte("key")); err != nil {
t.Fatalf("WriteTLSCertAndKey: %v", err)
}
gotFP, present := secret[keyACMEAcctFP]
if tt.wantFP == nil {
if present {
t.Errorf("unexpected fingerprint stamped: %x", gotFP)
}
return
}
if !bytes.Equal(gotFP, tt.wantFP) {
t.Errorf("fingerprint = %x, want %x", gotFP, tt.wantFP)
}
})
}
}
+22
View File
@@ -65,6 +65,28 @@ const (
LabelSecretTypeState = "state"
LabelSecretTypeCerts = "certs"
// ACMEAccountsSecretName is the name of the Secret in the operator
// namespace that holds shared ACME account private keys, one field per
// tailnet. Sharing one account key across all replicas of every
// ProxyGroup attached to a tailnet preserves Let's Encrypt's renewal
// exemption (via the ARI "replaces" extension) across pod restarts,
// ProxyGroup recreation, and ingress migration.
ACMEAccountsSecretName = "tailscale-acme-accounts"
// ACMEAccountDefaultKey is the field used inside ACMEAccountsSecretName
// for the "default" tailnet (i.e., when ProxyGroup.spec.tailnet is empty
// and the operator uses its own credentials).
ACMEAccountDefaultKey = "_default"
// ACMEAccountKeySuffix is appended to the tailnet identifier to form a
// field name within ACMEAccountsSecretName.
ACMEAccountKeySuffix = ".acme-account.key.pem"
// ACMEAccountsFinalizer blocks accidental deletion of
// ACMEAccountsSecretName; losing those keys breaks ARI "replaces"
// renewals for every cert in the tailnet. See #18251.
ACMEAccountsFinalizer = "tailscale.com/acme-account-protection"
KubeAPIServerConfigFile = "config.hujson"
APIServerProxyModeAuth APIServerProxyMode = "auth"
APIServerProxyModeNoAuth APIServerProxyMode = "noauth"