cmd/k8s-operator,ipn/store/kubestore,kube/kubetypes: share ACME account key per tailnet
Introduce a per-tailnet shared ACME account key so that all ingress ProxyGroup replicas on a tailnet present the same account identity to Let's Encrypt. This lets renewals claim the ARI "replaces" exemption from the 50-certs-per-week rate limit, surviving Pod restarts, ProxyGroup recreation, and cluster migrations. The operator provisions a "tailscale-acme-accounts" Secret in its namespace, guarded by a finalizer and a deletion warning event, and watched so it is recreated promptly if removed. Proxies migrate any pre-existing per-pod key into the shared Secret on first boot, adopt the shared key on subsequent boots, and restore it on cert writes if the Secret was recreated empty. Certs are stamped with the fingerprint of the issuing account so renewals skip the "replaces" claim when the account doesn't match. Opt-in per-ProxyGroup via the tailscale.com/share-acme-account annotation, or operator-wide via OPERATOR_SHARED_ACME_ACCOUNT_KEY. Updates #18251 Updates #20288 Signed-off-by: chaosinthecrd <tom@tmlabs.co.uk>
This commit is contained in:
committed by
Tom Meadows
parent
2900f3494a
commit
97a75c837d
@@ -124,6 +124,8 @@ spec:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.uid
|
||||
- name: OPERATOR_SHARED_ACME_ACCOUNT_KEY
|
||||
value: {{ .Values.operatorConfig.sharedACMEAccountKey | quote }}
|
||||
{{- with .Values.operatorConfig.extraEnv }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
|
||||
@@ -87,6 +87,13 @@ operatorConfig:
|
||||
# - name: EXTRA_VAR2
|
||||
# value: "value2"
|
||||
|
||||
# Default for the tailscale.com/share-acme-account annotation on new
|
||||
# ProxyGroups. When true, the operator provisions a shared per-tailnet
|
||||
# ACME account key Secret and configures proxies to use it, preserving
|
||||
# Let's Encrypt's ARI "replaces" renewal exemption across pod restarts
|
||||
# and ProxyGroup recreation. See #18251.
|
||||
sharedACMEAccountKey: false
|
||||
|
||||
# In the case that you already have a tailscale ingressclass in your cluster (or vcluster), you can disable the creation here
|
||||
ingressClass:
|
||||
# Allows for customization of the ingress class name used by the operator to identify ingresses to reconcile. This does
|
||||
|
||||
@@ -6874,6 +6874,8 @@ spec:
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.uid
|
||||
- name: OPERATOR_SHARED_ACME_ACCOUNT_KEY
|
||||
value: "false"
|
||||
image: tailscale/k8s-operator:stable
|
||||
imagePullPolicy: Always
|
||||
name: operator
|
||||
|
||||
@@ -96,6 +96,7 @@ func main() {
|
||||
tsFirewallMode = defaultEnv("PROXY_FIREWALL_MODE", "")
|
||||
defaultProxyClass = defaultEnv("PROXY_DEFAULT_CLASS", "")
|
||||
isDefaultLoadBalancer = defaultBool("OPERATOR_DEFAULT_LOAD_BALANCER", false)
|
||||
sharedACMEAccountKey = defaultBool("OPERATOR_SHARED_ACME_ACCOUNT_KEY", false)
|
||||
loginServer = strings.TrimSuffix(defaultEnv("OPERATOR_LOGIN_SERVER", ""), "/")
|
||||
ingressClassName = defaultEnv("OPERATOR_INGRESS_CLASS_NAME", "tailscale")
|
||||
operatorSAName = defaultEnv("OPERATOR_SERVICE_ACCOUNT_NAME", "operator")
|
||||
@@ -170,6 +171,7 @@ func main() {
|
||||
defaultProxyClass: defaultProxyClass,
|
||||
loginServer: loginServer,
|
||||
ingressClassName: ingressClassName,
|
||||
sharedACMEAccountKey: sharedACMEAccountKey,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -752,6 +754,7 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
proxyClassFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(proxyClassHandlerForProxyGroup(mgr.GetClient(), startlog))
|
||||
nodeFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(nodeHandlerForProxyGroup(mgr.GetClient(), opts.defaultProxyClass, startlog))
|
||||
saFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(serviceAccountHandlerForProxyGroup(mgr.GetClient(), startlog))
|
||||
acmeSecretFilterForProxyGroup := handler.EnqueueRequestsFromMapFunc(acmeAccountsSecretHandlerForProxyGroup(mgr.GetClient(), opts.tailscaleNamespace, opts.sharedACMEAccountKey, startlog))
|
||||
err = builder.ControllerManagedBy(mgr).
|
||||
For(&tsapi.ProxyGroup{}).
|
||||
Named("proxygroup-reconciler").
|
||||
@@ -760,6 +763,9 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
Watches(&corev1.ConfigMap{}, ownedByProxyGroupFilter).
|
||||
Watches(&corev1.ServiceAccount{}, saFilterForProxyGroup).
|
||||
Watches(&corev1.Secret{}, ownedByProxyGroupFilter).
|
||||
// The shared ACME accounts Secret has no ProxyGroup owner ref, so
|
||||
// watch it by name to react to its deletion/recreation.
|
||||
Watches(&corev1.Secret{}, acmeSecretFilterForProxyGroup).
|
||||
Watches(&rbacv1.Role{}, ownedByProxyGroupFilter).
|
||||
Watches(&rbacv1.RoleBinding{}, ownedByProxyGroupFilter).
|
||||
Watches(&tsapi.ProxyClass{}, proxyClassFilterForProxyGroup).
|
||||
@@ -780,6 +786,8 @@ func runReconcilers(opts reconcilerOpts) {
|
||||
loginServer: opts.tsServer.ControlURL,
|
||||
authKeyRateLimits: make(map[string]*rate.Limiter),
|
||||
authKeyReissuing: make(map[string]bool),
|
||||
|
||||
sharedACMEAccountKey: opts.sharedACMEAccountKey,
|
||||
})
|
||||
if err != nil {
|
||||
startlog.Fatalf("could not create ProxyGroup reconciler: %v", err)
|
||||
@@ -835,6 +843,13 @@ type reconcilerOpts struct {
|
||||
// ingressClassName is the name of the ingress class used by reconcilers of Ingress resources. This defaults
|
||||
// to "tailscale" but can be customised.
|
||||
ingressClassName string
|
||||
// sharedACMEAccountKey is the operator-wide default for the
|
||||
// shared-ACME-account feature. When true, every ProxyGroup uses the
|
||||
// shared per-tailnet account key unless the ProxyGroup explicitly
|
||||
// opts out via tailscale.com/share-acme-account=false. When false,
|
||||
// ProxyGroups opt in individually via
|
||||
// tailscale.com/share-acme-account=true.
|
||||
sharedACMEAccountKey bool
|
||||
// operatorSAName is the name of the ServiceAccount that the operator pod runs as. It is used as the target
|
||||
// ServiceAccount when minting tokens via the Kubernetes TokenRequest API for Tailnets that authenticate using
|
||||
// workload identity federation.
|
||||
@@ -1230,6 +1245,30 @@ func serviceAccountHandlerForProxyGroup(cl client.Client, logger *zap.SugaredLog
|
||||
}
|
||||
}
|
||||
|
||||
// acmeAccountsSecretHandlerForProxyGroup enqueues ProxyGroups that use the
|
||||
// shared ACME account when the shared ACME accounts Secret changes. The
|
||||
// Secret carries no owner reference, so the owner-based Secret watch never
|
||||
// matches it.
|
||||
func acmeAccountsSecretHandlerForProxyGroup(cl client.Client, tsNamespace string, sharedACMEAccountDefault bool, logger *zap.SugaredLogger) handler.MapFunc {
|
||||
return func(ctx context.Context, o client.Object) []reconcile.Request {
|
||||
if o.GetName() != kubetypes.ACMEAccountsSecretName || o.GetNamespace() != tsNamespace {
|
||||
return nil
|
||||
}
|
||||
pgList := new(tsapi.ProxyGroupList)
|
||||
if err := cl.List(ctx, pgList); err != nil {
|
||||
logger.Debugf("error listing ProxyGroups for shared ACME accounts Secret: %v", err)
|
||||
return nil
|
||||
}
|
||||
reqs := make([]reconcile.Request, 0, len(pgList.Items))
|
||||
for _, pg := range pgList.Items {
|
||||
if sharedACMEAccountEnabled(&pg, sharedACMEAccountDefault) {
|
||||
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(&pg)})
|
||||
}
|
||||
}
|
||||
return reqs
|
||||
}
|
||||
}
|
||||
|
||||
// serviceHandlerForIngress returns a handler for Service events for ingress
|
||||
// reconciler that ensures that if the Service associated with an event is of
|
||||
// interest to the reconciler, the associated Ingress(es) gets be reconciled.
|
||||
|
||||
@@ -56,6 +56,7 @@ const (
|
||||
reasonProxyGroupCreating = "ProxyGroupCreating"
|
||||
reasonProxyGroupInvalid = "ProxyGroupInvalid"
|
||||
reasonProxyGroupTailnetUnavailable = "ProxyGroupTailnetUnavailable"
|
||||
reasonACMEAccountsPendingDeletion = "ACMEAccountsPendingDeletion"
|
||||
|
||||
// Copied from k8s.io/apiserver/pkg/registry/generic/registry/store.go@cccad306d649184bf2a0e319ba830c53f65c445c
|
||||
optimisticLockErrorMsg = "the object has been modified; please apply your changes to the latest version and try again"
|
||||
@@ -102,6 +103,14 @@ type ProxyGroupReconciler struct {
|
||||
apiServerProxyGroups set.Slice[types.UID] // for kube-apiserver proxygroups gauge
|
||||
authKeyRateLimits map[string]*rate.Limiter // per-ProxyGroup rate limiters for auth key re-issuance.
|
||||
authKeyReissuing map[string]bool
|
||||
|
||||
// sharedACMEAccountKey is the operator-wide default for the
|
||||
// shared-ACME-account feature. When true, every ProxyGroup uses the
|
||||
// shared per-tailnet account key unless the ProxyGroup explicitly
|
||||
// opts out via tailscale.com/share-acme-account=false. When false,
|
||||
// only ProxyGroups annotated with tailscale.com/share-acme-account=true
|
||||
// use it.
|
||||
sharedACMEAccountKey bool
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) logger(name string) *zap.SugaredLogger {
|
||||
@@ -354,7 +363,7 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl
|
||||
}
|
||||
}
|
||||
|
||||
role := pgRole(pg, r.tsNamespace)
|
||||
role := pgRole(pg, r.tsNamespace, r.sharedACMEAccountEnabledFor(pg))
|
||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, role, func(r *rbacv1.Role) {
|
||||
r.ObjectMeta.Labels = role.ObjectMeta.Labels
|
||||
r.ObjectMeta.Annotations = role.ObjectMeta.Annotations
|
||||
@@ -394,13 +403,36 @@ func (r *ProxyGroupReconciler) maybeProvision(ctx context.Context, tsClient tscl
|
||||
}); err != nil {
|
||||
return r.notReadyErrf(pg, logger, "error provisioning ingress ConfigMap %q: %w", cm.Name, err)
|
||||
}
|
||||
|
||||
// Ensure the shared ACME accounts Secret exists (with finalizer)
|
||||
// when this ProxyGroup opts into the feature. Proxy pods
|
||||
// populate its fields on first cert issuance. See #18251.
|
||||
if r.sharedACMEAccountEnabledFor(pg) {
|
||||
acmeSecret := pgACMEAccountSecret(r.tsNamespace)
|
||||
if _, err := createOrUpdate(ctx, r.Client, r.tsNamespace, acmeSecret, func(existing *corev1.Secret) {
|
||||
if !existing.DeletionTimestamp.IsZero() {
|
||||
// Deletion can't be undone; warn so the account keys
|
||||
// get backed up before the finalizer is removed.
|
||||
msg := fmt.Sprintf("shared ACME accounts Secret %q is marked for deletion but retained by the %q finalizer. Its data remains readable until the finalizer is removed - back it up first to preserve the ACME account keys.", existing.Name, kubetypes.ACMEAccountsFinalizer)
|
||||
r.recorder.Event(existing, corev1.EventTypeWarning, reasonACMEAccountsPendingDeletion, msg)
|
||||
logger.Warn(msg)
|
||||
return
|
||||
}
|
||||
existing.Labels = acmeSecret.Labels
|
||||
if !slices.Contains(existing.Finalizers, kubetypes.ACMEAccountsFinalizer) {
|
||||
existing.Finalizers = append(existing.Finalizers, kubetypes.ACMEAccountsFinalizer)
|
||||
}
|
||||
}); err != nil {
|
||||
return r.notReadyErrf(pg, logger, "error provisioning shared ACME accounts Secret %q: %w", acmeSecret.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
defaultImage := r.tsProxyImage
|
||||
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
|
||||
defaultImage = r.k8sProxyImage
|
||||
}
|
||||
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass)
|
||||
ss, err := pgStatefulSet(pg, r.tsNamespace, defaultImage, r.tsFirewallMode, tailscaledPort, proxyClass, r.sharedACMEAccountEnabledFor(pg))
|
||||
if err != nil {
|
||||
return r.notReadyErrf(pg, logger, "error generating StatefulSet spec: %w", err)
|
||||
}
|
||||
@@ -1347,6 +1379,25 @@ func notReady(reason, msg string) (map[string][]netip.AddrPort, *notReadyReason,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// sharedACMEAccountEnabledFor reports whether the shared-ACME-account
|
||||
// feature should be applied to pg. The per-PG
|
||||
// tailscale.com/share-acme-account annotation wins when set; otherwise
|
||||
// the operator's OPERATOR_SHARED_ACME_ACCOUNT_KEY setting is the default
|
||||
// for every ProxyGroup.
|
||||
func (r *ProxyGroupReconciler) sharedACMEAccountEnabledFor(pg *tsapi.ProxyGroup) bool {
|
||||
return sharedACMEAccountEnabled(pg, r.sharedACMEAccountKey)
|
||||
}
|
||||
|
||||
// sharedACMEAccountEnabled reports whether pg should use the shared ACME
|
||||
// account, with the tailscale.com/share-acme-account annotation overriding
|
||||
// the operator-wide default.
|
||||
func sharedACMEAccountEnabled(pg *tsapi.ProxyGroup, operatorDefault bool) bool {
|
||||
if v, ok := pg.Annotations[AnnotationShareACMEAccount]; ok {
|
||||
return v == "true"
|
||||
}
|
||||
return operatorDefault
|
||||
}
|
||||
|
||||
func (r *ProxyGroupReconciler) notReadyErrf(pg *tsapi.ProxyGroup, logger *zap.SugaredLogger, format string, a ...any) (map[string][]netip.AddrPort, *notReadyReason, error) {
|
||||
err := fmt.Errorf(format, a...)
|
||||
if strings.Contains(err.Error(), optimisticLockErrorMsg) {
|
||||
|
||||
@@ -64,8 +64,12 @@ func pgNodePortService(pg *tsapi.ProxyGroup, name string, namespace string) *cor
|
||||
}
|
||||
|
||||
// Returns the base StatefulSet definition for a ProxyGroup. A ProxyClass may be
|
||||
// applied over the top after.
|
||||
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass) (*appsv1.StatefulSet, error) {
|
||||
// applied over the top after. shareACMEAccount, when true, injects the env
|
||||
// vars that route the pod's ACME account key to the shared per-tailnet
|
||||
// Secret and drops TS_DEBUG_ACME_FORCE_RENEWAL so ARI-based renewals are
|
||||
// attempted; the caller is responsible for checking the operator setting
|
||||
// and the PG opt-in annotation.
|
||||
func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string, port *uint16, proxyClass *tsapi.ProxyClass, shareACMEAccount bool) (*appsv1.StatefulSet, error) {
|
||||
if pg.Spec.Type == tsapi.ProxyGroupTypeKubernetesAPIServer {
|
||||
return kubeAPIServerStatefulSet(pg, namespace, image, port)
|
||||
}
|
||||
@@ -187,14 +191,6 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
|
||||
Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
|
||||
Value: "/etc/tsconfig/$(POD_NAME)",
|
||||
},
|
||||
{
|
||||
// This ensures that cert renewals can succeed if ACME account
|
||||
// keys have changed since issuance. We cannot guarantee or
|
||||
// validate that the account key has not changed, see
|
||||
// https://github.com/tailscale/tailscale/issues/18251
|
||||
Name: "TS_DEBUG_ACME_FORCE_RENEWAL",
|
||||
Value: "true",
|
||||
},
|
||||
}
|
||||
|
||||
if port != nil {
|
||||
@@ -252,6 +248,29 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
|
||||
Value: "true",
|
||||
},
|
||||
)
|
||||
if shareACMEAccount {
|
||||
envs = append(envs,
|
||||
corev1.EnvVar{
|
||||
Name: "TS_ACME_ACCOUNT_SECRET_NAME",
|
||||
Value: kubetypes.ACMEAccountsSecretName,
|
||||
},
|
||||
corev1.EnvVar{
|
||||
Name: "TS_ACME_ACCOUNT_FIELD",
|
||||
Value: pgACMEAccountField(pg),
|
||||
},
|
||||
)
|
||||
} else {
|
||||
// Without a shared account key we cannot guarantee that
|
||||
// the account key that issued the previous cert is the
|
||||
// same one attempting renewal. Force plain new-order flow
|
||||
// so renewals do not silently fail on rejected ARI
|
||||
// "replaces" claims. See
|
||||
// https://github.com/tailscale/tailscale/issues/18251.
|
||||
envs = append(envs, corev1.EnvVar{
|
||||
Name: "TS_DEBUG_ACME_FORCE_RENEWAL",
|
||||
Value: "true",
|
||||
})
|
||||
}
|
||||
}
|
||||
return append(c.Env, envs...)
|
||||
}()
|
||||
@@ -407,7 +426,7 @@ func pgServiceAccount(pg *tsapi.ProxyGroup, namespace string) *corev1.ServiceAcc
|
||||
}
|
||||
}
|
||||
|
||||
func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
|
||||
func pgRole(pg *tsapi.ProxyGroup, namespace string, shareACMEAccount bool) *rbacv1.Role {
|
||||
return &rbacv1.Role{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: pg.Name,
|
||||
@@ -439,6 +458,12 @@ func pgRole(pg *tsapi.ProxyGroup, namespace string) *rbacv1.Role {
|
||||
pgPodName(pg.Name, i), // State.
|
||||
)
|
||||
}
|
||||
// Ingress ProxyGroup write replicas need access to the
|
||||
// shared ACME account Secret so they can read the
|
||||
// per-tailnet account key and write it on first use.
|
||||
if pg.Spec.Type == tsapi.ProxyGroupTypeIngress && shareACMEAccount {
|
||||
secrets = append(secrets, kubetypes.ACMEAccountsSecretName)
|
||||
}
|
||||
return secrets
|
||||
}(),
|
||||
},
|
||||
@@ -477,6 +502,35 @@ func pgRoleBinding(pg *tsapi.ProxyGroup, namespace string) *rbacv1.RoleBinding {
|
||||
}
|
||||
}
|
||||
|
||||
// pgACMEAccountField returns the field name used inside the shared
|
||||
// tailscale-acme-accounts Secret for this ProxyGroup's tailnet. The blank
|
||||
// tailnet (operator-default credentials) is represented by a reserved
|
||||
// identifier so it gets a stable, unique field.
|
||||
func pgACMEAccountField(pg *tsapi.ProxyGroup) string {
|
||||
tn := pg.Spec.Tailnet
|
||||
if tn == "" {
|
||||
tn = kubetypes.ACMEAccountDefaultKey
|
||||
}
|
||||
return tn + kubetypes.ACMEAccountKeySuffix
|
||||
}
|
||||
|
||||
// pgACMEAccountSecret returns the shared per-tailnet ACME account key
|
||||
// Secret, keyed by tailnet inside its data. Not owned by any ProxyGroup
|
||||
// so it outlives ProxyGroup deletion.
|
||||
func pgACMEAccountSecret(namespace string) *corev1.Secret {
|
||||
return &corev1.Secret{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: kubetypes.ACMEAccountsSecretName,
|
||||
Namespace: namespace,
|
||||
Labels: map[string]string{
|
||||
kubetypes.LabelManaged: "true",
|
||||
},
|
||||
// Block accidental deletion.
|
||||
Finalizers: []string{kubetypes.ACMEAccountsFinalizer},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// kube-apiserver proxies in auth mode use a static ServiceAccount. Everything
|
||||
// else uses a per-ProxyGroup ServiceAccount.
|
||||
func pgServiceAccountName(pg *tsapi.ProxyGroup) string {
|
||||
|
||||
@@ -1136,14 +1136,15 @@ func TestProxyGroupTypes(t *testing.T) {
|
||||
|
||||
zl, _ := zap.NewDevelopment()
|
||||
reconciler := &ProxyGroupReconciler{
|
||||
tsNamespace: tsNamespace,
|
||||
tsProxyImage: testProxyImage,
|
||||
Client: fc,
|
||||
log: zl.Sugar(),
|
||||
clients: tsclient.NewProvider(&fakeTSClient{}),
|
||||
clock: tstest.NewClock(tstest.ClockOpts{}),
|
||||
authKeyRateLimits: make(map[string]*rate.Limiter),
|
||||
authKeyReissuing: make(map[string]bool),
|
||||
tsNamespace: tsNamespace,
|
||||
tsProxyImage: testProxyImage,
|
||||
Client: fc,
|
||||
log: zl.Sugar(),
|
||||
clients: tsclient.NewProvider(&fakeTSClient{}),
|
||||
clock: tstest.NewClock(tstest.ClockOpts{}),
|
||||
authKeyRateLimits: make(map[string]*rate.Limiter),
|
||||
authKeyReissuing: make(map[string]bool),
|
||||
sharedACMEAccountKey: true,
|
||||
}
|
||||
|
||||
t.Run("egress_type", func(t *testing.T) {
|
||||
@@ -1263,6 +1264,9 @@ func TestProxyGroupTypes(t *testing.T) {
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-ingress",
|
||||
UID: "test-ingress-uid",
|
||||
Annotations: map[string]string{
|
||||
AnnotationShareACMEAccount: "true",
|
||||
},
|
||||
},
|
||||
Spec: tsapi.ProxyGroupSpec{
|
||||
Type: tsapi.ProxyGroupTypeIngress,
|
||||
@@ -1283,6 +1287,44 @@ func TestProxyGroupTypes(t *testing.T) {
|
||||
verifyEnvVar(t, sts, "TS_INTERNAL_APP", kubetypes.AppProxyGroupIngress)
|
||||
verifyEnvVar(t, sts, "TS_SERVE_CONFIG", "/etc/proxies/serve-config.json")
|
||||
verifyEnvVar(t, sts, "TS_EXPERIMENTAL_CERT_SHARE", "true")
|
||||
verifyEnvVar(t, sts, "TS_ACME_ACCOUNT_SECRET_NAME", kubetypes.ACMEAccountsSecretName)
|
||||
// pg.Spec.Tailnet is empty here so the default tailnet field is used.
|
||||
verifyEnvVar(t, sts, "TS_ACME_ACCOUNT_FIELD", kubetypes.ACMEAccountDefaultKey+kubetypes.ACMEAccountKeySuffix)
|
||||
// TS_DEBUG_ACME_FORCE_RENEWAL must NOT be set when the PG is
|
||||
// opted in to the shared ACME account.
|
||||
for _, e := range sts.Spec.Template.Spec.Containers[0].Env {
|
||||
if e.Name == "TS_DEBUG_ACME_FORCE_RENEWAL" {
|
||||
t.Errorf("TS_DEBUG_ACME_FORCE_RENEWAL must not be set on ingress ProxyGroup pods that share an ACME account")
|
||||
}
|
||||
}
|
||||
|
||||
// Verify the shared ACME accounts Secret exists and has the
|
||||
// deletion finalizer (see tailscale/tailscale#18251).
|
||||
acmeSecret := &corev1.Secret{}
|
||||
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: kubetypes.ACMEAccountsSecretName}, acmeSecret); err != nil {
|
||||
t.Errorf("failed to get shared ACME accounts Secret: %v", err)
|
||||
}
|
||||
if !slices.Contains(acmeSecret.Finalizers, kubetypes.ACMEAccountsFinalizer) {
|
||||
t.Errorf("shared ACME accounts Secret missing finalizer %q (got %v)", kubetypes.ACMEAccountsFinalizer, acmeSecret.Finalizers)
|
||||
}
|
||||
|
||||
// Verify the per-ProxyGroup Role grants access to the shared
|
||||
// ACME accounts Secret (write replicas need it to read/write the
|
||||
// per-tailnet account key).
|
||||
role := &rbacv1.Role{}
|
||||
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, role); err != nil {
|
||||
t.Fatalf("failed to get ProxyGroup Role: %v", err)
|
||||
}
|
||||
var sawACMEAccess bool
|
||||
for _, rule := range role.Rules {
|
||||
if slices.Contains(rule.Verbs, "patch") && slices.Contains(rule.ResourceNames, kubetypes.ACMEAccountsSecretName) {
|
||||
sawACMEAccess = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !sawACMEAccess {
|
||||
t.Errorf("ProxyGroup Role does not grant patch access to %q", kubetypes.ACMEAccountsSecretName)
|
||||
}
|
||||
|
||||
// Verify ConfigMap volume mount
|
||||
cmName := fmt.Sprintf("%s-ingress-config", pg.Name)
|
||||
@@ -1312,6 +1354,60 @@ func TestProxyGroupTypes(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ingress_type_shared_acme_opt_out", func(t *testing.T) {
|
||||
// The reconciler has sharedACMEAccountKey=true, so ingress PGs
|
||||
// default to shared. Explicit tailscale.com/share-acme-account=false
|
||||
// must opt this PG out: no shared-Secret env vars, no Role
|
||||
// access to the shared Secret, and TS_DEBUG_ACME_FORCE_RENEWAL
|
||||
// must still be set so ARI "replaces" doesn't silently fail.
|
||||
pg := &tsapi.ProxyGroup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Name: "test-ingress-optout",
|
||||
UID: "test-ingress-optout-uid",
|
||||
Annotations: map[string]string{
|
||||
AnnotationShareACMEAccount: "false",
|
||||
},
|
||||
},
|
||||
Spec: tsapi.ProxyGroupSpec{
|
||||
Type: tsapi.ProxyGroupTypeIngress,
|
||||
Replicas: new(int32(0)),
|
||||
},
|
||||
}
|
||||
if err := fc.Create(t.Context(), pg); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
expectReconciled(t, reconciler, "", pg.Name)
|
||||
|
||||
sts := &appsv1.StatefulSet{}
|
||||
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
|
||||
t.Fatalf("failed to get StatefulSet: %v", err)
|
||||
}
|
||||
for _, e := range sts.Spec.Template.Spec.Containers[0].Env {
|
||||
switch e.Name {
|
||||
case "TS_ACME_ACCOUNT_SECRET_NAME", "TS_ACME_ACCOUNT_FIELD":
|
||||
t.Errorf("env %q unexpectedly present on opt-out PG", e.Name)
|
||||
}
|
||||
}
|
||||
var sawForceRenewal bool
|
||||
for _, e := range sts.Spec.Template.Spec.Containers[0].Env {
|
||||
if e.Name == "TS_DEBUG_ACME_FORCE_RENEWAL" {
|
||||
sawForceRenewal = true
|
||||
}
|
||||
}
|
||||
if !sawForceRenewal {
|
||||
t.Errorf("TS_DEBUG_ACME_FORCE_RENEWAL must be set on opt-out PG (avoids silent ARI \"replaces\" rejection)")
|
||||
}
|
||||
role := &rbacv1.Role{}
|
||||
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, role); err != nil {
|
||||
t.Fatalf("failed to get ProxyGroup Role: %v", err)
|
||||
}
|
||||
for _, rule := range role.Rules {
|
||||
if slices.Contains(rule.ResourceNames, kubetypes.ACMEAccountsSecretName) {
|
||||
t.Errorf("opt-out PG Role must not grant access to %q", kubetypes.ACMEAccountsSecretName)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("kubernetes_api_server_type", func(t *testing.T) {
|
||||
pg := &tsapi.ProxyGroup{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
@@ -1331,7 +1427,7 @@ func TestProxyGroupTypes(t *testing.T) {
|
||||
}
|
||||
|
||||
expectReconciled(t, reconciler, "", pg.Name)
|
||||
verifyProxyGroupCounts(t, reconciler, 1, 2, 1)
|
||||
verifyProxyGroupCounts(t, reconciler, 2, 2, 1)
|
||||
|
||||
sts := &appsv1.StatefulSet{}
|
||||
if err := fc.Get(t.Context(), client.ObjectKey{Namespace: tsNamespace, Name: pg.Name}, sts); err != nil {
|
||||
@@ -2036,10 +2132,11 @@ func verifyEnvVarNotPresent(t *testing.T, sts *appsv1.StatefulSet, name string)
|
||||
func expectProxyGroupResources(t *testing.T, fc client.WithWatch, pg *tsapi.ProxyGroup, shouldExist bool, proxyClass *tsapi.ProxyClass) {
|
||||
t.Helper()
|
||||
|
||||
role := pgRole(pg, tsNamespace)
|
||||
shareACMEAccount := pg.Annotations[AnnotationShareACMEAccount] == "true"
|
||||
role := pgRole(pg, tsNamespace, shareACMEAccount)
|
||||
roleBinding := pgRoleBinding(pg, tsNamespace)
|
||||
serviceAccount := pgServiceAccount(pg, tsNamespace)
|
||||
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass)
|
||||
statefulSet, err := pgStatefulSet(pg, tsNamespace, testProxyImage, "auto", nil, proxyClass, shareACMEAccount)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -68,6 +68,12 @@ const (
|
||||
|
||||
AnnotationProxyGroup = "tailscale.com/proxy-group"
|
||||
|
||||
// AnnotationShareACMEAccount opts a single ProxyGroup into ("true")
|
||||
// or out of ("false") using the shared per-tailnet ACME account key.
|
||||
// When absent, OPERATOR_SHARED_ACME_ACCOUNT_KEY on the operator is
|
||||
// the default. See tailscale/tailscale#18251.
|
||||
AnnotationShareACMEAccount = "tailscale.com/share-acme-account"
|
||||
|
||||
// Annotations settable by users on ingresses.
|
||||
AnnotationFunnel = "tailscale.com/funnel"
|
||||
AnnotationHTTPRedirect = "tailscale.com/http-redirect"
|
||||
|
||||
Reference in New Issue
Block a user