cmd/k8s-operator,ipn/ipnlocal: allow opting out of ACME order replace extension (#18252)

In dynamically changing environments where ACME account keys and certs
are stored separately, it can happen that the account key would get
deleted (and recreated) between issuances. If that is the case,
we currently fail renewals and the only way to recover is for users
to delete certs.
This adds a config knob to allow opting out of the replaces extension
and utilizes it in the Kubernetes operator where there are known
user workflows that could end up with this edge case.

Updates #18251

Signed-off-by: Irbe Krumina <irbe@tailscale.com>

cmd/k8s-operator/proxygroup_specs.go

@@ -182,6 +182,14 @@ func pgStatefulSet(pg *tsapi.ProxyGroup, namespace, image, tsFirewallMode string
 				Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
 				Value: "/etc/tsconfig/$(POD_NAME)",
 			},
+			{
+				// This ensures that cert renewals can succeed if ACME account
+				// keys have changed since issuance. We cannot guarantee or
+				// validate that the account key has not changed, see
+				// https://github.com/tailscale/tailscale/issues/18251
+				Name:  "TS_DEBUG_ACME_FORCE_RENEWAL",
+				Value: "true",
+			},
 		}
 
 		if port != nil {
@@ -347,6 +355,14 @@ func kubeAPIServerStatefulSet(pg *tsapi.ProxyGroup, namespace, image string, por
 											Name:      "$(POD_NAME)-config",
 										}.String(),
 									},
+									{
+										// This ensures that cert renewals can succeed if ACME account
+										// keys have changed since issuance. We cannot guarantee or
+										// validate that the account key has not changed, see
+										// https://github.com/tailscale/tailscale/issues/18251
+										Name:  "TS_DEBUG_ACME_FORCE_RENEWAL",
+										Value: "true",
+									},
 								}
 
 								if port != nil {

cmd/k8s-operator/sts.go

@@ -671,6 +671,14 @@ func (a *tailscaleSTSReconciler) reconcileSTS(ctx context.Context, logger *zap.S
 			Name:  "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR",
 			Value: "/etc/tsconfig/$(POD_NAME)",
 		},
+		corev1.EnvVar{
+			// This ensures that cert renewals can succeed if ACME account
+			// keys have changed since issuance. We cannot guarantee or
+			// validate that the account key has not changed, see
+			// https://github.com/tailscale/tailscale/issues/18251
+			Name:  "TS_DEBUG_ACME_FORCE_RENEWAL",
+			Value: "true",
+		},
 	)
 
 	if sts.ForwardClusterTrafficViaL7IngressProxy {

cmd/k8s-operator/testutils_test.go

@@ -92,6 +92,7 @@ func expectedSTS(t *testing.T, cl client.Client, opts configOpts) *appsv1.Statef
 			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
+			{Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"},
 		},
 		SecurityContext: &corev1.SecurityContext{
 			Privileged: ptr.To(true),
@@ -287,6 +288,7 @@ func expectedSTSUserspace(t *testing.T, cl client.Client, opts configOpts) *apps
 			{Name: "POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "", FieldPath: "metadata.uid"}, ResourceFieldRef: nil, ConfigMapKeyRef: nil, SecretKeyRef: nil}},
 			{Name: "TS_KUBE_SECRET", Value: "$(POD_NAME)"},
 			{Name: "TS_EXPERIMENTAL_VERSIONED_CONFIG_DIR", Value: "/etc/tsconfig/$(POD_NAME)"},
+			{Name: "TS_DEBUG_ACME_FORCE_RENEWAL", Value: "true"},
 			{Name: "TS_SERVE_CONFIG", Value: "/etc/tailscaled/$(POD_NAME)/serve-config"},
 			{Name: "TS_INTERNAL_APP", Value: opts.app},
 		},