cmd/k8s-operator: ensure EndpointSlices exist on every egress reconcile (#20347)

EndpointSlices were created in provision(), which was called only if certain
fields on the ExternalName Service had changed. If an EndpointSlice was
deleted, it was never re-created (because the owning Service had not
changed).

Move EndpointSlice provisioning after this gated provision step so that it
runs on every reconcile.

Fixes #20322

Change-Id: I416fb5e4b40f2029efb97aa6ca7ceb3e31b0d52d

Signed-off-by: Becky Pauley <becky@tailscale.com>

cmd/k8s-operator/egress-services.go

@@ -202,6 +202,10 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
 		return nil
 	}
 
+	if err := esr.ensureEndpointSlices(ctx, svc, clusterIPSvc, lg); err != nil {
+		return err
+	}
+
 	// Update ExternalName Service to point at the ClusterIP Service.
 	clusterDomain := retrieveClusterDomain(esr.tsNamespace, lg)
 	clusterIPSvcFQDN := fmt.Sprintf("%s.%s.svc.%s", clusterIPSvc.Name, clusterIPSvc.Namespace, clusterDomain)
@@ -218,6 +222,32 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
 	return nil
 }
 
+// ensureEndpointSlices ensures that an IPv4 EndpointSlice exists for the egress
+// service and that its ports are up to date.
+func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error {
+	crl := egressSvcEpsLabels(svc, clusterIPSvc)
+	eps := &discoveryv1.EndpointSlice{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
+			Namespace: esr.tsNamespace,
+			Labels:    crl,
+		},
+		AddressType: discoveryv1.AddressTypeIPv4,
+		Ports:       epsPortsFromSvc(clusterIPSvc),
+	}
+	if _, err := createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
+		e.Labels = eps.Labels
+		e.AddressType = eps.AddressType
+		e.Ports = eps.Ports
+		for _, p := range e.Endpoints {
+			p.Conditions.Ready = nil
+		}
+	}); err != nil {
+		return fmt.Errorf("error ensuring EndpointSlice: %w", err)
+	}
+	return nil
+}
+
 func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName string, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) (*corev1.Service, bool, error) {
 	lg.Infof("updating configuration...")
 	usedPorts, err := esr.usedPortsForPG(ctx, proxyGroupName)
@@ -316,29 +346,6 @@ func (esr *egressSvcsReconciler) provision(ctx context.Context, proxyGroupName s
 		}
 	}
 
-	crl := egressSvcEpsLabels(svc, clusterIPSvc)
-	// TODO(irbekrm): support IPv6, but need to investigate how kube proxy
-	// sets up Service -> Pod routing when IPv6 is involved.
-	eps := &discoveryv1.EndpointSlice{
-		ObjectMeta: metav1.ObjectMeta{
-			Name:      fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
-			Namespace: esr.tsNamespace,
-			Labels:    crl,
-		},
-		AddressType: discoveryv1.AddressTypeIPv4,
-		Ports:       epsPortsFromSvc(clusterIPSvc),
-	}
-	if eps, err = createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
-		e.Labels = eps.Labels
-		e.AddressType = eps.AddressType
-		e.Ports = eps.Ports
-		for _, p := range e.Endpoints {
-			p.Conditions.Ready = nil
-		}
-	}); err != nil {
-		return nil, false, fmt.Errorf("error ensuring EndpointSlice: %w", err)
-	}
-
 	cm, cfgs, err := egressSvcsConfigs(ctx, esr.Client, proxyGroupName, esr.tsNamespace)
 	if err != nil {
 		return nil, false, fmt.Errorf("error retrieving egress services configuration: %w", err)

cmd/k8s-operator/egress-services_test.go

@@ -117,6 +117,23 @@ func TestTailscaleEgressServices(t *testing.T) {
 		validateReadyService(t, fc, esr, svc, clock, zl, cm)
 	})
 
+	t.Run("endpointslice_deletion_recovery", func(t *testing.T) {
+		name := findGenNameForEgressSvcResources(t, fc, svc)
+		epsName := fmt.Sprintf("%s-ipv4", name)
+		// Delete the EndpointSlice and verify it is recreated.
+		eps := &discoveryv1.EndpointSlice{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      epsName,
+				Namespace: "operator-ns",
+			},
+		}
+		if err := fc.Delete(t.Context(), eps); err != nil {
+			t.Fatalf("error deleting EndpointSlice: %v", err)
+		}
+		expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
+		validateReadyService(t, fc, esr, svc, clock, zl, cm)
+	})
+
 	t.Run("delete_external_name_service", func(t *testing.T) {
 		name := findGenNameForEgressSvcResources(t, fc, svc)
 		if err := fc.Delete(context.Background(), svc); err != nil {