// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 package main import ( "context" "encoding/json" "fmt" "testing" "github.com/AlekSi/pointer" "github.com/google/go-cmp/cmp" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/kube/egressservices" "tailscale.com/tstest" "tailscale.com/tstime" ) func TestTailscaleEgressServices(t *testing.T) { pg := &tsapi.ProxyGroup{ TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{ Name: "foo", UID: types.UID("1234-UID"), }, Spec: tsapi.ProxyGroupSpec{ Replicas: pointer.To[int32](3), Type: tsapi.ProxyGroupTypeEgress, }, } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: pgEgressCMName("foo"), Namespace: "operator-ns", }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). WithObjects(pg, cm). WithStatusSubresource(pg). WithInterceptorFuncs(interceptor.Funcs{ Create: clusterIPInterceptor("10.96.0.1"), }). Build() zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } clock := tstest.NewClock(tstest.ClockOpts{}) esr := &egressSvcsReconciler{ Client: fc, logger: zl.Sugar(), clock: clock, tsNamespace: "operator-ns", } tailnetTargetFQDN := "foo.bar.ts.net." svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", UID: types.UID("1234-UID"), Annotations: map[string]string{ AnnotationTailnetTargetFQDN: tailnetTargetFQDN, AnnotationProxyGroup: "foo", }, }, Spec: corev1.ServiceSpec{ ExternalName: "placeholder", Type: corev1.ServiceTypeExternalName, Selector: nil, Ports: []corev1.ServicePort{ { Protocol: "TCP", Port: 80, }, }, }, } t.Run("service_one_unnamed_port", func(t *testing.T) { mustCreate(t, fc, svc) expectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_add_two_named_ports", func(t *testing.T) { svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}} mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { s.Spec.Ports = svc.Spec.Ports }) expectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_add_udp_port", func(t *testing.T) { svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Port: 53, Protocol: "UDP", Name: "dns"}) mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { s.Spec.Ports = svc.Spec.Ports }) expectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("service_change_protocol", func(t *testing.T) { svc.Spec.Ports = []corev1.ServicePort{{Protocol: "TCP", Port: 80, Name: "http"}, {Protocol: "TCP", Port: 443, Name: "https"}, {Port: 53, Protocol: "TCP", Name: "tcp_dns"}} mustUpdate(t, fc, "default", "test", func(s *corev1.Service) { s.Spec.Ports = svc.Spec.Ports }) expectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("endpointslice_deletion_recovery", func(t *testing.T) { name := findGenNameForEgressSvcResources(t, fc, svc) epsName := fmt.Sprintf("%s-ipv4", name) // Delete the EndpointSlice and verify it is recreated. eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: epsName, Namespace: "operator-ns", }, } if err := fc.Delete(t.Context(), eps); err != nil { t.Fatalf("error deleting EndpointSlice: %v", err) } expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) validateReadyService(t, fc, esr, svc, clock, zl, cm) }) t.Run("delete_external_name_service", func(t *testing.T) { name := findGenNameForEgressSvcResources(t, fc, svc) if err := fc.Delete(context.Background(), svc); err != nil { t.Fatalf("error deleting ExternalName Service: %v", err) } expectReconciled(t, esr, "default", "test") // Verify that ClusterIP Service and EndpointSlice have been deleted. expectMissing[corev1.Service](t, fc, "operator-ns", name) expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) // Verify that service config has been deleted from the ConfigMap. mustNotHaveConfigForSvc(t, fc, svc, cm) }) } func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReconciler, svc *corev1.Service, clock *tstest.Clock, zl *zap.Logger, cm *corev1.ConfigMap) { expectReconciled(t, esr, "default", "test") // Verify that a ClusterIP Service has been created. name := findGenNameForEgressSvcResources(t, fc, svc) expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc, removeClusterIPsFromSvc) clusterSvc := mustGetClusterIPSvc(t, fc, name) // Verify that an EndpointSlice has been created. expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) // Verify that ConfigMap contains configuration for the new egress service. mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl) r := svcConfiguredReason(svc, true, zl.Sugar()) // Verify that the user-created ExternalName Service has Configured set to true and ExternalName pointing to the // CluterIP Service. svc.Status.Conditions = []metav1.Condition{ condition(tsapi.EgressSvcValid, metav1.ConditionTrue, "EgressSvcValid", "EgressSvcValid", clock), condition(tsapi.EgressSvcConfigured, metav1.ConditionTrue, r, r, clock), } svc.ObjectMeta.Finalizers = []string{"tailscale.com/finalizer"} svc.Spec.ExternalName = fmt.Sprintf("%s.operator-ns.svc.cluster.local", name) expectEqual(t, fc, svc) } func condition(typ tsapi.ConditionType, st metav1.ConditionStatus, r, msg string, clock tstime.Clock) metav1.Condition { return metav1.Condition{ Type: string(typ), Status: st, LastTransitionTime: conditionTime(clock), Reason: r, Message: msg, } } func findGenNameForEgressSvcResources(t *testing.T, client client.Client, svc *corev1.Service) string { t.Helper() labels := egressSvcChildResourceLabels(svc) s, err := getSingleObject[corev1.Service](context.Background(), client, "operator-ns", labels) if err != nil { t.Fatalf("finding ClusterIP Service for ExternalName Service %s: %v", svc.Name, err) } if s == nil { t.Fatalf("no ClusterIP Service found for ExternalName Service %q", svc.Name) } return s.GetName() } func clusterIPSvc(name string, extNSvc *corev1.Service) *corev1.Service { labels := egressSvcChildResourceLabels(extNSvc) ports := make([]corev1.ServicePort, len(extNSvc.Spec.Ports)) for i, port := range extNSvc.Spec.Ports { ports[i] = corev1.ServicePort{ // Copy the port to avoid modifying the original. Name: port.Name, Port: port.Port, Protocol: port.Protocol, } if port.Name == "" { ports[i].Name = "tailscale-unnamed" } } ports = append(ports, corev1.ServicePort{ Name: "tailscale-health-check", Port: 9002, TargetPort: intstr.FromInt(9002), Protocol: "TCP", }) return &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "operator-ns", GenerateName: fmt.Sprintf("ts-%s-", extNSvc.Name), Labels: labels, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeClusterIP, IPFamilyPolicy: new(corev1.IPFamilyPolicyPreferDualStack), Ports: ports, }, } } func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Service { svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, Namespace: "operator-ns", }, } if err := cl.Get(context.Background(), client.ObjectKeyFromObject(svc), svc); err != nil { t.Fatalf("error retrieving Service") } return svc } func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service, addrType discoveryv1.AddressType) *discoveryv1.EndpointSlice { labels := egressSvcChildResourceLabels(extNSvc) labels[discoveryv1.LabelManagedBy] = "tailscale.com" labels[discoveryv1.LabelServiceName] = name suffix := "ipv4" if addrType == discoveryv1.AddressTypeIPv6 { suffix = "ipv6" } return &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-%s", name, suffix), Namespace: "operator-ns", Labels: labels, }, Ports: portsForEndpointSlice(clusterIPSvc), AddressType: addrType, } } func portsForEndpointSlice(svc *corev1.Service) []discoveryv1.EndpointPort { ports := make([]discoveryv1.EndpointPort, 0) for _, p := range svc.Spec.Ports { ports = append(ports, discoveryv1.EndpointPort{ Name: &p.Name, Protocol: &p.Protocol, Port: new(p.TargetPort.IntVal), }) } return ports } func mustHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc, clusterIPSvc *corev1.Service, cm *corev1.ConfigMap, lg *zap.Logger) { t.Helper() wantsCfg := egressSvcCfg(extNSvc, clusterIPSvc, clusterIPSvc.Namespace, lg.Sugar()) if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil { t.Fatalf("Error retrieving ConfigMap: %v", err) } name := tailnetSvcName(extNSvc) gotCfg := configFromCM(t, cm, name) if gotCfg == nil { t.Fatalf("No config found for service %q", name) } if diff := cmp.Diff(*gotCfg, wantsCfg); diff != "" { t.Fatalf("unexpected config for service %q (-got +want):\n%s", name, diff) } } func mustNotHaveConfigForSvc(t *testing.T, cl client.Client, extNSvc *corev1.Service, cm *corev1.ConfigMap) { t.Helper() if err := cl.Get(context.Background(), client.ObjectKeyFromObject(cm), cm); err != nil { t.Fatalf("Error retrieving ConfigMap: %v", err) } name := tailnetSvcName(extNSvc) gotCfg := configFromCM(t, cm, name) if gotCfg != nil { t.Fatalf("Config %#+v for service %q found when it should not be present", gotCfg, name) } } func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressservices.Config { t.Helper() cfgBs, ok := cm.BinaryData[egressservices.KeyEgressServices] if !ok { return nil } cfgs := egressservices.Configs{} if err := json.Unmarshal(cfgBs, &cfgs); err != nil { t.Fatalf("error unmarshalling config: %v", err) } cfg, ok := cfgs[svcName] if ok { return &cfg } return nil } func TestTailscaleEgressServicesDualStack(t *testing.T) { pg := &tsapi.ProxyGroup{ TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{ Name: "foo", UID: types.UID("1234-UID"), }, Spec: tsapi.ProxyGroupSpec{ Replicas: pointer.To[int32](3), Type: tsapi.ProxyGroupTypeEgress, }, } cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: pgEgressCMName("foo"), Namespace: "operator-ns", }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). WithObjects(pg, cm). WithStatusSubresource(pg). WithInterceptorFuncs(interceptor.Funcs{ Create: clusterIPInterceptor("10.96.0.1", "fd00::1"), }). Build() zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } clock := tstest.NewClock(tstest.ClockOpts{}) esr := &egressSvcsReconciler{ Client: fc, logger: zl.Sugar(), clock: clock, tsNamespace: "operator-ns", } svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", UID: types.UID("1234-UID"), Annotations: map[string]string{ AnnotationTailnetTargetFQDN: "foo.bar.ts.net.", AnnotationProxyGroup: "foo", }, }, Spec: corev1.ServiceSpec{ ExternalName: "placeholder", Type: corev1.ServiceTypeExternalName, Selector: nil, Ports: []corev1.ServicePort{ { Protocol: "TCP", Port: 80, }, }, }, } t.Run("dual_stack_creates_both_endpoint_slices", func(t *testing.T) { mustCreate(t, fc, svc) expectReconciled(t, esr, "default", "test") validateReadyService(t, fc, esr, svc, clock, zl, cm) // Also verify the IPv6 EndpointSlice was created. name := findGenNameForEgressSvcResources(t, fc, svc) clusterSvc := mustGetClusterIPSvc(t, fc, name) expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) }) t.Run("dual_stack_endpointslice_deletion_recovery", func(t *testing.T) { name := findGenNameForEgressSvcResources(t, fc, svc) // Delete both IPv4 and IPv6 EndpointSlices. for _, suffix := range []string{"ipv4", "ipv6"} { epsName := fmt.Sprintf("%s-%s", name, suffix) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: epsName, Namespace: "operator-ns", }, } if err := fc.Delete(t.Context(), eps); err != nil { t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err) } expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) } // Reconcile should recreate both. validateReadyService(t, fc, esr, svc, clock, zl, cm) clusterSvc := mustGetClusterIPSvc(t, fc, name) expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) }) t.Run("dual_stack_single_endpointslice_deletion_recovery", func(t *testing.T) { name := findGenNameForEgressSvcResources(t, fc, svc) // Delete only the IPv6 EndpointSlice. epsName := fmt.Sprintf("%s-ipv6", name) eps := &discoveryv1.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name: epsName, Namespace: "operator-ns", }, } if err := fc.Delete(t.Context(), eps); err != nil { t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err) } expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName) // Reconcile should recreate the missing IPv6 EndpointSlice while leaving // the IPv4 one untouched. validateReadyService(t, fc, esr, svc, clock, zl, cm) clusterSvc := mustGetClusterIPSvc(t, fc, name) expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6)) expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4)) }) t.Run("delete_dual_stack_service", func(t *testing.T) { name := findGenNameForEgressSvcResources(t, fc, svc) if err := fc.Delete(context.Background(), svc); err != nil { t.Fatalf("error deleting ExternalName Service: %v", err) } expectReconciled(t, esr, "default", "test") expectMissing[corev1.Service](t, fc, "operator-ns", name) expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name)) expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name)) mustNotHaveConfigForSvc(t, fc, svc, cm) }) } // clusterIPInterceptor returns an interceptor.Funcs Create function that // simulates the API server assigning ClusterIPs to ClusterIP Services. // This is required because the reconciler iterates ClusterIPs to create // per-family EndpointSlices but the fake client does not assign ClusterIPs. func clusterIPInterceptor(clusterIPs ...string) func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { return func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { if svc, ok := obj.(*corev1.Service); ok && svc.Spec.Type == corev1.ServiceTypeClusterIP { svc.Spec.ClusterIPs = clusterIPs svc.Spec.ClusterIP = clusterIPs[0] } return c.Create(ctx, obj, opts...) } }