// Copyright (c) Tailscale Inc & contributors // SPDX-License-Identifier: BSD-3-Clause //go:build !plan9 package main import ( "context" "encoding/json" "fmt" "math/rand/v2" "net/netip" "testing" "time" "go.uber.org/zap" corev1 "k8s.io/api/core/v1" discoveryv1 "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "tailscale.com/client/tailscale/v2" tsoperator "tailscale.com/k8s-operator" tsapi "tailscale.com/k8s-operator/apis/v1alpha1" "tailscale.com/k8s-operator/tsclient" "tailscale.com/kube/ingressservices" "tailscale.com/kube/kubetypes" "tailscale.com/tstest" "tailscale.com/util/mak" ) func TestServicePGReconciler(t *testing.T) { svcPGR, stateSecret, fc, ft, _ := setupServiceTest(t) svcs := []*corev1.Service{} config := []string{} for i := range 4 { svc, _ := setupTestService(t, fmt.Sprintf("test-svc-%d", i), "", fmt.Sprintf("1.2.3.%d", i), fc, stateSecret) svcs = append(svcs, svc) // Verify initial reconciliation expectReconciled(t, svcPGR, "default", svc.Name) config = append(config, fmt.Sprintf("svc:default-%s", svc.Name)) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, "test-pg", config) } for i, svc := range svcs { if err := fc.Delete(context.Background(), svc); err != nil { t.Fatalf("deleting Service: %v", err) } expectReconciled(t, svcPGR, "default", svc.Name) // Verify the ConfigMap was cleaned up cm := &corev1.ConfigMap{} if err := fc.Get(context.Background(), types.NamespacedName{ Name: "test-pg-ingress-config", Namespace: "operator-ns", }, cm); err != nil { t.Fatalf("getting ConfigMap: %v", err) } cfgs := ingressservices.Configs{} if err := json.Unmarshal(cm.BinaryData[ingressservices.IngressConfigKey], &cfgs); err != nil { t.Fatalf("unmarshaling serve config: %v", err) } if len(cfgs) > len(svcs)-(i+1) { t.Error("serve config not cleaned up") } config = removeEl(config, fmt.Sprintf("svc:default-%s", svc.Name)) verifyTailscaledConfig(t, fc, "test-pg", config) } } func TestServicePGReconciler_UpdateHostname(t *testing.T) { svcPGR, stateSecret, fc, ft, _ := setupServiceTest(t) cip := "4.1.6.7" svc, _ := setupTestService(t, "test-service", "", cip, fc, stateSecret) expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:default-%s", svc.Name), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:default-%s", svc.Name)}) hostname := "foobarbaz" mustUpdate(t, fc, svc.Namespace, svc.Name, func(s *corev1.Service) { mak.Set(&s.Annotations, AnnotationHostname, hostname) }) // NOTE: we need to update the ingress config Secret because there is no containerboot in the fake proxy Pod updateIngressConfigSecret(t, fc, stateSecret, hostname, cip) expectReconciled(t, svcPGR, "default", svc.Name) verifyTailscaleService(t, ft, fmt.Sprintf("svc:%s", hostname), []string{"do-not-validate"}) verifyTailscaledConfig(t, fc, "test-pg", []string{fmt.Sprintf("svc:%s", hostname)}) _, err := ft.VIPServices().Get(context.Background(), fmt.Sprintf("svc:default-%s", svc.Name)) if err == nil { t.Fatalf("svc:default-%s not cleaned up", svc.Name) } if !tailscale.IsNotFound(err) { t.Fatalf("unexpected error: %v", err) } } func setupServiceTest(t *testing.T) (*HAServiceReconciler, *corev1.Secret, client.Client, *fakeTSClient, *tstest.Clock) { // Pre-create the ProxyGroup pg := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pg", Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, }, } // Pre-create the ConfigMap for the ProxyGroup pgConfigMap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-ingress-config", Namespace: "operator-ns", }, BinaryData: map[string][]byte{ "serve-config.json": []byte(`{"Services":{}}`), }, } // Pre-create a config Secret for the ProxyGroup pgCfgSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: pgConfigSecretName("test-pg", 0), Namespace: "operator-ns", Labels: pgSecretLabels("test-pg", kubetypes.LabelSecretTypeConfig), }, Data: map[string][]byte{ tsoperator.TailscaledConfigFileName(pgMinCapabilityVersion): []byte(`{"Version":""}`), }, } pgStateSecret := &corev1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-0", Namespace: "operator-ns", }, Data: map[string][]byte{}, } pgPod := &corev1.Pod{ TypeMeta: metav1.TypeMeta{Kind: "Pod", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "test-pg-0", Namespace: "operator-ns", }, Status: corev1.PodStatus{ PodIPs: []corev1.PodIP{ { IP: "4.3.2.1", }, }, }, } fc := fake.NewClientBuilder(). WithScheme(tsapi.GlobalScheme). WithObjects(pg, pgCfgSecret, pgConfigMap, pgPod, pgStateSecret). WithStatusSubresource(pg). WithIndex(new(corev1.Service), indexIngressProxyGroup, indexPGIngresses). Build() // Set ProxyGroup status to ready pg.Status.Conditions = []metav1.Condition{ { Type: string(tsapi.ProxyGroupAvailable), Status: metav1.ConditionTrue, ObservedGeneration: 1, }, } if err := fc.Status().Update(context.Background(), pg); err != nil { t.Fatal(err) } ft := &fakeTSClient{ vipServices: make(map[string]tailscale.VIPService), } zl, err := zap.NewDevelopment() if err != nil { t.Fatal(err) } cl := tstest.NewClock(tstest.ClockOpts{}) svcPGR := &HAServiceReconciler{ Client: fc, clients: tsclient.NewProvider(ft), clock: cl, defaultTags: []string{"tag:k8s"}, tsNamespace: "operator-ns", logger: zl.Sugar(), recorder: record.NewFakeRecorder(10), } return svcPGR, pgStateSecret, fc, ft, cl } func TestValidateService(t *testing.T) { // Test that no more than one Kubernetes Service in a cluster refers to the same Tailscale Service. pgr, _, lc, _, cl := setupServiceTest(t) svc := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "my-app", Namespace: "ns-1", UID: types.UID("1234-UID"), Annotations: map[string]string{ "tailscale.com/proxy-group": "test-pg", "tailscale.com/hostname": "my-app", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.4", Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), }, } svc2 := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "my-app2", Namespace: "ns-2", UID: types.UID("1235-UID"), Annotations: map[string]string{ "tailscale.com/proxy-group": "test-pg", "tailscale.com/hostname": "my-app", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.5", Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), }, } wantSvc := &corev1.Service{ ObjectMeta: svc.ObjectMeta, TypeMeta: svc.TypeMeta, Spec: svc.Spec, Status: corev1.ServiceStatus{ Conditions: []metav1.Condition{ { Type: string(tsapi.IngressSvcValid), Status: metav1.ConditionFalse, Reason: reasonIngressSvcInvalid, LastTransitionTime: metav1.NewTime(cl.Now().Truncate(time.Second)), Message: `found duplicate Service "ns-2/my-app2" for hostname "my-app" - multiple HA Services for the same hostname on the same tailnet are not allowed`, }, }, }, } mustCreate(t, lc, svc) mustCreate(t, lc, svc2) expectReconciled(t, pgr, svc.Namespace, svc.Name) expectEqual(t, lc, wantSvc) } // Regression test for #20069. The pre-fix duplicate-hostname check scanned // every Service with shouldExpose=true, which meant a Service exposed on // one tailnet via the single-proxy path (svc.go) would block a ProxyGroup // ingress Service for the same hostname on a different tailnet. The // ProxyGroup path's validateService must skip Services that aren't // themselves managed by a ProxyGroup. func TestValidateService_SingleProxyServiceDoesNotCollideWithProxyGroup(t *testing.T) { pgr, _, lc, _, _ := setupServiceTest(t) // Service exposed via the single-proxy path: tailscale.com/expose=true // and no tailscale.com/proxy-group annotation. Its hostname matches // the ProxyGroup-managed Service below, but it lives in a different // reconciler entirely and must not be flagged as a duplicate. singleProxySvc := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "single-proxy", Namespace: "ns-1", UID: types.UID("single-proxy-uid"), Annotations: map[string]string{ "tailscale.com/expose": "true", "tailscale.com/hostname": "my-app", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.4", Type: corev1.ServiceTypeClusterIP, }, } // ProxyGroup-managed Service for the same hostname. pgSvc := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "pg-svc", Namespace: "ns-2", UID: types.UID("pg-svc-uid"), Annotations: map[string]string{ "tailscale.com/proxy-group": "test-pg", "tailscale.com/hostname": "my-app", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.5", Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), }, } mustCreate(t, lc, singleProxySvc) mustCreate(t, lc, pgSvc) expectReconciled(t, pgr, pgSvc.Namespace, pgSvc.Name) got := &corev1.Service{} if err := lc.Get(context.Background(), client.ObjectKeyFromObject(pgSvc), got); err != nil { t.Fatalf("get Service: %v", err) } for _, c := range got.Status.Conditions { if c.Type == string(tsapi.IngressSvcValid) && c.Status == metav1.ConditionFalse { t.Fatalf("ProxyGroup Service flagged invalid by a single-proxy Service on a different tailnet: %s", c.Message) } } } // Regression test for #20069. Two ProxyGroup-managed Services with the same // hostname but joined to different tailnets each have their own DNS // namespace and must not be flagged as duplicates. The Service being // reconciled here is on the default tailnet (so it uses the configured // fake tsclient); the conflicting Service already exists in-cluster on // a different tailnet and must be skipped by the duplicate check. func TestValidateService_DifferentTailnetDoesNotCollide(t *testing.T) { pgr, _, lc, _, _ := setupServiceTest(t) // Pre-create a ProxyGroup joined to a different tailnet. secondaryPG := &tsapi.ProxyGroup{ ObjectMeta: metav1.ObjectMeta{ Name: "secondary-pg", Generation: 1, }, Spec: tsapi.ProxyGroupSpec{ Type: tsapi.ProxyGroupTypeIngress, Tailnet: "secondary", }, } if err := lc.Create(context.Background(), secondaryPG); err != nil { t.Fatalf("create secondary ProxyGroup: %v", err) } // Pre-existing Service on the secondary tailnet with the conflicting hostname. otherTailnetSvc := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "other-tailnet-svc", Namespace: "ns-2", UID: types.UID("other-tailnet-svc-uid"), Annotations: map[string]string{ "tailscale.com/proxy-group": "secondary-pg", "tailscale.com/hostname": "my-app", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.5", Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), }, } mustCreate(t, lc, otherTailnetSvc) // Service being reconciled: same hostname, default tailnet ProxyGroup. primarySvc := &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "primary-svc", Namespace: "ns-1", UID: types.UID("primary-svc-uid"), Annotations: map[string]string{ "tailscale.com/proxy-group": "test-pg", "tailscale.com/hostname": "my-app", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "1.2.3.4", Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), }, } mustCreate(t, lc, primarySvc) expectReconciled(t, pgr, primarySvc.Namespace, primarySvc.Name) got := &corev1.Service{} if err := lc.Get(context.Background(), client.ObjectKeyFromObject(primarySvc), got); err != nil { t.Fatalf("get Service: %v", err) } for _, c := range got.Status.Conditions { if c.Type == string(tsapi.IngressSvcValid) && c.Status == metav1.ConditionFalse { t.Fatalf("ProxyGroup Service flagged invalid by a Service on a different tailnet with the same hostname: %s", c.Message) } } } func TestServicePGReconciler_MultiCluster(t *testing.T) { var ft *fakeTSClient for i := 0; i <= 10; i++ { pgr, stateSecret, fc, fti, _ := setupServiceTest(t) if i == 0 { ft = fti } else { pgr.clients = tsclient.NewProvider(ft) } svc, _ := setupTestService(t, "test-multi-cluster", "", "4.3.2.1", fc, stateSecret) expectReconciled(t, pgr, "default", svc.Name) tsSvcs, err := ft.VIPServices().List(t.Context()) if err != nil { t.Fatalf("getting Tailscale Service: %v", err) } if len(tsSvcs) != 1 { t.Fatalf("unexpected number of Tailscale Services (%d)", len(tsSvcs)) } for _, svc := range tsSvcs { t.Logf("found Tailscale Service with name %q", svc.Name) } } } func TestIgnoreRegularService(t *testing.T) { pgr, _, fc, ft, _ := setupServiceTest(t) svc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "test", Namespace: "default", // The apiserver is supposed to set the UID, but the fake client // doesn't. So, set it explicitly because other code later depends // on it being set. UID: types.UID("1234-UID"), Annotations: map[string]string{ "tailscale.com/expose": "true", }, }, Spec: corev1.ServiceSpec{ ClusterIP: "10.20.30.40", Type: corev1.ServiceTypeClusterIP, }, } mustCreate(t, fc, svc) expectReconciled(t, pgr, "default", "test") verifyTailscaledConfig(t, fc, "test-pg", nil) tsSvcs, err := ft.VIPServices().List(t.Context()) if err == nil { if len(tsSvcs) > 0 { t.Fatal("unexpected Tailscale Services found") } } } func removeEl(s []string, value string) []string { result := s[:0] for _, v := range s { if v != value { result = append(result, v) } } return result } func updateIngressConfigSecret(t *testing.T, fc client.Client, stateSecret *corev1.Secret, serviceName string, clusterIP string) { ingressConfig := ingressservices.Configs{ fmt.Sprintf("svc:%s", serviceName): ingressservices.Config{ IPv4Mapping: &ingressservices.Mapping{ TailscaleServiceIP: netip.MustParseAddr(vipTestIP), ClusterIP: netip.MustParseAddr(clusterIP), }, }, } ingressStatus := ingressservices.Status{ Configs: ingressConfig, PodIPv4: "4.3.2.1", } icJson, err := json.Marshal(ingressStatus) if err != nil { t.Fatalf("failed to json marshal ingress config: %s", err.Error()) } mustUpdate(t, fc, stateSecret.Namespace, stateSecret.Name, func(sec *corev1.Secret) { mak.Set(&sec.Data, ingressservices.IngressConfigKey, icJson) }) } func setupTestService(t *testing.T, svcName string, hostname string, clusterIP string, fc client.Client, stateSecret *corev1.Secret) (svc *corev1.Service, eps *discoveryv1.EndpointSlice) { uid := rand.IntN(100) svc = &corev1.Service{ TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: svcName, Namespace: "default", UID: types.UID(fmt.Sprintf("%d-UID", uid)), Annotations: map[string]string{ "tailscale.com/proxy-group": "test-pg", }, }, Spec: corev1.ServiceSpec{ Type: corev1.ServiceTypeLoadBalancer, LoadBalancerClass: new("tailscale"), ClusterIP: clusterIP, ClusterIPs: []string{clusterIP}, }, } eps = &discoveryv1.EndpointSlice{ TypeMeta: metav1.TypeMeta{Kind: "EndpointSlice", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: svcName, Namespace: "default", Labels: map[string]string{ discoveryv1.LabelServiceName: svcName, }, }, AddressType: discoveryv1.AddressTypeIPv4, Endpoints: []discoveryv1.Endpoint{ { Addresses: []string{"4.3.2.1"}, Conditions: discoveryv1.EndpointConditions{ Ready: new(true), }, }, }, } updateIngressConfigSecret(t, fc, stateSecret, fmt.Sprintf("default-%s", svcName), clusterIP) mustCreate(t, fc, svc) mustCreate(t, fc, eps) return svc, eps }