tailscale/k8s-operator/reconciler/peerrelay/service.go

// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause

//go:build !plan9

package peerrelay

import (
	"context"
	"fmt"
	"maps"
	"net/netip"
	"slices"
	"strconv"
	"time"

	"go.uber.org/zap"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/intstr"

	tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
	"tailscale.com/k8s-operator/reconciler"
)

const (
	// labelReplicaIndex stores the replica index of a managed Service so it can be matched back to a specific
	// peer relay instance.
	labelReplicaIndex = "tailscale.com/peer-relay-replica"

	// parentTypePeerRelay is the value used for reconciler.LabelParentType on PeerRelay-managed resources.
	parentTypePeerRelay = "peerrelay"

	// servicePortName names the UDP port exposed by each Service. Mostly cosmetic, but Kubernetes requires a name
	// once a Service has more than one port; using a stable name keeps the door open for that.
	servicePortName = "peerrelay"

	// servicePort is the UDP port that each peer relay container will listen on and that the LoadBalancer Service
	// exposes externally.
	servicePort = 41641

	annotationEIPAllocations = "service.beta.kubernetes.io/aws-load-balancer-eip-allocations"
	annotationSubnets        = "service.beta.kubernetes.io/aws-load-balancer-subnets"
)

// cloudAnnotations are the cloud-provider-specific annotations applied to every generated LoadBalancer Service to
// ensure the Service is provisioned with a publicly addressable IP rather than a DNS name.
var cloudAnnotations = map[string]string{
	// AWS: provision an internet-facing NLB in IP target mode via the AWS Load Balancer Controller.
	"service.beta.kubernetes.io/aws-load-balancer-type":            "external",
	"service.beta.kubernetes.io/aws-load-balancer-nlb-target-type": "ip",
	"service.beta.kubernetes.io/aws-load-balancer-scheme":          "internet-facing",
	"service.beta.kubernetes.io/aws-load-balancer-ip-address-type": "ipv4",

	// Azure: pin the LB to external.
	"service.beta.kubernetes.io/azure-load-balancer-internal": "false",
}

func peerRelayLabels(prName string) map[string]string {
	return reconciler.Labels(parentTypePeerRelay, prName, "")
}

func peerRelayServiceLabels(prName string, idx int32) map[string]string {
	labels := peerRelayLabels(prName)
	labels[labelReplicaIndex] = strconv.FormatInt(int64(idx), 10)
	return labels
}

func resourceName(prName string) string {
	return "peerrelay-" + prName
}

func replicaName(prName string, idx int32) string {
	return fmt.Sprintf("%s-%d", resourceName(prName), idx)
}

func peerRelayServiceAnnotations(pr *tsapi.PeerRelay, idx int32) map[string]string {
	annotations := make(map[string]string, len(cloudAnnotations))

	if pr.Spec.Service != nil {
		maps.Copy(annotations, pr.Spec.Service.Annotations)
	}

	maps.Copy(annotations, cloudAnnotations)

	// Per-replica AWS pinning always wins over anything in spec.service.annotations or the cloud defaults so users
	// can rely on spec.aws.elasticIPs being the single source of truth for each replica's EIP + subnet.
	if pr.Spec.AWS != nil && int(idx) < len(pr.Spec.AWS.ElasticIPs) {
		eip := pr.Spec.AWS.ElasticIPs[idx]
		annotations[annotationEIPAllocations] = eip.AllocationID
		annotations[annotationSubnets] = eip.SubnetID
	}

	return annotations
}

func (r *Reconciler) peerRelayService(pr *tsapi.PeerRelay, idx int32) *corev1.Service {
	name := replicaName(pr.Name, idx)

	return &corev1.Service{
		TypeMeta: metav1.TypeMeta{
			APIVersion: "v1",
			Kind:       "Service",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:        name,
			Namespace:   r.tailscaleNamespace,
			Labels:      peerRelayServiceLabels(pr.Name, idx),
			Annotations: peerRelayServiceAnnotations(pr, idx),
		},
		Spec: corev1.ServiceSpec{
			Type: corev1.ServiceTypeLoadBalancer,
			// The Service targets the specific StatefulSet pod for this replica. The StatefulSet controller
			// automatically sets this label on each pod.
			Selector: map[string]string{
				"statefulset.kubernetes.io/pod-name": name,
			},
			Ports: []corev1.ServicePort{
				{
					Name:       servicePortName,
					Protocol:   corev1.ProtocolUDP,
					Port:       servicePort,
					TargetPort: intstr.FromInt32(servicePort),
				},
			},
		},
	}
}

func replicaIndexFromLabels(labels map[string]string) (int32, bool) {
	raw, ok := labels[labelReplicaIndex]
	if !ok {
		return 0, false
	}

	n, err := strconv.ParseInt(raw, 10, 32)
	if err != nil {
		return 0, false
	}

	return int32(n), true
}

func (r *Reconciler) peerRelayEndpoint(ctx context.Context, logger *zap.SugaredLogger, svc *corev1.Service, prev *tsapi.PeerRelayEndpoint) *tsapi.PeerRelayEndpoint {
	idx, ok := replicaIndexFromLabels(svc.Labels)
	if !ok {
		return nil
	}

	for _, ing := range svc.Status.LoadBalancer.Ingress {
		if ing.IP != "" {
			return &tsapi.PeerRelayEndpoint{Replica: idx, Address: ing.IP, Port: servicePort}
		}
	}

	// Just return nil if we're not dealing with AWS fun.
	if _, ok = svc.Annotations[annotationEIPAllocations]; !ok {
		return nil
	}

	// If we were not able to obtain an IP address, we fall back to an IPv4 lookup. This is specifically for the case
	// of AWS where NLB-backed Service resources are only ever given hostnames. We expect users to also provide
	// an annotation with their elastic IP allocations so that there is only ever 1 IP address behind the hostname, so
	// we perform a lookup so that the user doesn't also need to provide that IP address.
	for _, ing := range svc.Status.LoadBalancer.Ingress {
		if ing.Hostname == "" {
			continue
		}

		resolveCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
		defer cancel()

		addrs, err := r.resolver(resolveCtx, "ip4", ing.Hostname)
		if err != nil || len(addrs) == 0 {
			logger.Warnf("failed to resolve LoadBalancer hostname %q for Service %q: %v", ing.Hostname, svc.Name, err)
			// Preserve the previously-known endpoint (if any) so that a failure here doesn't erase status.endpoints.
			if prev != nil {
				return prev
			}

			continue
		}

		slices.SortFunc(addrs, netip.Addr.Compare)
		return &tsapi.PeerRelayEndpoint{Replica: idx, Address: addrs[0].String(), Port: servicePort}
	}

	return nil
}