WIP: rebase fork onto upstream/main (v1.103.0) #15

Closed
codinget wants to merge 670 commits from webnet into save/webnet-2026-07-29
16 changed files with 774 additions and 93 deletions
Showing only changes of commit 6ee7bcb458 - Show all commits
+43 -18
View File
@@ -57,7 +57,8 @@ type egressProxy struct {
netmapChan chan netmapState // chan to receive netmap state updates on
podIPv4 string // never empty string, currently only IPv4 is supported
podIPv4 string // empty if Pod does not have IPv4 address
podIPv6 string // empty if Pod does not have IPv6 address
// tailnetFQDNs is the egress service FQDN to tailnet IP mappings that
// were last used to configure firewall rules for this proxy.
@@ -138,6 +139,7 @@ type egressProxyRunOpts struct {
stateSecret string
netmapChan chan netmapState
podIPv4 string
podIPv6 string
tailnetAddrs []netip.Prefix
}
@@ -150,6 +152,7 @@ func (ep *egressProxy) configure(opts egressProxyRunOpts) {
ep.stateSecret = opts.stateSecret
ep.netmapChan = opts.netmapChan
ep.podIPv4 = opts.podIPv4
ep.podIPv6 = opts.podIPv6
ep.tailnetAddrs = opts.tailnetAddrs
ep.client = &http.Client{} // default HTTP client
sleepDuration := time.Second
@@ -418,7 +421,7 @@ func (ep *egressProxy) getStatus(ctx context.Context) (*egressservices.Status, e
if err := json.Unmarshal([]byte(raw), status); err != nil {
return nil, fmt.Errorf("error unmarshalling previous config: %w", err)
}
if reflect.DeepEqual(status.PodIPv4, ep.podIPv4) {
if status.PodIPv4 == ep.podIPv4 && status.PodIPv6 == ep.podIPv6 {
return status, nil
}
return nil, nil
@@ -432,6 +435,7 @@ func (ep *egressProxy) setStatus(ctx context.Context, status *egressservices.Sta
status = &egressservices.Status{}
}
status.PodIPv4 = ep.podIPv4
status.PodIPv6 = ep.podIPv6
secret, err := ep.kc.GetSecret(ctx, ep.stateSecret)
if err != nil {
return fmt.Errorf("error retrieving state Secret: %w", err)
@@ -622,6 +626,8 @@ func servicesStatusIsEqual(st, st1 *egressservices.Status) bool {
}
st.PodIPv4 = ""
st1.PodIPv4 = ""
st.PodIPv6 = ""
st1.PodIPv6 = ""
return reflect.DeepEqual(*st, *st1)
}
@@ -673,24 +679,29 @@ func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs egressse
continue
}
svc := s
// TODO(beckypauley): In dual-stack clusters, this is a best-effort check as we do not control which IP family is used.
// This confirms removal from routing on this node for one family only. The other IP family then relies on the longSleep below.
wg.Go(func() {
log.Printf("Ensuring that cluster traffic is no longer routed to %q via this Pod...", svc)
podIP, header := ep.podIPv4, kubetypes.PodIPv4Header
if podIP == "" {
podIP, header = ep.podIPv6, kubetypes.PodIPv6Header
}
if ep.podDrained(ctx, svc, hep, podIP, header, hp) {
return
}
ticker := time.NewTicker(ep.shortSleep)
defer ticker.Stop()
for {
if ctx.Err() != nil { // kubelet's HTTP request timeout
select {
case <-ctx.Done(): // kubelet's HTTP request timeout
log.Printf("Cluster traffic for %s did not stop being routed to this Pod.", svc)
return
case <-ticker.C:
if ep.podDrained(ctx, svc, hep, podIP, header, hp) {
return
}
}
found, err := lookupPodRoute(ctx, hep, ep.podIPv4, hp, ep.client)
if err != nil {
log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err)
break
}
if !found {
log.Printf("service %q is no longer routed through this Pod", svc)
break
}
log.Printf("service %q is still routed through this Pod, waiting...", svc)
time.Sleep(ep.shortSleep)
}
})
}
@@ -704,9 +715,9 @@ func (ep *egressProxy) waitTillSafeToShutdown(ctx context.Context, cfgs egressse
// lookupPodRoute calls the healthcheck endpoint repeat times and returns true if the endpoint returns with the podIP
// header at least once.
func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client httpClient) (bool, error) {
func lookupPodRoute(ctx context.Context, hep, podIP, podIPHeader string, repeat int, client httpClient) (bool, error) {
for range repeat {
f, err := lookup(ctx, hep, podIP, client)
f, err := lookup(ctx, hep, podIP, podIPHeader, client)
if err != nil {
return false, err
}
@@ -718,7 +729,7 @@ func lookupPodRoute(ctx context.Context, hep, podIP string, repeat int, client h
}
// lookup calls the healthcheck endpoint and returns true if the response contains the podIP header.
func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, error) {
func lookup(ctx context.Context, hep, podIP, podIPHeader string, client httpClient) (bool, error) {
req, err := http.NewRequestWithContext(ctx, httpm.GET, hep, nil)
if err != nil {
return false, fmt.Errorf("error creating new HTTP request: %v", err)
@@ -733,7 +744,7 @@ func lookup(ctx context.Context, hep, podIP string, client httpClient) (bool, er
return true, nil
}
defer resp.Body.Close()
gotIP := resp.Header.Get(kubetypes.PodIPv4Header)
gotIP := resp.Header.Get(podIPHeader)
return strings.EqualFold(podIP, gotIP), nil
}
@@ -762,3 +773,17 @@ func (ep *egressProxy) getHEPPings() (int, error) {
}
return hp, nil
}
func (ep *egressProxy) podDrained(ctx context.Context, svc, hep, podIP, header string, hp int) bool {
found, err := lookupPodRoute(ctx, hep, podIP, header, hp, ep.client)
if err != nil {
log.Printf("unable to reach endpoint %q, assuming the routing rules for this Pod have been deleted: %v", hep, err)
return true
}
if !found {
log.Printf("service %q is no longer routed through this Pod", svc)
return true
}
log.Printf("service %q is still routed through this Pod, waiting...", svc)
return false
}
+3 -1
View File
@@ -15,6 +15,7 @@ import (
"strings"
"sync"
"testing"
"time"
"tailscale.com/kube/egressservices"
"tailscale.com/kube/kubetypes"
@@ -269,7 +270,8 @@ func TestWaitTillSafeToShutdown(t *testing.T) {
}
ep := &egressProxy{
podIPv4: podIP,
podIPv4: podIP,
shortSleep: time.Millisecond,
client: &mockHTTPClient{
podIP: podIP,
anotherIP: anotherIP,
+3 -2
View File
@@ -423,7 +423,7 @@ func run() error {
mux := http.NewServeMux()
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf)
healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, cfg.PodIPv6, log.Printf)
close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
defer close()
@@ -439,7 +439,7 @@ func run() error {
if cfg.localHealthEnabled() {
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, log.Printf)
healthCheck = healthz.RegisterHealthHandlers(mux, cfg.PodIPv4, cfg.PodIPv6, log.Printf)
}
if cfg.egressSvcsTerminateEPEnabled() {
@@ -948,6 +948,7 @@ runLoop:
stateSecret: cfg.KubeSecret,
netmapChan: egressSvcsNotify,
podIPv4: cfg.PodIPv4,
podIPv6: cfg.PodIPv6,
tailnetAddrs: addrs,
}
go func() {
+23 -12
View File
@@ -106,16 +106,19 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
}
newEndpoints := make([]discoveryv1.Endpoint, 0)
for _, pod := range podList.Items {
ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, lg)
ready, err := er.podIsReadyToRouteTraffic(ctx, pod, &cfg, tailnetSvc, eps.AddressType, lg)
if err != nil {
return res, fmt.Errorf("error verifying if Pod is ready to route traffic: %w", err)
}
if !ready {
continue // maybe next time
}
podIP, err := podIPv4(&pod) // we currently only support IPv4
podIP, err := podIPForFamily(&pod, eps.AddressType)
if err != nil {
return res, fmt.Errorf("error determining IPv4 address for Pod: %w", err)
return res, fmt.Errorf("error determining Pod IP for %s EndpointSlice: %w", eps.AddressType, err)
}
if podIP == "" {
continue // Pod doesn't have an IP for this address family
}
newEndpoints = append(newEndpoints, discoveryv1.Endpoint{
Hostname: (*string)(&pod.UID),
@@ -140,13 +143,16 @@ func (er *egressEpsReconciler) Reconcile(ctx context.Context, req reconcile.Requ
return res, nil
}
func podIPv4(pod *corev1.Pod) (string, error) {
func podIPForFamily(pod *corev1.Pod, addrType discoveryv1.AddressType) (string, error) {
for _, ip := range pod.Status.PodIPs {
parsed, err := netip.ParseAddr(ip.IP)
if err != nil {
return "", fmt.Errorf("error parsing IP address %s: %w", ip, err)
}
if parsed.Is4() {
switch {
case addrType == discoveryv1.AddressTypeIPv4 && parsed.Is4():
return parsed.String(), nil
case addrType == discoveryv1.AddressTypeIPv6 && parsed.Is6():
return parsed.String(), nil
}
}
@@ -156,20 +162,19 @@ func podIPv4(pod *corev1.Pod) (string, error) {
// podIsReadyToRouteTraffic returns true if it appears that the proxy Pod has configured firewall rules to be able to
// route traffic to the given tailnet service. It retrieves the proxy's state Secret and compares the tailnet service
// status written there to the desired service configuration.
func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, lg *zap.SugaredLogger) (bool, error) {
func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod corev1.Pod, cfg *egressservices.Config, tailnetSvcName string, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) (bool, error) {
lg = lg.With("proxy_pod", pod.Name)
lg.Debug("checking whether proxy is ready to route to egress service")
if !pod.DeletionTimestamp.IsZero() {
lg.Debug("proxy Pod is being deleted, ignore")
return false, nil
}
podIP, err := podIPv4(&pod)
podIP, err := podIPForFamily(&pod, addrType)
switch {
case err != nil:
return false, fmt.Errorf("error determining Pod IP address: %v", err)
case podIP == "":
lg.Warn("Pod does not have an IPv4 address, and IPv6 is not currently supported")
lg.Debugf("Pod does not have an address for family %s", addrType)
return false, nil
}
@@ -199,9 +204,15 @@ func (er *egressEpsReconciler) podIsReadyToRouteTraffic(ctx context.Context, pod
if err = json.Unmarshal(svcStatusBS, svcStatus); err != nil {
return false, fmt.Errorf("error unmarshalling egress service status: %w", err)
}
if !strings.EqualFold(podIP, svcStatus.PodIPv4) {
lg.Infof("proxy's egress service status is for Pod IP %q, current proxy's Pod IP %q, waiting for the proxy to reconfigure...", svcStatus.PodIPv4, podIP)
var statusIP string
switch addrType {
case discoveryv1.AddressTypeIPv4:
statusIP = svcStatus.PodIPv4
case discoveryv1.AddressTypeIPv6:
statusIP = svcStatus.PodIPv6
}
if !strings.EqualFold(podIP, statusIP) {
lg.Infof("proxy's egress service status is for Pod IP %q, current proxy's Pod IP %q, waiting for the proxy to reconfigure...", statusIP, podIP)
return false, nil
}
+117 -5
View File
@@ -98,7 +98,7 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
t.Run("pods_are_ready_to_route_traffic", func(t *testing.T) {
pod, stateS := podAndSecretForProxyGroup("foo")
stBs := serviceStatusForPodIP(t, svc, pod.Status.PodIPs[0].IP, port)
stBs := serviceStatusForPodIPs(t, svc, pod.Status.PodIPs[0].IP, "", port)
mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
@@ -115,8 +115,8 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
expectEqual(t, fc, eps)
})
t.Run("status_does_not_match_pod_ip", func(t *testing.T) {
_, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1
stBs := serviceStatusForPodIP(t, svc, "10.0.0.2", port) // status is for a Pod with IP 10.0.0.2
_, stateS := podAndSecretForProxyGroup("foo") // replica Pod has IP 10.0.0.1
stBs := serviceStatusForPodIPs(t, svc, "10.0.0.2", "", port) // status is for a Pod with IP 10.0.0.2
mustUpdate(t, fc, "operator-ns", stateS.Name, func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
@@ -124,6 +124,117 @@ func TestTailscaleEgressEndpointSlices(t *testing.T) {
eps.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, eps)
})
// Dual-stack.
epsV6 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "foo-ipv6",
Namespace: "operator-ns",
Labels: map[string]string{
LabelParentName: "test",
LabelParentNamespace: "default",
labelSvcType: typeEgress,
labelProxyGroup: "foo",
},
},
AddressType: discoveryv1.AddressTypeIPv6,
}
mustCreate(t, fc, epsV6)
t.Run("dual_stack_pod_ready_to_route", func(t *testing.T) {
mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
dualPod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo-0",
Namespace: "operator-ns",
Labels: pgLabels("foo", nil),
UID: "foo",
},
Status: corev1.PodStatus{
PodIPs: []corev1.PodIP{{IP: "10.0.0.1"}, {IP: "fd00::1"}},
},
}
mustCreate(t, fc, dualPod)
stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "fd00::1", port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo")
eps.Endpoints = []discoveryv1.Endpoint{{
Addresses: []string{"10.0.0.1"},
Hostname: new("foo"),
Conditions: discoveryv1.EndpointConditions{
Serving: new(true),
Ready: new(true),
Terminating: new(false),
},
}}
expectEqual(t, fc, eps)
expectReconciled(t, er, "operator-ns", "foo-ipv6")
epsV6.Endpoints = []discoveryv1.Endpoint{{
Addresses: []string{"fd00::1"},
Hostname: new("foo"),
Conditions: discoveryv1.EndpointConditions{
Serving: new(true),
Ready: new(true),
Terminating: new(false),
},
}}
expectEqual(t, fc, epsV6)
})
// IPv6-only.
t.Run("ipv4_only_pod_skipped_for_ipv6_slice", func(t *testing.T) {
mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
ipv4Pod, _ := podAndSecretForProxyGroup("foo")
mustCreate(t, fc, ipv4Pod)
stBs := serviceStatusForPodIPs(t, svc, "10.0.0.1", "", port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo-ipv6")
// IPv4-only pod should not appear in the IPv6 EndpointSlice.
epsV6.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, epsV6)
})
ipv6Pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "foo-0",
Namespace: "operator-ns",
Labels: pgLabels("foo", nil),
UID: "foo",
},
Status: corev1.PodStatus{
PodIPs: []corev1.PodIP{{IP: "fd00::1"}},
},
}
t.Run("ipv6_status_does_not_match_pod_ip", func(t *testing.T) {
mustDeleteAll(t, fc, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo-0", Namespace: "operator-ns"}})
mustCreate(t, fc, ipv6Pod)
stBs := serviceStatusForPodIPs(t, svc, "", "fd00::99", port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo-ipv6")
epsV6.Endpoints = []discoveryv1.Endpoint{}
expectEqual(t, fc, epsV6)
})
t.Run("ipv6_pod_ready_to_route", func(t *testing.T) {
stBs := serviceStatusForPodIPs(t, svc, "", ipv6Pod.Status.PodIPs[0].IP, port)
mustUpdate(t, fc, "operator-ns", "foo-0", func(s *corev1.Secret) {
mak.Set(&s.Data, egressservices.KeyEgressServices, stBs)
})
expectReconciled(t, er, "operator-ns", "foo-ipv6")
epsV6.Endpoints = append(epsV6.Endpoints, discoveryv1.Endpoint{
Addresses: []string{"fd00::1"},
Hostname: new("foo"),
Conditions: discoveryv1.EndpointConditions{
Serving: new(true),
Ready: new(true),
Terminating: new(false),
},
})
expectEqual(t, fc, epsV6)
})
}
func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.ConfigMap {
@@ -157,7 +268,7 @@ func configMapForSvc(t *testing.T, svc *corev1.Service, p uint16) *corev1.Config
return cm
}
func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint16) []byte {
func serviceStatusForPodIPs(t *testing.T, svc *corev1.Service, ipv4, ipv6 string, p uint16) []byte {
t.Helper()
ports := make(map[egressservices.PortMap]struct{})
for _, port := range svc.Spec.Ports {
@@ -172,7 +283,8 @@ func serviceStatusForPodIP(t *testing.T, svc *corev1.Service, ip string, p uint1
}
svcName := tailnetSvcName(svc)
st := egressservices.Status{
PodIPv4: ip,
PodIPv4: ipv4,
PodIPv6: ipv6,
Services: map[string]*egressservices.ServiceStatus{svcName: &svcSt},
}
bs, err := json.Marshal(st)
+18 -6
View File
@@ -10,6 +10,7 @@ import (
"errors"
"fmt"
"net/http"
"net/netip"
"slices"
"strings"
"sync/atomic"
@@ -227,13 +228,24 @@ func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *c
lg.Debugf("Pod does not have health check enabled, unable to verify if it is currently routable via Service")
return cannotVerify, nil
}
wantsIP, err := podIPv4(pod)
if err != nil {
return -1, fmt.Errorf("error determining Pod's IP address: %w", err)
}
if wantsIP == "" {
// Use the Pod's primary IP (PodIPs[0]) to identify this Pod in the health check
// response. The primary IP family is determined by the cluster's IP family configuration.
// Note: we do not control which IP family the request uses, so on a dual-stack
// cluster either IPv4 or IPv6 could be used. In either case, a matching IP header
// comfirms the request reached this Pod.
if len(pod.Status.PodIPs) == 0 || pod.Status.PodIPs[0].IP == "" {
return podNotReady, nil
}
wantsIP := pod.Status.PodIPs[0].IP
parsed, err := netip.ParseAddr(wantsIP)
if err != nil {
return -1, fmt.Errorf("error parsing Pod IP %q: %w", wantsIP, err)
}
header := kubetypes.PodIPv4Header
if parsed.Is6() {
header = kubetypes.PodIPv6Header
}
ctx, cancel := context.WithTimeout(ctx, time.Second*3)
defer cancel()
@@ -250,7 +262,7 @@ func (er *egressPodsReconciler) lookupPodRouteViaSvc(ctx context.Context, pod *c
return unreachable, nil
}
defer resp.Body.Close()
gotIP := resp.Header.Get(kubetypes.PodIPv4Header)
gotIP := resp.Header.Get(header)
if gotIP == "" {
lg.Debugf("Health check does not return Pod's IP header, unable to verify if Pod is currently routable via Service")
return cannotVerify, nil
+51 -1
View File
@@ -420,6 +420,44 @@ func TestEgressPodReadiness(t *testing.T) {
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc, svc2, svc3)
})
t.Run("ipv6_only_pod_already_routed_to", func(t *testing.T) {
pod := podTemplate.DeepCopy()
pod.Status.PodIPs = []corev1.PodIP{{IP: "fd00::2"}}
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
resp := readyRespsV6("fd00::2", 1)
httpCl := fakeHTTPClient{
t: t,
state: map[string][]fakeResponse{hep: resp},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
})
t.Run("dual_stack_pod", func(t *testing.T) {
pod := podTemplate.DeepCopy()
pod.Status.PodIPs = []corev1.PodIP{{IP: "10.0.0.2"}, {IP: "fd00::2"}}
svc, hep := newSvc("svc", 9002)
mustCreateAll(t, fc, svc, pod)
// Dual-stack pod: the reconciler uses PodIPs[0] (the primary IP),
// which in this case is IPv4.
resp := readyResps("10.0.0.2", 1)
httpCl := fakeHTTPClient{
t: t,
state: map[string][]fakeResponse{hep: resp},
}
rec.httpClient = &httpCl
expectReconciled(t, rec, "operator-ns", pod.Name)
podSetReady(pod, cl)
expectEqual(t, fc, pod)
mustDeleteAll(t, fc, pod, svc)
})
}
func readyResps(ip string, num int) (resps []fakeResponse) {
@@ -429,6 +467,13 @@ func readyResps(ip string, num int) (resps []fakeResponse) {
return resps
}
func readyRespsV6(ip string, num int) (resps []fakeResponse) {
for range num {
resps = append(resps, fakeResponse{statusCode: 200, podIP: ip, header: kubetypes.PodIPv6Header})
}
return resps
}
func unreadyResps(ip string, num int) (resps []fakeResponse) {
for range num {
resps = append(resps, fakeResponse{statusCode: 503, podIP: ip})
@@ -513,7 +558,11 @@ func (f *fakeHTTPClient) Do(req *http.Request) (*http.Response, error) {
Header: make(http.Header),
Body: io.NopCloser(bytes.NewReader([]byte{})),
}
r.Header.Add(kubetypes.PodIPv4Header, resp.podIP)
h := kubetypes.PodIPv4Header
if resp.header != "" {
h = resp.header
}
r.Header.Add(h, resp.podIP)
return &r, nil
}
@@ -521,4 +570,5 @@ type fakeResponse struct {
err error
statusCode int
podIP string // for the Pod IP header
header string // header key to use; defaults to PodIPv4Header
}
+62 -20
View File
@@ -9,6 +9,7 @@ import (
"context"
"errors"
"fmt"
"slices"
"strings"
"go.uber.org/zap"
@@ -24,6 +25,7 @@ import (
tsoperator "tailscale.com/k8s-operator"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/tstime"
"tailscale.com/util/set"
)
const (
@@ -72,19 +74,57 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
}()
crl := egressSvcChildResourceLabels(svc)
eps, err := getSingleObject[discoveryv1.EndpointSlice](ctx, esrr.Client, esrr.tsNamespace, crl)
if err != nil {
err = fmt.Errorf("error getting EndpointSlice: %w", err)
epsList := &discoveryv1.EndpointSliceList{}
if err = esrr.List(ctx, epsList, client.InNamespace(esrr.tsNamespace), client.MatchingLabels(crl)); err != nil {
err = fmt.Errorf("error listing EndpointSlices: %w", err)
reason = reasonReadinessCheckFailed
msg = err.Error()
return res, err
}
if eps == nil {
lg.Infof("EndpointSlice for Service does not yet exist, waiting...")
if len(epsList.Items) == 0 {
lg.Infof("EndpointSlices for Service do not yet exist, waiting...")
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
st = metav1.ConditionFalse
return res, nil
}
// If an EndpointSlice for an expected family is missing, we mark the Service as NotReady.
//
// Setting the NotReady condition here is also used for best-effort recovery. The
// egress-svcs-reconciler does not watch EndpointSlices, so a deleted EndpointSlice is only
// recreated when this status change re-triggers a Service reconcile.
//
// TODO(beckypauley): refactor so EndpointSlice recovery is not dependent on Service status.
clusterIPSvc, err := getSingleObject[corev1.Service](ctx, esrr.Client, esrr.tsNamespace, crl)
if err != nil {
err = fmt.Errorf("error retrieving ClusterIP Service: %w", err)
reason = reasonReadinessCheckFailed
msg = err.Error()
return res, err
}
if clusterIPSvc == nil {
lg.Infof("ClusterIP Service for egress Service does not yet exist, waiting...")
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
st = metav1.ConditionFalse
return res, nil
}
gotAddrTypes := make(set.Set[discoveryv1.AddressType], len(epsList.Items))
for _, eps := range epsList.Items {
gotAddrTypes.Add(eps.AddressType)
}
wantAddrTypes, err := addrTypesForClusterIPSvc(clusterIPSvc)
if err != nil {
reason = reasonReadinessCheckFailed
msg = err.Error()
return res, err
}
for _, wantAddrType := range wantAddrTypes {
if !gotAddrTypes.Contains(wantAddrType) {
lg.Infof("EndpointSlice for %s is missing, waiting...", wantAddrType)
reason, msg = reasonClusterResourcesNotReady, reasonClusterResourcesNotReady
st = metav1.ConditionFalse
return res, nil
}
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: svc.Annotations[AnnotationProxyGroup],
@@ -119,6 +159,7 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
}
podLabels := pgLabels(pg.Name, nil)
var readyReplicas int32
nextReplica:
for i := range replicas {
podLabels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", i)
pod, err := getSingleObject[corev1.Pod](ctx, esrr.Client, esrr.tsNamespace, podLabels)
@@ -136,18 +177,16 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
}
lg.Debugf("looking at Pod with IPs %v", pod.Status.PodIPs)
ready := false
for _, ep := range eps.Endpoints {
lg.Debugf("looking at endpoint with addresses %v", ep.Addresses)
if endpointReadyForPod(&ep, pod, lg) {
lg.Debugf("endpoint is ready for Pod")
ready = true
break
for _, eps := range epsList.Items {
lg.Debugf("looking at %s EndpointSlice %s", eps.AddressType, eps.Name)
if !slices.ContainsFunc(eps.Endpoints, func(ep discoveryv1.Endpoint) bool {
return endpointReadyForPod(&ep, pod, eps.AddressType, lg)
}) {
continue nextReplica
}
}
if ready {
readyReplicas++
}
lg.Debugf("endpoint is ready for Pod")
readyReplicas++
}
msg = fmt.Sprintf(msgReadyToRouteTemplate, readyReplicas, replicas)
if readyReplicas == 0 {
@@ -164,12 +203,15 @@ func (esrr *egressSvcsReadinessReconciler) Reconcile(ctx context.Context, req re
return res, nil
}
// endpointReadyForPod returns true if the endpoint is for the Pod's IPv4 address and is ready to serve traffic.
// Endpoint must not be nil.
func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, lg *zap.SugaredLogger) bool {
podIP, err := podIPv4(pod)
// endpointReadyForPod returns true if the endpoint is for the Pod's address (for the given address family)
// and is ready to serve traffic. Endpoint must not be nil.
func endpointReadyForPod(ep *discoveryv1.Endpoint, pod *corev1.Pod, addrType discoveryv1.AddressType, lg *zap.SugaredLogger) bool {
podIP, err := podIPForFamily(pod, addrType)
if err != nil {
lg.Warnf("error retrieving Pod's IPv4 address: %v", err)
lg.Warnf("error retrieving Pod's %s address: %v", addrType, err)
return false
}
if podIP == "" {
return false
}
@@ -47,7 +47,14 @@ func TestEgressServiceReadiness(t *testing.T) {
},
},
}
fakeClusterIPSvc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "my-app", Namespace: "operator-ns"}}
fakeClusterIPSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "operator-ns",
Labels: egressSvcChildResourceLabels(egressSvc),
},
Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1"}},
}
labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
@@ -63,6 +70,7 @@ func TestEgressServiceReadiness(t *testing.T) {
},
}
mustCreate(t, fc, egressSvc)
mustCreate(t, fc, fakeClusterIPSvc)
setClusterNotReady(egressSvc, cl, zl.Sugar())
t.Run("endpointslice_does_not_exist", func(t *testing.T) {
expectReconciled(t, rec, "dev", "my-app")
@@ -117,6 +125,212 @@ func TestEgressServiceReadiness(t *testing.T) {
})
}
func TestEgressServiceReadinessDualStack(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
zl, _ := zap.NewDevelopment()
cl := tstest.NewClock(tstest.ClockOpts{})
rec := &egressSvcsReadinessReconciler{
tsNamespace: "operator-ns",
Client: fc,
logger: zl.Sugar(),
clock: cl,
}
tailnetFQDN := "my-app.tailnetxyz.ts.net"
egressSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "dev",
Annotations: map[string]string{
AnnotationProxyGroup: "dev",
AnnotationTailnetTargetFQDN: tailnetFQDN,
},
},
}
fakeClusterIPSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "operator-ns",
Labels: egressSvcChildResourceLabels(egressSvc),
},
Spec: corev1.ServiceSpec{ClusterIPs: []string{"10.0.0.1", "fd00::1"}},
}
labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
epsV4 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app-ipv4",
Namespace: "operator-ns",
Labels: labels,
},
AddressType: discoveryv1.AddressTypeIPv4,
}
labelsV6 := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
epsV6 := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app-ipv6",
Namespace: "operator-ns",
Labels: labelsV6,
},
AddressType: discoveryv1.AddressTypeIPv6,
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "dev",
},
Spec: tsapi.ProxyGroupSpec{
Replicas: new(int32(1)),
Type: tsapi.ProxyGroupTypeEgress,
},
}
mustCreate(t, fc, egressSvc)
mustCreate(t, fc, fakeClusterIPSvc)
mustCreate(t, fc, epsV4)
mustCreate(t, fc, epsV6)
mustCreate(t, fc, pg)
setPGReady(pg, cl, zl.Sugar())
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
p.Status = pg.Status
})
// Create a dual-stack pod.
p := pod(pg, 0)
p.Status.PodIPs = append(p.Status.PodIPs, corev1.PodIP{IP: "fd00::0"})
mustCreate(t, fc, p)
mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
existing.Status.PodIPs = p.Status.PodIPs
})
t.Run("not_ready_missing_from_ipv6_slice", func(t *testing.T) {
setEndpointForReplicaWithIP("10.0.0.0", epsV4)
mustUpdate(t, fc, epsV4.Namespace, epsV4.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = epsV4.Endpoints
})
expectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
expectEqual(t, fc, egressSvc)
})
t.Run("ready_in_both_slices", func(t *testing.T) {
setEndpointForReplicaWithIP("fd00::", epsV6)
mustUpdate(t, fc, epsV6.Namespace, epsV6.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = epsV6.Endpoints
})
expectReconciled(t, rec, "dev", "my-app")
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
expectEqual(t, fc, egressSvc)
})
t.Run("not_ready_when_ipv6_slice_missing", func(t *testing.T) {
// Delete the IPv6 EndpointSlice while the ClusterIP Service still
// wants an IPv6 family; the Service should report NotReady even though
// the IPv4 EndpointSlice is healthy.
if err := fc.Delete(t.Context(), epsV6); err != nil {
t.Fatalf("error deleting IPv6 EndpointSlice: %v", err)
}
expectReconciled(t, rec, "dev", "my-app")
setClusterNotReady(egressSvc, cl, zl.Sugar())
expectEqual(t, fc, egressSvc)
})
}
func TestEgressServiceReadinessIPv6Only(t *testing.T) {
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithStatusSubresource(&tsapi.ProxyGroup{}).
Build()
zl, _ := zap.NewDevelopment()
cl := tstest.NewClock(tstest.ClockOpts{})
rec := &egressSvcsReadinessReconciler{
tsNamespace: "operator-ns",
Client: fc,
logger: zl.Sugar(),
clock: cl,
}
egressSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "dev",
Annotations: map[string]string{
AnnotationProxyGroup: "dev",
AnnotationTailnetTargetFQDN: "my-app.tailnetxyz.ts.net",
},
},
}
fakeClusterIPSvc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app",
Namespace: "operator-ns",
Labels: egressSvcChildResourceLabels(egressSvc),
},
Spec: corev1.ServiceSpec{ClusterIPs: []string{"fd00::1"}},
}
labels := egressSvcEpsLabels(egressSvc, fakeClusterIPSvc)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: "my-app-ipv6",
Namespace: "operator-ns",
Labels: labels,
},
AddressType: discoveryv1.AddressTypeIPv6,
}
pg := &tsapi.ProxyGroup{
ObjectMeta: metav1.ObjectMeta{
Name: "dev",
},
}
mustCreate(t, fc, egressSvc)
mustCreate(t, fc, fakeClusterIPSvc)
mustCreate(t, fc, eps)
mustCreate(t, fc, pg)
setPGReady(pg, cl, zl.Sugar())
mustUpdateStatus(t, fc, pg.Namespace, pg.Name, func(p *tsapi.ProxyGroup) {
p.Status = pg.Status
})
// Create IPv6-only pods.
for i := range pgReplicas(pg) {
p := ipv6OnlyPod(pg, i)
mustCreate(t, fc, p)
mustUpdateStatus(t, fc, p.Namespace, p.Name, func(existing *corev1.Pod) {
existing.Status.PodIPs = p.Status.PodIPs
})
}
t.Run("no_ready_replicas", func(t *testing.T) {
expectReconciled(t, rec, "dev", "my-app")
setNotReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg))
expectEqual(t, fc, egressSvc)
})
t.Run("all_replicas_ready", func(t *testing.T) {
for i := range pgReplicas(pg) {
p := ipv6OnlyPod(pg, i)
setEndpointForReplicaWithIP(p.Status.PodIPs[0].IP, eps)
}
mustUpdate(t, fc, eps.Namespace, eps.Name, func(e *discoveryv1.EndpointSlice) {
e.Endpoints = eps.Endpoints
})
setReady(egressSvc, cl, zl.Sugar(), pgReplicas(pg), pgReplicas(pg))
expectReconciled(t, rec, "dev", "my-app")
expectEqual(t, fc, egressSvc)
})
}
func ipv6OnlyPod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
labels := pgLabels(pg.Name, nil)
labels[appsv1.PodIndexLabel] = fmt.Sprintf("%d", ordinal)
ip := fmt.Sprintf("fd00::%d", ordinal+1) // +1 to avoid fd00::0 normalization issues
return &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%d", pg.Name, ordinal),
Namespace: "operator-ns",
Labels: labels,
},
Status: corev1.PodStatus{
PodIPs: []corev1.PodIP{{IP: ip}},
},
}
}
func setClusterNotReady(svc *corev1.Service, cl tstime.Clock, lg *zap.SugaredLogger) {
tsoperator.SetServiceCondition(svc, tsapi.EgressSvcReady, metav1.ConditionFalse, reasonClusterResourcesNotReady, reasonClusterResourcesNotReady, cl, lg)
}
@@ -166,3 +380,14 @@ func pod(pg *tsapi.ProxyGroup, ordinal int32) *corev1.Pod {
},
}
}
func setEndpointForReplicaWithIP(ip string, eps *discoveryv1.EndpointSlice) {
eps.Endpoints = append(eps.Endpoints, discoveryv1.Endpoint{
Addresses: []string{ip},
Conditions: discoveryv1.EndpointConditions{
Ready: new(true),
Serving: new(true),
Terminating: new(false),
},
})
}
+47 -18
View File
@@ -12,6 +12,7 @@ import (
"errors"
"fmt"
"math/rand/v2"
"net/netip"
"reflect"
"slices"
"strings"
@@ -222,28 +223,56 @@ func (esr *egressSvcsReconciler) maybeProvision(ctx context.Context, svc *corev1
return nil
}
// ensureEndpointSlices ensures that an IPv4 EndpointSlice exists for the egress
// service and that its ports are up to date.
// addrTypesForClusterIPSvc returns the EndpointSlice address types (IP families)
// that the given ClusterIP Service supports, derived from its ClusterIPs.
// TODO(beckypauley): this could read Spec.IPFamilies directly instead of parsing
// ClusterIPs to determine the family.
func addrTypesForClusterIPSvc(clusterIPSvc *corev1.Service) ([]discoveryv1.AddressType, error) {
addrTypes := make([]discoveryv1.AddressType, 0, len(clusterIPSvc.Spec.ClusterIPs))
for _, clusterIP := range clusterIPSvc.Spec.ClusterIPs {
ip, err := netip.ParseAddr(clusterIP)
if err != nil {
return nil, fmt.Errorf("error parsing ClusterIP %q: %w", clusterIP, err)
}
addrType := discoveryv1.AddressTypeIPv4
if ip.Is6() {
addrType = discoveryv1.AddressTypeIPv6
}
addrTypes = append(addrTypes, addrType)
}
return addrTypes, nil
}
// ensureEndpointSlices ensures that EndpointSlices exist for the egress service
// for each IP family supported by the cluster, and that their ports are up to
// date.
func (esr *egressSvcsReconciler) ensureEndpointSlices(ctx context.Context, svc, clusterIPSvc *corev1.Service, lg *zap.SugaredLogger) error {
crl := egressSvcEpsLabels(svc, clusterIPSvc)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-ipv4", clusterIPSvc.Name),
Namespace: esr.tsNamespace,
Labels: crl,
},
AddressType: discoveryv1.AddressTypeIPv4,
Ports: epsPortsFromSvc(clusterIPSvc),
// Only create EndpointSlices for IP families supported by the cluster.
addrTypes, err := addrTypesForClusterIPSvc(clusterIPSvc)
if err != nil {
return err
}
if _, err := createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
e.Labels = eps.Labels
e.AddressType = eps.AddressType
e.Ports = eps.Ports
for _, p := range e.Endpoints {
p.Conditions.Ready = nil
for _, addrType := range addrTypes {
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-%s", clusterIPSvc.Name, strings.ToLower(string(addrType))),
Namespace: esr.tsNamespace,
Labels: crl,
},
AddressType: addrType,
Ports: epsPortsFromSvc(clusterIPSvc),
}
if _, err := createOrUpdate(ctx, esr.Client, esr.tsNamespace, eps, func(e *discoveryv1.EndpointSlice) {
e.Labels = eps.Labels
e.AddressType = eps.AddressType
e.Ports = eps.Ports
for _, p := range e.Endpoints {
p.Conditions.Ready = nil
}
}); err != nil {
return fmt.Errorf("error ensuring %s EndpointSlice: %w", addrType, err)
}
}); err != nil {
return fmt.Errorf("error ensuring EndpointSlice: %w", err)
}
return nil
}
+155 -5
View File
@@ -21,6 +21,7 @@ import (
"k8s.io/apimachinery/pkg/util/intstr"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
tsapi "tailscale.com/k8s-operator/apis/v1alpha1"
"tailscale.com/kube/egressservices"
@@ -50,6 +51,9 @@ func TestTailscaleEgressServices(t *testing.T) {
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, cm).
WithStatusSubresource(pg).
WithInterceptorFuncs(interceptor.Funcs{
Create: clusterIPInterceptor("10.96.0.1"),
}).
Build()
zl, err := zap.NewDevelopment()
if err != nil {
@@ -152,10 +156,10 @@ func validateReadyService(t *testing.T, fc client.WithWatch, esr *egressSvcsReco
expectReconciled(t, esr, "default", "test")
// Verify that a ClusterIP Service has been created.
name := findGenNameForEgressSvcResources(t, fc, svc)
expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc)
expectEqual(t, fc, clusterIPSvc(name, svc), removeTargetPortsFromSvc, removeClusterIPsFromSvc)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
// Verify that an EndpointSlice has been created.
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc))
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4))
// Verify that ConfigMap contains configuration for the new egress service.
mustHaveConfigForSvc(t, fc, svc, clusterSvc, cm, zl)
r := svcConfiguredReason(svc, true, zl.Sugar())
@@ -241,18 +245,22 @@ func mustGetClusterIPSvc(t *testing.T, cl client.Client, name string) *corev1.Se
return svc
}
func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service) *discoveryv1.EndpointSlice {
func endpointSlice(name string, extNSvc, clusterIPSvc *corev1.Service, addrType discoveryv1.AddressType) *discoveryv1.EndpointSlice {
labels := egressSvcChildResourceLabels(extNSvc)
labels[discoveryv1.LabelManagedBy] = "tailscale.com"
labels[discoveryv1.LabelServiceName] = name
suffix := "ipv4"
if addrType == discoveryv1.AddressTypeIPv6 {
suffix = "ipv6"
}
return &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: fmt.Sprintf("%s-ipv4", name),
Name: fmt.Sprintf("%s-%s", name, suffix),
Namespace: "operator-ns",
Labels: labels,
},
Ports: portsForEndpointSlice(clusterIPSvc),
AddressType: discoveryv1.AddressTypeIPv4,
AddressType: addrType,
}
}
@@ -312,3 +320,145 @@ func configFromCM(t *testing.T, cm *corev1.ConfigMap, svcName string) *egressser
}
return nil
}
func TestTailscaleEgressServicesDualStack(t *testing.T) {
pg := &tsapi.ProxyGroup{
TypeMeta: metav1.TypeMeta{Kind: "ProxyGroup", APIVersion: "tailscale.com/v1alpha1"},
ObjectMeta: metav1.ObjectMeta{
Name: "foo",
UID: types.UID("1234-UID"),
},
Spec: tsapi.ProxyGroupSpec{
Replicas: pointer.To[int32](3),
Type: tsapi.ProxyGroupTypeEgress,
},
}
cm := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: pgEgressCMName("foo"),
Namespace: "operator-ns",
},
}
fc := fake.NewClientBuilder().
WithScheme(tsapi.GlobalScheme).
WithObjects(pg, cm).
WithStatusSubresource(pg).
WithInterceptorFuncs(interceptor.Funcs{
Create: clusterIPInterceptor("10.96.0.1", "fd00::1"),
}).
Build()
zl, err := zap.NewDevelopment()
if err != nil {
t.Fatal(err)
}
clock := tstest.NewClock(tstest.ClockOpts{})
esr := &egressSvcsReconciler{
Client: fc,
logger: zl.Sugar(),
clock: clock,
tsNamespace: "operator-ns",
}
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
UID: types.UID("1234-UID"),
Annotations: map[string]string{
AnnotationTailnetTargetFQDN: "foo.bar.ts.net.",
AnnotationProxyGroup: "foo",
},
},
Spec: corev1.ServiceSpec{
ExternalName: "placeholder",
Type: corev1.ServiceTypeExternalName,
Selector: nil,
Ports: []corev1.ServicePort{
{
Protocol: "TCP",
Port: 80,
},
},
},
}
t.Run("dual_stack_creates_both_endpoint_slices", func(t *testing.T) {
mustCreate(t, fc, svc)
expectReconciled(t, esr, "default", "test")
validateReadyService(t, fc, esr, svc, clock, zl, cm)
// Also verify the IPv6 EndpointSlice was created.
name := findGenNameForEgressSvcResources(t, fc, svc)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
})
t.Run("dual_stack_endpointslice_deletion_recovery", func(t *testing.T) {
name := findGenNameForEgressSvcResources(t, fc, svc)
// Delete both IPv4 and IPv6 EndpointSlices.
for _, suffix := range []string{"ipv4", "ipv6"} {
epsName := fmt.Sprintf("%s-%s", name, suffix)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: epsName,
Namespace: "operator-ns",
},
}
if err := fc.Delete(t.Context(), eps); err != nil {
t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err)
}
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
}
// Reconcile should recreate both.
validateReadyService(t, fc, esr, svc, clock, zl, cm)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
})
t.Run("dual_stack_single_endpointslice_deletion_recovery", func(t *testing.T) {
name := findGenNameForEgressSvcResources(t, fc, svc)
// Delete only the IPv6 EndpointSlice.
epsName := fmt.Sprintf("%s-ipv6", name)
eps := &discoveryv1.EndpointSlice{
ObjectMeta: metav1.ObjectMeta{
Name: epsName,
Namespace: "operator-ns",
},
}
if err := fc.Delete(t.Context(), eps); err != nil {
t.Fatalf("error deleting EndpointSlice %s: %v", epsName, err)
}
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", epsName)
// Reconcile should recreate the missing IPv6 EndpointSlice while leaving
// the IPv4 one untouched.
validateReadyService(t, fc, esr, svc, clock, zl, cm)
clusterSvc := mustGetClusterIPSvc(t, fc, name)
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv6))
expectEqual(t, fc, endpointSlice(name, svc, clusterSvc, discoveryv1.AddressTypeIPv4))
})
t.Run("delete_dual_stack_service", func(t *testing.T) {
name := findGenNameForEgressSvcResources(t, fc, svc)
if err := fc.Delete(context.Background(), svc); err != nil {
t.Fatalf("error deleting ExternalName Service: %v", err)
}
expectReconciled(t, esr, "default", "test")
expectMissing[corev1.Service](t, fc, "operator-ns", name)
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv4", name))
expectMissing[discoveryv1.EndpointSlice](t, fc, "operator-ns", fmt.Sprintf("%s-ipv6", name))
mustNotHaveConfigForSvc(t, fc, svc, cm)
})
}
// clusterIPInterceptor returns an interceptor.Funcs Create function that
// simulates the API server assigning ClusterIPs to ClusterIP Services.
// This is required because the reconciler iterates ClusterIPs to create
// per-family EndpointSlices but the fake client does not assign ClusterIPs.
func clusterIPInterceptor(clusterIPs ...string) func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
return func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error {
if svc, ok := obj.(*corev1.Service); ok && svc.Spec.Type == corev1.ServiceTypeClusterIP {
svc.Spec.ClusterIPs = clusterIPs
svc.Spec.ClusterIP = clusterIPs[0]
}
return c.Create(ctx, obj, opts...)
}
}
+5
View File
@@ -984,6 +984,11 @@ func removeTargetPortsFromSvc(svc *corev1.Service) {
svc.Spec.Ports = newPorts
}
func removeClusterIPsFromSvc(svc *corev1.Service) {
svc.Spec.ClusterIP = ""
svc.Spec.ClusterIPs = nil
}
func removeAuthKeyIfExistsModifier(t *testing.T) func(s *corev1.Secret) {
return func(secret *corev1.Secret) {
t.Helper()
+9 -2
View File
@@ -301,8 +301,15 @@ func run(logger *zap.SugaredLogger) error {
}
if cfg.Parsed.HealthCheckEnabled.EqualBool(true) {
ipV4, _ := ts.TailscaleIPs()
hz := healthz.RegisterHealthHandlers(mux, ipV4.String(), logger.Infof)
ipV4, ipV6 := ts.TailscaleIPs()
var v4, v6 string
if ipV4.IsValid() {
v4 = ipV4.String()
}
if ipV6.IsValid() {
v6 = ipV6.String()
}
hz := healthz.RegisterHealthHandlers(mux, v4, v6, logger.Infof)
group.Go(func() error {
err := hz.MonitorHealth(ctx, lc)
if err == nil || errors.Is(err, context.Canceled) {
+1
View File
@@ -94,6 +94,7 @@ func (p PortMaps) MarshalJSON() ([]byte, error) {
// services for a proxy identified by the PodIP.
type Status struct {
PodIPv4 string `json:"podIPv4"`
PodIPv6 string `json:"podIPv6,omitempty"`
// All egress service status keyed by service name.
Services map[string]*ServiceStatus `json:"services"`
}
+9 -2
View File
@@ -26,6 +26,7 @@ type Healthz struct {
sync.Mutex
hasAddrs bool
podIPv4 string
podIPv6 string
logger logger.Logf
}
@@ -34,7 +35,12 @@ func (h *Healthz) ServeHTTP(w http.ResponseWriter, r *http.Request) {
defer h.Unlock()
if h.hasAddrs {
w.Header().Add(kubetypes.PodIPv4Header, h.podIPv4)
if h.podIPv4 != "" {
w.Header().Set(kubetypes.PodIPv4Header, h.podIPv4)
}
if h.podIPv6 != "" {
w.Header().Set(kubetypes.PodIPv6Header, h.podIPv6)
}
if _, err := w.Write([]byte("ok")); err != nil {
http.Error(w, fmt.Sprintf("error writing status: %v", err), http.StatusInternalServerError)
}
@@ -74,9 +80,10 @@ func (h *Healthz) MonitorHealth(ctx context.Context, lc *local.Client) error {
// RegisterHealthHandlers registers a simple health handler at /healthz.
// A containerized tailscale instance is considered healthy if
// it has at least one tailnet IP address.
func RegisterHealthHandlers(mux *http.ServeMux, podIPv4 string, logger logger.Logf) *Healthz {
func RegisterHealthHandlers(mux *http.ServeMux, podIPv4, podIPv6 string, logger logger.Logf) *Healthz {
h := &Healthz{
podIPv4: podIPv4,
podIPv6: podIPv6,
logger: logger,
}
mux.Handle("GET /healthz", h)
+2
View File
@@ -52,6 +52,8 @@ const (
// Pod's IPv4 address header key as returned by containerboot health check endpoint.
PodIPv4Header string = "Pod-IPv4"
// Pod's IPv6 address header key as returned by containerboot health check endpoint.
PodIPv6Header string = "Pod-IPv6"
EgessServicesPreshutdownEP = "/internal-egress-services-preshutdown"