webnet/tailscale
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/{containerboot,k8s-operator},kube/kubetypes: unadvertise ingress services on shutdown (#15451)

Browse Source 
Ensure no services are advertised as part of shutting down tailscaled.
Prefs are only edited if services are currently advertised, and they're
edited we wait for control's ~15s (+ buffer) delay to failover.

Note that editing prefs will trigger a synchronous write to the state
Secret, so it may fail to persist state if the ProxyGroup is getting
scaled down and therefore has its RBAC deleted at the same time, but that
failure doesn't stop prefs being updated within the local backend,
doesn't  affect connectivity to control, and the state Secret is
about to get deleted anyway, so the only negative side effect is a harmless
error log during shutdown. Control still learns that the node is no
longer advertising the service and triggers the failover.

Note that the first version of this used a PreStop lifecycle hook, but
that only supports GET methods and we need the shutdown to trigger side
effects (updating prefs) so it didn't seem appropriate to expose that
functionality on a GET endpoint that's accessible on the k8s network.

Updates tailscale/corp#24795

Change-Id: I0a9a4fe7a5395ca76135ceead05cbc3ee32b3d3c
Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>
This commit is contained in:
Tom Proctor
2025-04-09 10:11:15 +01:00
committed by GitHub
parent 8e1aa86bdb
commit dd95a83a65
5 changed files with 75 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/containerboot/main.go
+17 -13
View File
@@ -195,18 +195,21 @@ func run() error {
return fmt.Errorf("failed to bring up tailscale: %w", err)
}
killTailscaled := func() {
// The default termination grace period for a Pod is 30s. We wait 25s at
// most so that we still reserve some of that budget for tailscaled
// to receive and react to a SIGTERM before the SIGKILL that k8s
// will send at the end of the grace period.
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
defer cancel()
if err := ensureServicesNotAdvertised(ctx, client); err != nil {
log.Printf("Error ensuring services are not advertised: %v", err)
}
if hasKubeStateStore(cfg) {
// Check we're not shutting tailscaled down while it's still writing
// state. If we authenticate and fail to write all the state, we'll
// never recover automatically.
//
// The default termination grace period for a Pod is 30s. We wait 25s at
// most so that we still reserve some of that budget for tailscaled
// to receive and react to a SIGTERM before the SIGKILL that k8s
// will send at the end of the grace period.
ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second)
defer cancel()
log.Printf("Checking for consistent state")
err := kc.waitForConsistentState(ctx)
if err != nil {
@@ -226,7 +229,7 @@ func run() error {
mux := http.NewServeMux()
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.HealthCheckAddrPort)
healthCheck = healthHandlers(mux, cfg.PodIPv4)
healthCheck = registerHealthHandlers(mux, cfg.PodIPv4)
close := runHTTPServer(mux, cfg.HealthCheckAddrPort)
defer close()
@@ -237,15 +240,16 @@ func run() error {
if cfg.localMetricsEnabled() {
log.Printf("Running metrics endpoint at %s/metrics", cfg.LocalAddrPort)
metricsHandlers(mux, client, cfg.DebugAddrPort)
registerMetricsHandlers(mux, client, cfg.DebugAddrPort)
}
if cfg.localHealthEnabled() {
log.Printf("Running healthcheck endpoint at %s/healthz", cfg.LocalAddrPort)
healthCheck = healthHandlers(mux, cfg.PodIPv4)
healthCheck = registerHealthHandlers(mux, cfg.PodIPv4)
}
if cfg.EgressProxiesCfgPath != "" {
log.Printf("Running preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP)
if cfg.egressSvcsTerminateEPEnabled() {
log.Printf("Running egress preshutdown hook at %s%s", cfg.LocalAddrPort, kubetypes.EgessServicesPreshutdownEP)
ep.registerHandlers(mux)
}