webnet/tailscale
1
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

control/controlknobs,net/{batching,tstun},wgengine: add nodecaps to disable UDP & TUN GRO/GSO

Browse Source 
Add four control-plane node attributes that let us disable UDP GSO/GRO
on the magicsock UDP socket and UDP/TCP GRO on the Tailscale TUN
device.

These complement the pre-existing TS_DEBUG_DISABLE_UDP_{GRO,GSO} and
TS_TUN_DISABLE_{UDP,TCP}_GRO envknobs. They exist so we can mitigate
upstream Linux kernel regressions on a deployed fleet without
requiring a client release, after two incidents (#13041, #19777) where
buggy kernel patches landed upstream and the fix took an excessively
long time to reach downstream distros.

Knob changes are reacted to in setNetworkMapInternal / SetNetworkMap via
a comparison against a cached "last applied" value and only an actual
transition triggers work: magicsock Rebind()+ReSTUN for UDP,
ApplyGROKnobs for TUN. The TUN side is gated by buildfeatures.HasGRO and
is one-way (wireguard-go GRO disablement is sticky); re-enabling
requires a client restart.

Updates #13041
Updates #19777

Change-Id: I802993070afa659cc06809bb0bfbb7f8a0cdb273
Signed-off-by: James Tucker <james@tailscale.com>
This commit is contained in:
James Tucker
2026-05-27 17:10:14 -07:00
committed by James Tucker
parent 94af1b00fb
commit 25b8ed8d9e
13 changed files with 212 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
wgengine/magicsock/magicsock.go
+44 -4
View File
@@ -397,6 +397,16 @@ type Conn struct {
// experiencing a write error, and is used to throttle the rate of rebinds.
lastErrRebind syncs.AtomicValue[time.Time]
// appliedDisableUDPGRO and appliedDisableUDPGSO cache the last UDP offload
// controlknobs values we reacted to. They are compared against the live
// knob values during netmap updates so we can detect a control-plane
// transition and trigger a [Conn.Rebind] to re-evaluate the
// UDP_GRO/UDP_SEGMENT socket options. Guarded by c.mu. Only consulted on
// Linux; on other platforms tryEnableUDPOffload is a no-op so any
// transition is meaningless and no rebind is fired.
appliedDisableUDPGRO bool
appliedDisableUDPGSO bool
// staticEndpoints are user set endpoints that this node should
// advertise amongst its wireguard endpoints. It is user's
// responsibility to ensure that traffic from these endpoints is routed
@@ -634,6 +644,15 @@ func NewConn(opts Options) (*Conn, error) {
c.eventBus = opts.EventBus
c.port.Store(uint32(opts.Port))
c.controlKnobs = opts.ControlKnobs
if runtime.GOOS == "linux" && c.controlKnobs != nil {
// Seed the cached "last applied" UDP offload knob values so the first
// netmap update doesn't spuriously trigger a rebind: bindSocket (called
// shortly after NewConn) will read these same knob values when
// configuring UDP_GRO/UDP_SEGMENT, so they're already in sync. We only
// do this on Linux because tryEnableUDPOffload is a no-op elsewhere.
c.appliedDisableUDPGRO = c.controlKnobs.DisableUDPGRO.Load()
c.appliedDisableUDPGSO = c.controlKnobs.DisableUDPGSO.Load()
}
c.epFunc = opts.endpointsFunc()
c.derpActiveFunc = opts.derpActiveFunc()
c.idleFunc = opts.IdleFunc
@@ -2996,6 +3015,8 @@ func (c *Conn) setNetworkMapInternal(self tailcfg.NodeView, peers []tailcfg.Node
!self.HasCap(tailcfg.NodeAttrDisableRelayClient) &&
!self.HasCap(tailcfg.NodeAttrOnlyTCP443)
udpOffloadKnobsChanged := false
var curGRO, curGSO bool
c.mu.Lock()
relayClientChanged := c.relayClientEnabled != relayClientEnabled
c.relayClientEnabled = relayClientEnabled
@@ -3004,12 +3025,31 @@ func (c *Conn) setNetworkMapInternal(self tailcfg.NodeView, peers []tailcfg.Node
peersSnap := c.peerSnapshotLocked()
isClosed := c.closed
c.usingCachedNetmap.Store(isCached)
if runtime.GOOS == "linux" && c.controlKnobs != nil {
curGRO = c.controlKnobs.DisableUDPGRO.Load()
curGSO = c.controlKnobs.DisableUDPGSO.Load()
if curGRO != c.appliedDisableUDPGRO || curGSO != c.appliedDisableUDPGSO {
c.appliedDisableUDPGRO = curGRO
c.appliedDisableUDPGSO = curGSO
udpOffloadKnobsChanged = true
}
}
c.mu.Unlock() // release c.mu before potentially calling c.updateRelayServersSet which is O(m * n)
if isClosed {
return // nothing to do here, the conn is closed and the update is no longer relevant
}
if udpOffloadKnobsChanged {
// A control-plane node attribute toggled UDP GRO or UDP GSO. Rebind
// the UDP sockets so tryEnableUDPOffload re-runs and applies the new
// values, then ReSTUN to refresh endpoints.
c.logf("magicsock: UDP offload knobs changed (DisableUDPGRO=%v DisableUDPGSO=%v); rebinding",
curGRO, curGSO)
c.Rebind()
go c.ReSTUN("udp-offload-knobs-changed")
}
if peersChanged || relayClientChanged {
if !relayClientEnabled {
// [relayManager]'s run loop updates [relayManager.hasPeerRelayServers].
@@ -3631,13 +3671,13 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur
defer ruc.mu.Unlock()
if runtime.GOOS == "js" {
ruc.setConnLocked(newBlockForeverConn(), "", c.bind.BatchSize())
ruc.setConnLocked(newBlockForeverConn(), "", c.bind.BatchSize(), c.controlKnobs)
return nil
}
if debugAlwaysDERP() {
c.logf("disabled %v per TS_DEBUG_ALWAYS_USE_DERP", network)
ruc.setConnLocked(newBlockForeverConn(), "", c.bind.BatchSize())
ruc.setConnLocked(newBlockForeverConn(), "", c.bind.BatchSize(), c.controlKnobs)
return nil
}
@@ -3696,7 +3736,7 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur
if debugBindSocket() {
c.logf("magicsock: bindSocket: successfully listened %v port %d", network, port)
}
ruc.setConnLocked(pconn, network, c.bind.BatchSize())
ruc.setConnLocked(pconn, network, c.bind.BatchSize(), c.controlKnobs)
if network == "udp4" {
c.health.SetUDP4Unbound(false)
}
@@ -3707,7 +3747,7 @@ func (c *Conn) bindSocket(ruc *RebindingUDPConn, network string, curPortFate cur
// Set pconn to a dummy conn whose reads block until closed.
// This keeps the receive funcs alive for a future in which
// we get a link change and we can try binding again.
ruc.setConnLocked(newBlockForeverConn(), "", c.bind.BatchSize())
ruc.setConnLocked(newBlockForeverConn(), "", c.bind.BatchSize(), c.controlKnobs)
if network == "udp4" {
c.health.SetUDP4Unbound(true)
}
wgengine/magicsock/magicsock_test.go
+2 -2
View File
@@ -2242,8 +2242,8 @@ func TestRebindingUDPConn(t *testing.T) {
t.Fatal(err)
}
defer realConn.Close()
c.setConnLocked(realConn.(nettype.PacketConn), "udp4", 1)
c.setConnLocked(newBlockForeverConn(), "", 1)
c.setConnLocked(realConn.(nettype.PacketConn), "udp4", 1, nil)
c.setConnLocked(newBlockForeverConn(), "", 1, nil)
}
// https://github.com/tailscale/tailscale/issues/6680: don't ignore
wgengine/magicsock/rebinding_conn.go
+4 -3
View File
@@ -12,6 +12,7 @@ import (
"syscall"
"golang.org/x/net/ipv6"
"tailscale.com/control/controlknobs"
"tailscale.com/net/batching"
"tailscale.com/net/netaddr"
"tailscale.com/net/packet"
@@ -41,9 +42,9 @@ type RebindingUDPConn struct {
// nettype.PacketConn to a batchingConn when appropriate. This upgrade is
// intentionally pushed closest to where read/write ops occur in order to avoid
// disrupting surrounding code that assumes nettype.PacketConn is a
// *net.UDPConn.
func (c *RebindingUDPConn) setConnLocked(p nettype.PacketConn, network string, batchSize int) {
upc := batching.TryUpgradeToConn(p, network, batchSize, "magicsock_udp_rxq_overflows")
// *net.UDPConn. knobs may be nil.
func (c *RebindingUDPConn) setConnLocked(p nettype.PacketConn, network string, batchSize int, knobs *controlknobs.Knobs) {
upc := batching.TryUpgradeToConn(p, network, batchSize, "magicsock_udp_rxq_overflows", knobs)
c.pconn = upc
c.pconnAtomic.Store(&upc)
c.port = uint16(c.localAddrLocked().Port)
wgengine/userspace.go
+36 -1
View File
@@ -130,6 +130,14 @@ type userspaceEngine struct {
reconfigureVPN func() error // or nil
conn25PacketHooks Conn25PacketHooks // or nil
// lastAppliedDisableTUNUDPGRO and lastAppliedDisableTUNTCPGRO cache the
// controlknobs values that were last applied to the TUN device. They are
// read and updated under e.mu and only consulted when buildfeatures.HasGRO
// is true. Note: wireguard-go's GRO disablement is one-way (sticky), so
// transitions from disabled back to enabled require a client restart.
lastAppliedDisableTUNUDPGRO bool
lastAppliedDisableTUNTCPGRO bool
mu sync.Mutex // guards following; see lock order comment below
netMap *netmap.NetworkMap // or nil
closing bool // Close was called (even if we're still closing)
@@ -564,7 +572,15 @@ func NewUserspaceEngine(logf logger.Logf, conf Config) (_ Engine, reterr error)
if err := e.router.Up(); err != nil {
return nil, fmt.Errorf("router.Up: %w", err)
}
tsTUNDev.SetLinkFeaturesPostUp()
tsTUNDev.SetLinkFeaturesPostUp(e.controlKnobs)
if buildfeatures.HasGRO && runtime.GOOS == "linux" && e.controlKnobs != nil {
// Seed the cached "last applied" TUN GRO knob values so the first
// netmap update doesn't spuriously call ApplyGROKnobs:
// SetLinkFeaturesPostUp above already applied these same values. We
// only do this on Linux because ApplyGROKnobs is a no-op elsewhere.
e.lastAppliedDisableTUNUDPGRO = e.controlKnobs.DisableTUNUDPGRO.Load()
e.lastAppliedDisableTUNTCPGRO = e.controlKnobs.DisableTUNTCPGRO.Load()
}
// It's a little pointless to apply no-op settings here (they
// should already be empty?), but it at least exercises the
@@ -1278,7 +1294,26 @@ func (e *userspaceEngine) linkChange(delta *netmon.ChangeDelta) {
func (e *userspaceEngine) SetNetworkMap(nm *netmap.NetworkMap) {
e.mu.Lock()
e.netMap = nm
tunGROKnobsChanged := false
var curUDP, curTCP bool
if buildfeatures.HasGRO && runtime.GOOS == "linux" && e.controlKnobs != nil {
curUDP = e.controlKnobs.DisableTUNUDPGRO.Load()
curTCP = e.controlKnobs.DisableTUNTCPGRO.Load()
// Only act on transitions toward "disabled"; wireguard-go's GRO
// disablement is sticky and cannot be reversed without restart.
if (curUDP && !e.lastAppliedDisableTUNUDPGRO) ||
(curTCP && !e.lastAppliedDisableTUNTCPGRO) {
tunGROKnobsChanged = true
}
e.lastAppliedDisableTUNUDPGRO = curUDP
e.lastAppliedDisableTUNTCPGRO = curTCP
}
e.mu.Unlock()
if buildfeatures.HasGRO && tunGROKnobsChanged {
e.logf("wgengine: TUN GRO knobs changed (DisableTUNUDPGRO=%v DisableTUNTCPGRO=%v); applying",
curUDP, curTCP)
e.tundev.ApplyGROKnobs(e.controlKnobs)
}
if e.networkLogger.Running() {
e.networkLogger.ReconfigNetworkMap(nm)
}