util/linuxfw,wgengine/router: allow incoming CGNAT range traffic with nodeattr

Clients with the newly added node attribute
`"disable-linux-cgnat-drop-rule"` will not automatically drop inbound
traffic on non-Tailscale network interfaces with the source IP in the
CGNAT IP range. This is an initial proof-of-concept for enabling
connectivity with off-Tailnet CGNAT endpoints.

Fixes tailscale/corp#36270.

Signed-off-by: Naman Sood <mail@nsood.in>
This commit is contained in:
Naman Sood
2026-04-14 16:45:06 -04:00
committed by GitHub
parent 5834058269
commit 6301a6ce4b
14 changed files with 527 additions and 69 deletions
+2
View File
@@ -95,3 +95,5 @@ func (f *FakeNetfilterRunner) DeleteSvc(svc, tun string, targetIPs []netip.Addr,
func (f *FakeNetfilterRunner) EnsurePortMapRuleForSvc(svc, tun string, targetIP netip.Addr, pm PortMap) error {
return nil
}
func (f *FakeNetfilterRunner) AddExternalCGNATRules(mode CGNATMode, tunname string) error { return nil }
func (f *FakeNetfilterRunner) DelExternalCGNATRules(mode CGNATMode, tunname string) error { return nil }
+63 -17
View File
@@ -214,23 +214,8 @@ func (i *iptablesRunner) AddBase(tunname string) error {
// addBase4 adds some basic IPv4 processing rules to be
// supplemented by later calls to other helpers.
func (i *iptablesRunner) addBase4(tunname string) error {
// Only allow CGNAT range traffic to come from tailscale0. There
// is an exception carved out for ranges used by ChromeOS, for
// which we fall out of the Tailscale chain.
//
// Note, this will definitely break nodes that end up using the
// CGNAT range for other purposes :(.
args := []string{"!", "-i", tunname, "-s", tsaddr.ChromeOSVMRange().String(), "-j", "RETURN"}
if err := i.ipt4.Append("filter", "ts-input", args...); err != nil {
return fmt.Errorf("adding %v in v4/filter/ts-input: %w", args, err)
}
args = []string{"!", "-i", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"}
if err := i.ipt4.Append("filter", "ts-input", args...); err != nil {
return fmt.Errorf("adding %v in v4/filter/ts-input: %w", args, err)
}
// Explicitly allow all other inbound traffic to the tun interface
args = []string{"-i", tunname, "-j", "ACCEPT"}
// Explicitly allow all inbound traffic to the tun interface
args := []string{"-i", tunname, "-j", "ACCEPT"}
if err := i.ipt4.Append("filter", "ts-input", args...); err != nil {
return fmt.Errorf("adding %v in v4/filter/ts-input: %w", args, err)
}
@@ -682,6 +667,67 @@ func (i *iptablesRunner) DelMagicsockPortRule(port uint16, network string) error
return nil
}
// buildExternalCGNATRules abstracts out logic for constructing firewall rules
// for handling non-Tailscale CGNAT traffic, since these rules need to be
// identical across [AddExternalCGNATRules] and [DelExternalCGNATRules].
func buildExternalCGNATRules(mode CGNATMode, tunname string) ([][]string, error) {
switch mode {
case CGNATModeDrop:
// Only allow CGNAT range traffic to come from the Tailscale interface.
// There is an exception carved out for ranges used by ChromeOS, for
// which we fall out of the Tailscale chain.
return [][]string{
{"!", "-i", tunname, "-s", tsaddr.ChromeOSVMRange().String(), "-j", "RETURN"},
{"!", "-i", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"},
}, nil
case CGNATModeReturn:
// Fall out of the Tailscale chain for CGNAT traffic that doesn't
// originate from the Tailscale interface.
return [][]string{
{"!", "-i", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "RETURN"},
}, nil
default:
return nil, fmt.Errorf("unsupported mode %q", mode)
}
}
// AddExternalCGNATRules adds rules to the ts-input chain to deal with
// traffic from the CGNAT range that arrives on non-Tailscale network
// interfaces.
func (i *iptablesRunner) AddExternalCGNATRules(mode CGNATMode, tunname string) error {
rules, err := buildExternalCGNATRules(mode, tunname)
if err != nil {
return fmt.Errorf("build cgnat mode rule: %v", err)
}
for _, rule := range rules {
if err := i.ipt4.Append("filter", "ts-input", rule...); err != nil {
return fmt.Errorf("adding %v in v4/filter/ts-input: %w", rule, err)
}
}
return nil
}
// DelExternalCGNATRules removes the rules created by AddExternalCGNATRules,
// if they exist.
func (i *iptablesRunner) DelExternalCGNATRules(mode CGNATMode, tunname string) error {
rules, err := buildExternalCGNATRules(mode, tunname)
if err != nil {
return fmt.Errorf("build cgnat mode rule: %v", err)
}
for _, rule := range rules {
if found, err := i.ipt4.Exists("filter", "ts-input", rule...); err != nil {
return fmt.Errorf("checking for %v in v4/filter/ts-input: %w", rule, err)
} else if !found {
// Don't need to delete a rule that isn't there.
continue
}
if err := i.ipt4.Delete("filter", "ts-input", rule...); err != nil {
return fmt.Errorf("deleting %v in v4/filter/ts-input: %w", rule, err)
}
}
return nil
}
// delTSHook deletes hook in a chain that jumps to a ts-chain. If the hook does not
// exist, it's a no-op since the desired state is already achieved but we log the
// error because error code from the iptables module resists unwrapping.
+53 -2
View File
@@ -126,8 +126,6 @@ func TestAddAndDeleteBase(t *testing.T) {
// Check that the rules were created.
tsRulesV4 := []fakeRule{ // table/chain/rule
{"filter", "ts-input", []string{"!", "-i", tunname, "-s", tsaddr.ChromeOSVMRange().String(), "-j", "RETURN"}},
{"filter", "ts-input", []string{"!", "-i", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"}},
{"filter", "ts-forward", []string{"-o", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"}},
}
@@ -504,3 +502,56 @@ func TestAddAndDelConnmarkSaveRule(t *testing.T) {
}
})
}
func TestAddAndDelCGNATRules(t *testing.T) {
iptr := newFakeIPTablesRunner()
tunname := "tun0"
// We need the chains to exist so we can add rules into them.
if err := iptr.AddChains(); err != nil {
t.Fatal(err)
}
tests := []struct {
mode CGNATMode
wantRules []fakeRule
}{
{
CGNATModeDrop, []fakeRule{
{"filter", "ts-input", []string{"!", "-i", tunname, "-s", tsaddr.ChromeOSVMRange().String(), "-j", "RETURN"}},
{"filter", "ts-input", []string{"!", "-i", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "DROP"}},
},
},
{
CGNATModeReturn, []fakeRule{
{"filter", "ts-input", []string{"!", "-i", tunname, "-s", tsaddr.CGNATRange().String(), "-j", "RETURN"}},
},
},
}
for _, tt := range tests {
if err := iptr.AddExternalCGNATRules(tt.mode, tunname); err != nil {
t.Fatal(err)
}
for _, tr := range tt.wantRules {
if exists, err := iptr.ipt4.Exists(tr.table, tr.chain, tr.args...); err != nil {
t.Fatalf("mode %q: error checking for rule: %v", tt.mode, err)
} else if !exists {
t.Errorf("mode %q: rule %s/%s/%s doesn't exist", tt.mode, tr.table, tr.chain, strings.Join(tr.args, " "))
}
}
if err := iptr.DelExternalCGNATRules(tt.mode, tunname); err != nil {
t.Fatal(err)
}
for _, tr := range tt.wantRules {
if exists, err := iptr.ipt4.Exists(tr.table, tr.chain, tr.args...); err != nil {
t.Fatalf("mode %q: error checking for rule: %v", tt.mode, err)
} else if exists {
t.Errorf("mode %q: rule %s/%s/%s not deleted", tt.mode, tr.table, tr.chain, strings.Join(tr.args, " "))
}
}
}
}
+7
View File
@@ -53,6 +53,13 @@ const (
FirewallModeNfTables FirewallMode = "nftables"
)
type CGNATMode string
const (
CGNATModeDrop CGNATMode = "DROP"
CGNATModeReturn CGNATMode = "RETURN"
)
// The following bits are added to packet marks for Tailscale use.
//
// We tried to pick bits sufficiently out of the way that it's
+147 -6
View File
@@ -593,6 +593,15 @@ type NetfilterRunner interface {
// DelMagicsockPortRule removes the rule created by AddMagicsockPortRule,
// if it exists.
DelMagicsockPortRule(port uint16, network string) error
// AddExternalCGNATRules adds rules to the ts-input chain to deal with
// traffic from the CGNAT range that arrives on non-Tailscale network
// interfaces.
AddExternalCGNATRules(mode CGNATMode, tunname string) error
// DelExternalCGNATRules removes the rules created by AddExternalCGNATRules,
// if they exist.
DelExternalCGNATRules(mode CGNATMode, tunname string) error
}
// New creates a NetfilterRunner, auto-detecting whether to use
@@ -1221,6 +1230,27 @@ func addReturnChromeOSVMRangeRule(c *nftables.Conn, table *nftables.Table, chain
return nil
}
// delReturnChromeOSVMRangeRule deletes the rule created by addReturnChromeOSVMRangeRule,
// if it exists.
func delReturnChromeOSVMRangeRule(c *nftables.Conn, table *nftables.Table, chain *nftables.Chain, tunname string) error {
rule, err := createRangeRule(table, chain, tunname, tsaddr.ChromeOSVMRange(), expr.VerdictReturn)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
rule, err = findRule(c, rule)
if err != nil {
return fmt.Errorf("find rule: %v", err)
}
if rule == nil {
return nil
}
_ = c.DelRule(rule)
if err := c.Flush(); err != nil {
return fmt.Errorf("flush del rule: %w", err)
}
return nil
}
// addDropCGNATRangeRule adds a rule to drop if the source IP is in the
// CGNAT range.
func addDropCGNATRangeRule(c *nftables.Conn, table *nftables.Table, chain *nftables.Chain, tunname string) error {
@@ -1235,6 +1265,62 @@ func addDropCGNATRangeRule(c *nftables.Conn, table *nftables.Table, chain *nftab
return nil
}
// delDropCGNATRangeRule deletes the rule created by addDropCGNATRangeRule,
// if it exists.
func delDropCGNATRangeRule(c *nftables.Conn, table *nftables.Table, chain *nftables.Chain, tunname string) error {
rule, err := createRangeRule(table, chain, tunname, tsaddr.CGNATRange(), expr.VerdictDrop)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
rule, err = findRule(c, rule)
if err != nil {
return fmt.Errorf("find rule: %v", err)
}
if rule == nil {
return nil
}
_ = c.DelRule(rule)
if err := c.Flush(); err != nil {
return fmt.Errorf("flush del rule: %w", err)
}
return nil
}
// addReturnCGNATRangeRule adds a rule to return if the source IP is in the
// CGNAT range.
func addReturnCGNATRangeRule(c *nftables.Conn, table *nftables.Table, chain *nftables.Chain, tunname string) error {
rule, err := createRangeRule(table, chain, tunname, tsaddr.CGNATRange(), expr.VerdictReturn)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
_ = c.AddRule(rule)
if err = c.Flush(); err != nil {
return fmt.Errorf("add rule: %w", err)
}
return nil
}
// delReturnCGNATRangeRule deletes the rule created by addReturnCGNATRangeRule,
// if it exists.
func delReturnCGNATRangeRule(c *nftables.Conn, table *nftables.Table, chain *nftables.Chain, tunname string) error {
rule, err := createRangeRule(table, chain, tunname, tsaddr.CGNATRange(), expr.VerdictReturn)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
rule, err = findRule(c, rule)
if err != nil {
return fmt.Errorf("find rule: %v", err)
}
if rule == nil {
return nil
}
_ = c.DelRule(rule)
if err := c.Flush(); err != nil {
return fmt.Errorf("flush del rule: %w", err)
}
return nil
}
// createSetSubnetRouteMarkRule creates a rule to set the subnet route
// mark if the packet is from the given interface.
func createSetSubnetRouteMarkRule(table *nftables.Table, chain *nftables.Chain, tunname string) (*nftables.Rule, error) {
@@ -1502,6 +1588,67 @@ func (n *nftablesRunner) DelMagicsockPortRule(port uint16, network string) error
return nil
}
// AddExternalCGNATRules adds rules to the ts-input chain to deal with
// traffic from the CGNAT range that arrives on non-Tailscale network
// interfaces.
func (n *nftablesRunner) AddExternalCGNATRules(mode CGNATMode, tunname string) error {
conn := n.conn
inputChain, err := getChainFromTable(conn, n.nft4.Filter, chainNameInput)
if err != nil {
return fmt.Errorf("get input chain v4: %v", err)
}
switch mode {
case CGNATModeDrop:
if err = addReturnChromeOSVMRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("add return chromeos vm range rule v4: %w", err)
}
if err = addDropCGNATRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("add drop cgnat range rule v4: %w", err)
}
case CGNATModeReturn:
if err = addReturnCGNATRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("add return cgnat range rule v4: %w", err)
}
default:
return fmt.Errorf("unsupported cgnat mode %q", mode)
}
if err = conn.Flush(); err != nil {
return fmt.Errorf("flush cgnat rules v4: %w", err)
}
return nil
}
// DelExternalCGNATRules removes the rules created by AddExternalCGNATRules,
// if they exist.
func (n *nftablesRunner) DelExternalCGNATRules(mode CGNATMode, tunname string) error {
conn := n.conn
inputChain, err := getChainFromTable(conn, n.nft4.Filter, chainNameInput)
if err != nil {
return fmt.Errorf("get input chain v4: %v", err)
}
switch mode {
case CGNATModeDrop:
if err = delReturnChromeOSVMRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("del return chromeos vm range rule v4: %w", err)
}
if err = delDropCGNATRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("del drop cgnat range rule v4: %w", err)
}
case CGNATModeReturn:
if err = delReturnCGNATRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("del return cgnat range rule v4: %w", err)
}
default:
return fmt.Errorf("unsupported mode %q", mode)
}
if err = conn.Flush(); err != nil {
return fmt.Errorf("flush cgnat rules v4: %w", err)
}
return nil
}
// createAcceptIncomingPacketRule creates a rule to accept incoming packets to
// the given interface.
func createAcceptIncomingPacketRule(table *nftables.Table, chain *nftables.Chain, tunname string) *nftables.Rule {
@@ -1555,12 +1702,6 @@ func (n *nftablesRunner) addBase4(tunname string) error {
if err != nil {
return fmt.Errorf("get input chain v4: %v", err)
}
if err = addReturnChromeOSVMRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("add return chromeos vm range rule v4: %w", err)
}
if err = addDropCGNATRangeRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("add drop cgnat range rule v4: %w", err)
}
if err = addAcceptIncomingPacketRule(conn, n.nft4.Filter, inputChain, tunname); err != nil {
return fmt.Errorf("add accept incoming packet rule v4: %w", err)
}
+97 -21
View File
@@ -633,7 +633,7 @@ func TestAddAndDelNetfilterChains(t *testing.T) {
func getTsChains(
conn *nftables.Conn,
proto nftables.TableFamily) (*nftables.Chain, *nftables.Chain, *nftables.Chain, error) {
chains, err := conn.ListChainsOfTableFamily(nftables.TableFamilyIPv4)
chains, err := conn.ListChainsOfTableFamily(proto)
if err != nil {
return nil, nil, nil, fmt.Errorf("list chains failed: %w", err)
}
@@ -658,17 +658,7 @@ func findV4BaseRules(
forwChain *nftables.Chain,
tunname string) ([]*nftables.Rule, error) {
want := []*nftables.Rule{}
rule, err := createRangeRule(inpChain.Table, inpChain, tunname, tsaddr.ChromeOSVMRange(), expr.VerdictReturn)
if err != nil {
return nil, fmt.Errorf("create rule: %w", err)
}
want = append(want, rule)
rule, err = createRangeRule(inpChain.Table, inpChain, tunname, tsaddr.CGNATRange(), expr.VerdictDrop)
if err != nil {
return nil, fmt.Errorf("create rule: %w", err)
}
want = append(want, rule)
rule, err = createDropOutgoingPacketFromCGNATRangeRuleWithTunname(forwChain.Table, forwChain, tunname)
rule, err := createDropOutgoingPacketFromCGNATRangeRuleWithTunname(forwChain.Table, forwChain, tunname)
if err != nil {
return nil, fmt.Errorf("create rule: %w", err)
}
@@ -745,7 +735,7 @@ func TestNFTAddAndDelNetfilterBase(t *testing.T) {
if err != nil {
t.Fatalf("getTsChains() failed: %v", err)
}
checkChainRules(t, conn, inputV4, 3)
checkChainRules(t, conn, inputV4, 1)
checkChainRules(t, conn, forwardV4, 4)
checkChainRules(t, conn, postroutingV4, 0)
@@ -763,8 +753,8 @@ func TestNFTAddAndDelNetfilterBase(t *testing.T) {
if err != nil {
t.Fatalf("getTsChains() failed: %v", err)
}
checkChainRules(t, conn, inputV6, 3)
checkChainRules(t, conn, forwardV6, 4)
checkChainRules(t, conn, inputV6, 1)
checkChainRules(t, conn, forwardV6, 3)
checkChainRules(t, conn, postroutingV6, 0)
_, err = findCommonBaseRules(conn, forwardV6, "testTunn")
@@ -783,6 +773,92 @@ func TestNFTAddAndDelNetfilterBase(t *testing.T) {
}
}
func findCGNATRules(
conn *nftables.Conn,
inpChain *nftables.Chain,
mode CGNATMode,
tunname string,
) error {
want := []*nftables.Rule{}
switch mode {
case CGNATModeDrop:
rule, err := createRangeRule(inpChain.Table, inpChain, tunname, tsaddr.ChromeOSVMRange(), expr.VerdictReturn)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
want = append(want, rule)
rule, err = createRangeRule(inpChain.Table, inpChain, tunname, tsaddr.CGNATRange(), expr.VerdictDrop)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
want = append(want, rule)
case CGNATModeReturn:
rule, err := createRangeRule(inpChain.Table, inpChain, tunname, tsaddr.CGNATRange(), expr.VerdictReturn)
if err != nil {
return fmt.Errorf("create rule: %w", err)
}
want = append(want, rule)
default:
return fmt.Errorf("unknown mode %q", mode)
}
for _, rule := range want {
_, err := findRule(conn, rule)
if err != nil {
return fmt.Errorf("find rule: %w", err)
}
}
return nil
}
func TestNFTAddAndDelCGNATRules(t *testing.T) {
modes := []CGNATMode{CGNATModeDrop, CGNATModeReturn}
for _, mode := range modes {
t.Run(string(mode), func(t *testing.T) {
conn := newSysConn(t)
runner := newFakeNftablesRunnerWithConn(t, conn, false)
if err := runner.AddChains(); err != nil {
t.Fatalf("AddChains() failed: %v", err)
}
defer runner.DelChains()
inputV4, _, _, err := getTsChains(conn, nftables.TableFamilyIPv4)
if err != nil {
t.Fatalf("getTsChains() failed: %v", err)
}
checkChainRules(t, conn, inputV4, 0)
tunname := "tun0"
if err := runner.AddExternalCGNATRules(mode, tunname); err != nil {
t.Fatalf("add rules: %v", err)
}
switch mode {
case CGNATModeDrop:
checkChainRules(t, conn, inputV4, 2)
case CGNATModeReturn:
checkChainRules(t, conn, inputV4, 1)
default:
t.Fatalf("unknown mode %q", mode)
}
if err := findCGNATRules(conn, inputV4, mode, tunname); err != nil {
t.Fatalf("find rules: %v", err)
}
if err := runner.DelExternalCGNATRules(mode, tunname); err != nil {
t.Fatalf("delete rules: %v", err)
}
// Verify that all the rules have been deleted (0 remaining).
checkChainRules(t, conn, inputV4, 0)
})
}
}
func findLoopBackRule(conn *nftables.Conn, proto nftables.TableFamily, table *nftables.Table, chain *nftables.Chain, addr netip.Addr) (*nftables.Rule, error) {
matchingAddr := addr.AsSlice()
saddrExpr, err := newLoadSaddrExpr(proto, 1)
@@ -845,16 +921,16 @@ func TestNFTAddAndDelLoopbackRule(t *testing.T) {
runner.AddBase("testTunn")
defer runner.DelBase()
checkChainRules(t, conn, inputV4, 3)
checkChainRules(t, conn, inputV6, 3)
checkChainRules(t, conn, inputV4, 1)
checkChainRules(t, conn, inputV6, 1)
addr := netip.MustParseAddr("192.168.0.2")
addrV6 := netip.MustParseAddr("2001:db8::2")
runner.AddLoopbackRule(addr)
runner.AddLoopbackRule(addrV6)
checkChainRules(t, conn, inputV4, 4)
checkChainRules(t, conn, inputV6, 4)
checkChainRules(t, conn, inputV4, 2)
checkChainRules(t, conn, inputV6, 2)
existingLoopBackRule, err := findLoopBackRule(conn, nftables.TableFamilyIPv4, runner.nft4.Filter, inputV4, addr)
if err != nil {
@@ -877,8 +953,8 @@ func TestNFTAddAndDelLoopbackRule(t *testing.T) {
runner.DelLoopbackRule(addr)
runner.DelLoopbackRule(addrV6)
checkChainRules(t, conn, inputV4, 3)
checkChainRules(t, conn, inputV6, 3)
checkChainRules(t, conn, inputV4, 1)
checkChainRules(t, conn, inputV6, 1)
}
func TestNFTAddAndDelHookRule(t *testing.T) {