diff --git a/tka/sync.go b/tka/sync.go index 20a033864..8804bcd27 100644 --- a/tka/sync.go +++ b/tka/sync.go @@ -9,6 +9,8 @@ import ( "errors" "fmt" "os" + + "tailscale.com/util/testenv" ) // ErrNoIntersection is returned when a shared AUM could @@ -257,3 +259,59 @@ func (a *Authority) MissingAUMs(storage Chonk, remoteOffer SyncOffer) ([]AUM, er panic("unreachable") } + +// seedNode is an authority-chonk pair that can be seeded by [SeedAUMs]. +type seedNode struct { + authority *Authority + storage Chonk +} + +// CreateSeedNode creates a node for use with [SeedAUMs]. +func CreateSeedNode(t testenv.TB, authority *Authority, storage Chonk) seedNode { + t.Helper() + return seedNode{authority, storage} +} + +// SeedAUMs generates many AUMs by repeatedly adding and removing keys +// from the TKA. +// +// The AUMs are written to all the supplied nodes, so if you pass more +// than one, you can build up a long sync history. +// +// This is only for use in testing. +func SeedAUMs(t testenv.TB, count int, signer Signer, nodes ...seedNode) { + t.Helper() + + if len(nodes) == 0 { + panic("called SeedAUMs without any nodes") + } + primaryNode := nodes[0] + + // The key that we'll repeatedly add/remove in the TKA. + key := Key{Kind: Key25519, Public: []byte{1, 1, 1}, Votes: 1} + + for i := 0; i < count/2; i++ { + for _, action := range []string{"add", "remove"} { + updater := primaryNode.authority.NewUpdater(signer) + if action == "add" { + if err := updater.AddKey(key); err != nil { + t.Fatalf("error from updater.AddKey: %v") + } + } else { + if err := updater.RemoveKey(key.MustID()); err != nil { + t.Fatalf("error from updater.RemoveKey: %v") + } + } + aum, err := updater.Finalize(primaryNode.storage) + if err != nil { + t.Fatalf("error from authority.Finalize: %v", err) + } + + for _, n := range nodes { + if err := n.authority.Inform(n.storage, aum); err != nil { + t.Fatalf("error from authority.Inform: %v", err) + } + } + } + } +} diff --git a/tka/sync_test.go b/tka/sync_test.go index ef3e9f2ac..7f5fc9eac 100644 --- a/tka/sync_test.go +++ b/tka/sync_test.go @@ -435,11 +435,9 @@ func TestSyncSimpleE2E(t *testing.T) { // Regression test for http://go/corp/40404 func TestSyncFromFarBehind(t *testing.T) { pub1, priv1 := testingKey25519(t, 1) - pub2, _ := testingKey25519(t, 2) signer1 := signer25519(priv1) key1 := Key{Kind: Key25519, Public: pub1, Votes: 2} - key2 := Key{Kind: Key25519, Public: pub2, Votes: 2} // Setup: persistentAuthority (control plane) vs compactingAuthority (client node). state := State{ @@ -462,20 +460,9 @@ func TestSyncFromFarBehind(t *testing.T) { compactingAuthority := must.Get(Bootstrap(compactingStorage, genesisAUM)) // 1. Generate enough history to trigger checkpoints. - for range checkpointEvery * 2 { - update := persistentAuthority.NewUpdater(signer1) - - must.Do(update.AddKey(key2)) - addKey := must.Get(update.Finalize(persistentStorage)) - must.Do(persistentAuthority.Inform(persistentStorage, addKey)) - must.Do(compactingAuthority.Inform(compactingStorage, addKey)) - - update = persistentAuthority.NewUpdater(signer1) - must.Do(update.RemoveKey(key2.MustID())) - removeKey := must.Get(update.Finalize(persistentStorage)) - must.Do(persistentAuthority.Inform(persistentStorage, removeKey)) - must.Do(compactingAuthority.Inform(compactingStorage, removeKey)) - } + persistentNode := CreateSeedNode(t, persistentAuthority, persistentStorage) + compactingNode := CreateSeedNode(t, compactingAuthority, compactingStorage) + SeedAUMs(t, checkpointEvery*2, signer1, persistentNode, compactingNode) t.Logf("genesis and first batch of AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize()) @@ -496,18 +483,102 @@ func TestSyncFromFarBehind(t *testing.T) { // // If you keep increasing this number, eventually the sync will fail because you // hit the hard-coded limits on iteration during the sync process. - for persistentSize() < compactingSize()+800 { - b := persistentAuthority.NewUpdater(signer1) + SeedAUMs(t, compactingSize()-persistentSize()+800, signer1, persistentNode) - must.Do(b.AddKey(key2)) - addKey := must.Get(b.Finalize(persistentStorage)) - must.Do(persistentAuthority.Inform(persistentStorage, addKey)) + t.Logf("post-compacting and extra AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize()) - b = persistentAuthority.NewUpdater(signer1) - must.Do(b.RemoveKey(key2.MustID())) - removeKey := must.Get(b.Finalize(persistentStorage)) - must.Do(persistentAuthority.Inform(persistentStorage, removeKey)) + // 4. Verify Intersection. + // The node should find an intersection even with a 500-AUM gap. + persistentOffer := must.Get(persistentAuthority.SyncOffer(persistentStorage)) + compactingOffer := must.Get(compactingAuthority.SyncOffer(compactingStorage)) + + if _, err := compactingAuthority.MissingAUMs(compactingStorage, persistentOffer); err != nil { + t.Errorf("node failed to find intersection with far-ahead control plane: %v", err) + } + + // 5. Check that the persistent authority can find an intersection with the + // compacting authority, and has missing AUMs to send it. + missing, err := persistentAuthority.MissingAUMs(persistentStorage, compactingOffer) + if len(missing) == 0 { + t.Errorf("control plane did not find any missing AUMs for node") + } + if err != nil { + t.Errorf("control plane failed to find missing AUMs for node: %v", err) + } +} + +// TestSyncFromFarBehindFork checks that nodes with compacted state that have +// also branched from the active chain can still find a common ancestor when +// the remote is significantly ahead of the inersection point. +// +// We simulate a node that has compacted its early history and is now ~500 AUMs +// behind the control plane, plus a few AUMs extra, a distance that previously +// caused exponential sampling in SyncOffer to skip the node's entire local history. +// +// Regression test for http://go/corp/40404 +func TestSyncFromFarBehindFork(t *testing.T) { + // Set up two signing keys. They have a different number of votes, so if there's + // a fork, the chain with the winning key will take precedence. + majorityPub, majorityPriv := testingKey25519(t, 1) + losingPub, losingPriv := testingKey25519(t, 2) + winningSigner := signer25519(majorityPriv) + losingSigner := signer25519(losingPriv) + + losingKey := Key{Kind: Key25519, Public: majorityPub, Votes: 1} + winningKey := Key{Kind: Key25519, Public: losingPub, Votes: 2} + + // Setup: persistentAuthority (control plane) vs compactingAuthority (client node). + state := State{ + Keys: []Key{losingKey, winningKey}, + DisablementValues: [][]byte{DisablementKDF([]byte{1, 2, 3})}, + } + + persistentStorage, compactingStorage := ChonkMem(), ChonkMem() + persistentSize := func() int { return len(must.Get(persistentStorage.AllAUMs())) } + compactingSize := func() int { return len(must.Get(compactingStorage.AllAUMs())) } + + // Backdate the clock on the compactingStorage so all AUMs will be old enough + // to be considered for compacting. + clock := tstest.NewClock(tstest.ClockOpts{ + Start: time.Now().Add(-(CompactionDefaults.MinAge + 24*time.Hour)), + }) + compactingStorage.SetClock(clock) + + persistentAuthority, genesisAUM := must.Get2(Create(persistentStorage, state, winningSigner)) + compactingAuthority := must.Get(Bootstrap(compactingStorage, genesisAUM)) + + // 1. Generate enough history to trigger checkpoints. + persistentNode := CreateSeedNode(t, persistentAuthority, persistentStorage) + compactingNode := CreateSeedNode(t, compactingAuthority, compactingStorage) + SeedAUMs(t, checkpointEvery*2, winningSigner, persistentNode, compactingNode) + + t.Logf("genesis and first batch of AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize()) + + // 2. Compact the node state. + // + // It now has a different 'oldestAncestor' than the control plane. + beforeCompacting := compactingSize() + must.Do(compactingAuthority.Compact(compactingStorage, CompactionDefaults)) + afterCompacting := compactingSize() + + if beforeCompacting == afterCompacting { + t.Errorf("expected Compact to reduce the number of AUMs, but unchanged: size = %d", afterCompacting) } + + // 2. Advance the node state slightly beyond the control plane, using the + // losing signer. + SeedAUMs(t, 1, losingSigner, compactingNode) + + // 3. Advance the control plane far beyond the node, using the winning signer. + // + // Now the node is forked from the control plane state, and the control plane's + // chain will win because its chain was signed by a key with more votes. + // + // As of 2026-04-17, the largest TKA has ~750 AUMs. + // + // If you keep increasing this number, eventually the sync will fail because you + // hit the hard-coded limits on iteration during the sync process. + SeedAUMs(t, compactingSize()-persistentSize()+800, winningSigner, persistentNode) t.Logf("post-compacting and extra AUMs: persistent = %d, compacting = %d", persistentSize(), compactingSize())