tka: implement compaction logic

Signed-off-by: Tom DNetto <tom@tailscale.com>

tka/tailchonk.go

@@ -5,10 +5,12 @@ package tka
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/fxamacker/cbor/v2"
 	"tailscale.com/atomicfile"
@@ -53,6 +55,24 @@ type Chonk interface {
 	LastActiveAncestor() (*AUMHash, error)
 }
 
+// CompactableChonk implementation are extensions of Chonk, which are
+// able to be operated by compaction logic to deleted old AUMs.
+type CompactableChonk interface {
+	Chonk
+
+	// AllAUMs returns all AUMs stored in the chonk.
+	AllAUMs() ([]AUMHash, error)
+
+	// CommitTime returns the time at which the AUM was committed.
+	//
+	// If the AUM does not exist, then os.ErrNotExist is returned.
+	CommitTime(hash AUMHash) (time.Time, error)
+
+	// PurgeAUMs permanently and irrevocably deletes the specified
+	// AUMs from storage.
+	PurgeAUMs(hashes []AUMHash) error
+}
+
 // Mem implements in-memory storage of TKA state, suitable for
 // tests.
 //
@@ -437,3 +457,302 @@ func (c *FS) commit(h AUMHash, updater func(*fsHashInfo)) error {
 	}
 	return atomicfile.WriteFile(filepath.Join(dir, base), buff.Bytes(), 0644)
 }
+
+// CompactionOptions describes tuneables to use when compacting a Chonk.
+type CompactionOptions struct {
+	// The minimum number of ancestor AUMs to remember. The actual length
+	// of the chain post-compaction may be longer to reach a Checkpoint AUM.
+	MinChain int
+	// The minimum duration to store an AUM before it is a candidate for deletion.
+	MinAge time.Duration
+}
+
+// retainState tracks the state of an AUM hash as it is being considered for
+// deletion.
+type retainState uint8
+
+// Valid retainState flags.
+const (
+	retainStateActive    retainState = 1 << iota // The AUM is part of the active chain and less than MinChain hops from HEAD.
+	retainStateYoung                             // The AUM is younger than MinAge.
+	retainStateLeaf                              // The AUM is a descendant of an AUM to be retained.
+	retainStateAncestor                          // The AUM is part of a chain between a retained AUM and the new lastActiveAncestor.
+	retainStateCandidate                         // The AUM is part of the active chain.
+
+	// retainAUMMask is a bit mask of any bit which should prevent
+	// the deletion of an AUM.
+	retainAUMMask retainState = retainStateActive | retainStateYoung | retainStateLeaf | retainStateAncestor
+)
+
+// markActiveChain marks AUMs in the active chain.
+// All AUMs that are within minChain ancestors of head are
+// marked retainStateActive, and all remaining ancestors are
+// marked retainStateCandidate.
+//
+// markActiveChain returns the next ancestor AUM which is a checkpoint AUM.
+func markActiveChain(storage Chonk, verdict map[AUMHash]retainState, minChain int, head AUMHash) (lastActiveAncestor AUMHash, err error) {
+	next, err := storage.AUM(head)
+	if err != nil {
+		return AUMHash{}, err
+	}
+
+	for i := 0; i < minChain; i++ {
+		h := next.Hash()
+		verdict[h] |= retainStateActive
+
+		parent, hasParent := next.Parent()
+		if !hasParent {
+			// Genesis AUM (beginning of time). The chain isnt long enough to need truncating.
+			return h, nil
+		}
+
+		if next, err = storage.AUM(parent); err != nil {
+			if err == os.ErrNotExist {
+				// We've reached the end of the chain we have stored.
+				return h, nil
+			}
+			return AUMHash{}, fmt.Errorf("reading active chain (retainStateActive) (%d): %w", i, err)
+		}
+	}
+
+	// If we got this far, we have at least minChain AUMs stored, and minChain number
+	// of ancestors have been marked for retention. We now continue to iterate backwards
+	// till we find an AUM which we can compact to (a Checkpoint AUM).
+	for {
+		h := next.Hash()
+		verdict[h] |= retainStateActive
+		if next.MessageKind == AUMCheckpoint {
+			lastActiveAncestor = h
+			break
+		}
+
+		parent, hasParent := next.Parent()
+		if !hasParent {
+			return AUMHash{}, errors.New("reached genesis AUM without finding an appropriate lastActiveAncestor")
+		}
+		if next, err = storage.AUM(parent); err != nil {
+			return AUMHash{}, fmt.Errorf("searching for compaction target: %w", err)
+		}
+	}
+
+	// Mark remaining known ancestors as retainStateCandidate.
+	for {
+		parent, hasParent := next.Parent()
+		if !hasParent {
+			break
+		}
+		verdict[parent] |= retainStateCandidate
+		if next, err = storage.AUM(parent); err != nil {
+			if err == os.ErrNotExist {
+				// We've reached the end of the chain we have stored.
+				break
+			}
+			return AUMHash{}, fmt.Errorf("reading active chain (retainStateCandidate): %w", err)
+		}
+	}
+
+	return lastActiveAncestor, nil
+}
+
+// markYoungAUMs marks all AUMs younger than minAge for retention. All
+// candidate AUMs must exist in verdict.
+func markYoungAUMs(storage CompactableChonk, verdict map[AUMHash]retainState, minAge time.Duration) error {
+	minTime := time.Now().Add(-minAge)
+	for h, _ := range verdict {
+		commitTime, err := storage.CommitTime(h)
+		if err != nil {
+			return err
+		}
+
+		if commitTime.After(minTime) {
+			verdict[h] |= retainStateYoung
+		}
+	}
+	return nil
+}
+
+// markAncestorIntersectionAUMs walks backwards from all AUMs to be retained,
+// ensuring they intersect with candidateAncestor. All AUMs between a retained
+// AUM and candidateAncestor are marked for retention.
+//
+// If there is no intersection between candidateAncestor and the ancestors of
+// a retained AUM (this can happen if a retained AUM intersects the main chain
+// before candidateAncestor) then candidate ancestor is recomputed based on
+// the new oldest intersection.
+//
+// The final value for lastActiveAncestor is returned.
+func markAncestorIntersectionAUMs(storage Chonk, verdict map[AUMHash]retainState, candidateAncestor AUMHash) (lastActiveAncestor AUMHash, err error) {
+	toScan := make([]AUMHash, 0, len(verdict))
+	for h, v := range verdict {
+		if (v & retainAUMMask) == 0 {
+			continue // not marked for retention, so dont need to consider it
+		}
+		if h == candidateAncestor {
+			continue
+		}
+		toScan = append(toScan, h)
+	}
+
+	var didAdjustCandidateAncestor bool
+	for len(toScan) > 0 {
+		nextIterScan := make([]AUMHash, 0, len(verdict))
+		for _, h := range toScan {
+			if verdict[h]&retainStateAncestor != 0 {
+				// This AUM and its ancestors have already been iterated.
+				continue
+			}
+			verdict[h] |= retainStateAncestor
+
+			a, err := storage.AUM(h)
+			if err != nil {
+				return AUMHash{}, fmt.Errorf("reading %v: %w", h, err)
+			}
+			parent, hasParent := a.Parent()
+			if !hasParent {
+				return AUMHash{}, errors.New("reached genesis AUM without intersecting with candidate ancestor")
+			}
+
+			if verdict[parent]&retainAUMMask != 0 {
+				// Includes candidateAncestor (has retainStateActive set)
+				continue
+			}
+			if verdict[parent]&retainStateCandidate != 0 {
+				// We've intersected with the active chain but haven't done so through
+				// candidateAncestor. That means that we intersect the active chain
+				// before candidateAncestor, hence candidateAncestor actually needs
+				// to be earlier than it is now.
+				candidateAncestor = parent
+				didAdjustCandidateAncestor = true
+				verdict[parent] |= retainStateAncestor
+
+				// There could be AUMs on the active chain between our new candidateAncestor
+				// and the old one, make sure they are marked as retained.
+				next := parent
+			childLoop:
+				for {
+					children, err := storage.ChildAUMs(next)
+					if err != nil {
+						return AUMHash{}, fmt.Errorf("reading children %v: %w", next, err)
+					}
+					// While there can be many children of an AUM, there can only be
+					// one child on the active chain (it will have retainStateCandidate set).
+					for _, a := range children {
+						h := a.Hash()
+						if v := verdict[h]; v&retainStateCandidate != 0 && v&retainStateActive == 0 {
+							verdict[h] |= retainStateAncestor
+							next = h
+							continue childLoop
+						}
+					}
+					break
+				}
+			}
+
+			nextIterScan = append(nextIterScan, parent)
+		}
+		toScan = nextIterScan
+	}
+
+	// If candidateAncestor was adjusted backwards, then it may not be a checkpoint
+	// (and hence a valid compaction candidate). If so, iterate backwards and adjust
+	// the candidateAncestor till we find a checkpoint.
+	if didAdjustCandidateAncestor {
+		var next AUM
+		if next, err = storage.AUM(candidateAncestor); err != nil {
+			return AUMHash{}, fmt.Errorf("searching for compaction target: %w", err)
+		}
+
+		for {
+			h := next.Hash()
+			verdict[h] |= retainStateActive
+			if next.MessageKind == AUMCheckpoint {
+				candidateAncestor = h
+				break
+			}
+
+			parent, hasParent := next.Parent()
+			if !hasParent {
+				return AUMHash{}, errors.New("reached genesis AUM without finding an appropriate candidateAncestor")
+			}
+			if next, err = storage.AUM(parent); err != nil {
+				return AUMHash{}, fmt.Errorf("searching for compaction target: %w", err)
+			}
+		}
+	}
+
+	return candidateAncestor, nil
+}
+
+// markDescendantAUMs marks all children of a retained AUM as retained.
+func markDescendantAUMs(storage Chonk, verdict map[AUMHash]retainState) error {
+	toScan := make([]AUMHash, 0, len(verdict))
+	for h, v := range verdict {
+		if v&retainAUMMask == 0 {
+			continue // not marked, so dont need to mark descendants
+		}
+		toScan = append(toScan, h)
+	}
+
+	for len(toScan) > 0 {
+		nextIterScan := make([]AUMHash, 0, len(verdict))
+		for _, h := range toScan {
+			if verdict[h]&retainStateLeaf != 0 {
+				// This AUM and its decendants have already been marked.
+				continue
+			}
+			verdict[h] |= retainStateLeaf
+
+			children, err := storage.ChildAUMs(h)
+			if err != nil {
+				return err
+			}
+			for _, a := range children {
+				nextIterScan = append(nextIterScan, a.Hash())
+			}
+		}
+		toScan = nextIterScan
+	}
+
+	return nil
+}
+
+// Compact deletes old AUMs from storage, based on the parameters given in opts.
+func Compact(storage CompactableChonk, head AUMHash, opts CompactionOptions) (lastActiveAncestor AUMHash, err error) {
+	if opts.MinChain == 0 {
+		return AUMHash{}, errors.New("opts.MinChain must be set")
+	}
+	if opts.MinAge == 0 {
+		return AUMHash{}, errors.New("opts.MinAge must be set")
+	}
+
+	all, err := storage.AllAUMs()
+	if err != nil {
+		return AUMHash{}, fmt.Errorf("AllAUMs: %w", err)
+	}
+	verdict := make(map[AUMHash]retainState, len(all))
+	for _, h := range all {
+		verdict[h] = 0
+	}
+
+	if lastActiveAncestor, err = markActiveChain(storage, verdict, opts.MinChain, head); err != nil {
+		return AUMHash{}, fmt.Errorf("marking active chain: %w", err)
+	}
+	if err := markYoungAUMs(storage, verdict, opts.MinAge); err != nil {
+		return AUMHash{}, fmt.Errorf("marking young AUMs: %w", err)
+	}
+	if err := markDescendantAUMs(storage, verdict); err != nil {
+		return AUMHash{}, fmt.Errorf("marking decendant AUMs: %w", err)
+	}
+	if lastActiveAncestor, err = markAncestorIntersectionAUMs(storage, verdict, lastActiveAncestor); err != nil {
+		return AUMHash{}, fmt.Errorf("marking ancestor intersection: %w", err)
+	}
+
+	toDelete := make([]AUMHash, 0, len(verdict))
+	for h, v := range verdict {
+		if v&retainAUMMask == 0 { // no retention set
+			toDelete = append(toDelete, h)
+		}
+	}
+
+	return lastActiveAncestor, storage.PurgeAUMs(toDelete)
+}

tka/tailchonk_test.go

@@ -4,12 +4,15 @@
 package tka
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
 	"golang.org/x/crypto/blake2s"
 )
 
@@ -171,3 +174,431 @@ func TestTailchonkFS_Commit(t *testing.T) {
 		t.Errorf("stat of AUM parent failed: %v", err)
 	}
 }
+
+func TestMarkActiveChain(t *testing.T) {
+	type aumTemplate struct {
+		AUM AUM
+	}
+
+	tcs := []struct {
+		name                string
+		minChain            int
+		chain               []aumTemplate
+		expectLastActiveIdx int // expected lastActiveAncestor, corresponds to an index on chain.
+	}{
+		{
+			name:     "genesis",
+			minChain: 2,
+			chain: []aumTemplate{
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+			},
+			expectLastActiveIdx: 0,
+		},
+		{
+			name:     "simple truncate",
+			minChain: 2,
+			chain: []aumTemplate{
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+			},
+			expectLastActiveIdx: 1,
+		},
+		{
+			name:     "long truncate",
+			minChain: 5,
+			chain: []aumTemplate{
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+			},
+			expectLastActiveIdx: 2,
+		},
+		{
+			name:     "truncate finding checkpoint",
+			minChain: 2,
+			chain: []aumTemplate{
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMAddKey, Key: &Key{}}}, // Should keep searching upwards for a checkpoint
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+				{AUM: AUM{MessageKind: AUMCheckpoint, State: &State{}}},
+			},
+			expectLastActiveIdx: 1,
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			verdict := make(map[AUMHash]retainState, len(tc.chain))
+
+			// Build the state of the tailchonk for tests.
+			storage := &Mem{}
+			var prev AUMHash
+			for i := range tc.chain {
+				if !prev.IsZero() {
+					tc.chain[i].AUM.PrevAUMHash = make([]byte, len(prev[:]))
+					copy(tc.chain[i].AUM.PrevAUMHash, prev[:])
+				}
+				if err := storage.CommitVerifiedAUMs([]AUM{tc.chain[i].AUM}); err != nil {
+					t.Fatal(err)
+				}
+
+				h := tc.chain[i].AUM.Hash()
+				prev = h
+				verdict[h] = 0
+			}
+
+			got, err := markActiveChain(storage, verdict, tc.minChain, prev)
+			if err != nil {
+				t.Logf("state = %+v", verdict)
+				t.Fatalf("markActiveChain() failed: %v", err)
+			}
+			want := tc.chain[tc.expectLastActiveIdx].AUM.Hash()
+			if got != want {
+				t.Logf("state = %+v", verdict)
+				t.Errorf("lastActiveAncestor = %v, want %v", got, want)
+			}
+
+			// Make sure the verdict array was marked correctly.
+			for i := range tc.chain {
+				h := tc.chain[i].AUM.Hash()
+				if i >= tc.expectLastActiveIdx {
+					if (verdict[h] & retainStateActive) == 0 {
+						t.Errorf("verdict[%v] = %v, want %v set", h, verdict[h], retainStateActive)
+					}
+				} else {
+					if (verdict[h] & retainStateCandidate) == 0 {
+						t.Errorf("verdict[%v] = %v, want %v set", h, verdict[h], retainStateCandidate)
+					}
+				}
+			}
+		})
+	}
+}
+
+func TestMarkDescendantAUMs(t *testing.T) {
+	c := newTestchain(t, `
+        genesis -> B -> C -> C2
+                   | -> D
+                   | -> E -> F -> G -> H
+                        | -> E2
+
+        // tweak seeds so hashes arent identical
+        C.hashSeed = 1
+        D.hashSeed = 2
+        E.hashSeed = 3
+        E2.hashSeed = 4
+    `)
+
+	verdict := make(map[AUMHash]retainState, len(c.AUMs))
+	for _, a := range c.AUMs {
+		verdict[a.Hash()] = 0
+	}
+
+	// Mark E & C.
+	verdict[c.AUMHashes["C"]] = retainStateActive
+	verdict[c.AUMHashes["E"]] = retainStateActive
+
+	if err := markDescendantAUMs(c.Chonk(), verdict); err != nil {
+		t.Errorf("markDescendantAUMs() failed: %v", err)
+	}
+
+	// Make sure the descendants got marked.
+	hs := c.AUMHashes
+	for _, h := range []AUMHash{hs["C2"], hs["F"], hs["G"], hs["H"], hs["E2"]} {
+		if (verdict[h] & retainStateLeaf) == 0 {
+			t.Errorf("%v was not marked as a descendant", h)
+		}
+	}
+	for _, h := range []AUMHash{hs["genesis"], hs["B"], hs["D"]} {
+		if (verdict[h] & retainStateLeaf) != 0 {
+			t.Errorf("%v was marked as a descendant and shouldnt be", h)
+		}
+	}
+}
+
+func TestMarkAncestorIntersectionAUMs(t *testing.T) {
+	fakeState := &State{
+		Keys:               []Key{{Kind: Key25519, Votes: 1}},
+		DisablementSecrets: [][]byte{bytes.Repeat([]byte{1}, 32)},
+	}
+
+	tcs := []struct {
+		name            string
+		chain           *testChain
+		verdicts        map[string]retainState
+		initialAncestor string
+		wantAncestor    string
+		wantRetained    []string
+		wantDeleted     []string
+	}{
+		{
+			name: "genesis",
+			chain: newTestchain(t, `
+                A
+                A.template = checkpoint`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
+			initialAncestor: "A",
+			wantAncestor:    "A",
+			verdicts: map[string]retainState{
+				"A": retainStateActive,
+			},
+			wantRetained: []string{"A"},
+		},
+		{
+			name: "no adjustment",
+			chain: newTestchain(t, `
+                DEAD -> A -> B -> C
+                A.template = checkpoint
+                B.template = checkpoint`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
+			initialAncestor: "A",
+			wantAncestor:    "A",
+			verdicts: map[string]retainState{
+				"A":    retainStateActive,
+				"B":    retainStateActive,
+				"C":    retainStateActive,
+				"DEAD": retainStateCandidate,
+			},
+			wantRetained: []string{"A", "B", "C"},
+			wantDeleted:  []string{"DEAD"},
+		},
+		{
+			name: "fork",
+			chain: newTestchain(t, `
+                A -> B -> C -> D
+                          | -> FORK
+                A.template = checkpoint
+                C.template = checkpoint
+                D.template = checkpoint
+                FORK.hashSeed = 2`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
+			initialAncestor: "D",
+			wantAncestor:    "C",
+			verdicts: map[string]retainState{
+				"A":    retainStateCandidate,
+				"B":    retainStateCandidate,
+				"C":    retainStateCandidate,
+				"D":    retainStateActive,
+				"FORK": retainStateYoung,
+			},
+			wantRetained: []string{"C", "D", "FORK"},
+			wantDeleted:  []string{"A", "B"},
+		},
+		{
+			name: "fork finding earlier checkpoint",
+			chain: newTestchain(t, `
+                A -> B -> C -> D -> E -> F
+                          | -> FORK
+                A.template = checkpoint
+                B.template = checkpoint
+                E.template = checkpoint
+                FORK.hashSeed = 2`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
+			initialAncestor: "E",
+			wantAncestor:    "B",
+			verdicts: map[string]retainState{
+				"A":    retainStateCandidate,
+				"B":    retainStateCandidate,
+				"C":    retainStateCandidate,
+				"D":    retainStateCandidate,
+				"E":    retainStateActive,
+				"F":    retainStateActive,
+				"FORK": retainStateYoung,
+			},
+			wantRetained: []string{"B", "C", "D", "E", "F", "FORK"},
+			wantDeleted:  []string{"A"},
+		},
+		{
+			name: "fork multi",
+			chain: newTestchain(t, `
+                A -> B -> C -> D -> E
+                               | -> DEADFORK
+                C -> FORK
+                A.template = checkpoint
+                C.template = checkpoint
+                D.template = checkpoint
+                E.template = checkpoint
+                FORK.hashSeed = 2
+                DEADFORK.hashSeed = 3`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
+			initialAncestor: "D",
+			wantAncestor:    "C",
+			verdicts: map[string]retainState{
+				"A":        retainStateCandidate,
+				"B":        retainStateCandidate,
+				"C":        retainStateCandidate,
+				"D":        retainStateActive,
+				"E":        retainStateActive,
+				"FORK":     retainStateYoung,
+				"DEADFORK": 0,
+			},
+			wantRetained: []string{"C", "D", "E", "FORK"},
+			wantDeleted:  []string{"A", "B", "DEADFORK"},
+		},
+		{
+			name: "fork multi 2",
+			chain: newTestchain(t, `
+                A -> B -> C -> D -> E -> F -> G
+
+                F -> F1
+                D -> F2
+                B -> F3
+
+                A.template = checkpoint
+                B.template = checkpoint
+                D.template = checkpoint
+                F.template = checkpoint
+                F1.hashSeed = 2
+                F2.hashSeed = 3
+                F3.hashSeed = 4`, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState})),
+			initialAncestor: "F",
+			wantAncestor:    "B",
+			verdicts: map[string]retainState{
+				"A":  retainStateCandidate,
+				"B":  retainStateCandidate,
+				"C":  retainStateCandidate,
+				"D":  retainStateCandidate,
+				"E":  retainStateCandidate,
+				"F":  retainStateActive,
+				"G":  retainStateActive,
+				"F1": retainStateYoung,
+				"F2": retainStateYoung,
+				"F3": retainStateYoung,
+			},
+			wantRetained: []string{"B", "C", "D", "E", "F", "G", "F1", "F2", "F3"},
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			verdict := make(map[AUMHash]retainState, len(tc.verdicts))
+			for name, v := range tc.verdicts {
+				verdict[tc.chain.AUMHashes[name]] = v
+			}
+
+			got, err := markAncestorIntersectionAUMs(tc.chain.Chonk(), verdict, tc.chain.AUMHashes[tc.initialAncestor])
+			if err != nil {
+				t.Logf("state = %+v", verdict)
+				t.Fatalf("markAncestorIntersectionAUMs() failed: %v", err)
+			}
+			if want := tc.chain.AUMHashes[tc.wantAncestor]; got != want {
+				t.Logf("state = %+v", verdict)
+				t.Errorf("lastActiveAncestor = %v, want %v", got, want)
+			}
+
+			for _, name := range tc.wantRetained {
+				h := tc.chain.AUMHashes[name]
+				if v := verdict[h]; v&retainAUMMask == 0 {
+					t.Errorf("AUM %q was not retained: verdict = %v", name, v)
+				}
+			}
+			for _, name := range tc.wantDeleted {
+				h := tc.chain.AUMHashes[name]
+				if v := verdict[h]; v&retainAUMMask != 0 {
+					t.Errorf("AUM %q was retained: verdict = %v", name, v)
+				}
+			}
+
+			if t.Failed() {
+				for name, hash := range tc.chain.AUMHashes {
+					t.Logf("AUM[%q] = %v", name, hash)
+				}
+			}
+		})
+	}
+}
+
+type compactingChonkFake struct {
+	Mem
+
+	aumAge     map[AUMHash]time.Time
+	t          *testing.T
+	wantDelete []AUMHash
+}
+
+func (c *compactingChonkFake) AllAUMs() ([]AUMHash, error) {
+	out := make([]AUMHash, 0, len(c.Mem.aums))
+	for h, _ := range c.Mem.aums {
+		out = append(out, h)
+	}
+	return out, nil
+}
+
+func (c *compactingChonkFake) CommitTime(hash AUMHash) (time.Time, error) {
+	return c.aumAge[hash], nil
+}
+
+func (c *compactingChonkFake) PurgeAUMs(hashes []AUMHash) error {
+	lessHashes := func(a, b AUMHash) bool {
+		return bytes.Compare(a[:], b[:]) < 0
+	}
+	if diff := cmp.Diff(c.wantDelete, hashes, cmpopts.SortSlices(lessHashes)); diff != "" {
+		c.t.Errorf("deletion set differs (-want, +got):\n%s", diff)
+	}
+	return nil
+}
+
+func TestCompact(t *testing.T) {
+	fakeState := &State{
+		Keys:               []Key{{Kind: Key25519, Votes: 1}},
+		DisablementSecrets: [][]byte{bytes.Repeat([]byte{1}, 32)},
+	}
+
+	// A & B are deleted because the new lastActiveAncestor advances beyond them.
+	// OLD is deleted because it does not match retention criteria, and
+	// though it is a descendant of the new lastActiveAncestor (C), it is not a
+	// descendant of a retained AUM.
+	// G, & H are retained as recent (MinChain=2) ancestors of HEAD.
+	// E & F are retained because they are between retained AUMs (G+) and
+	// their newest checkpoint ancestor.
+	// D is retained because it is the newest checkpoint ancestor from
+	// MinChain-retained AUMs.
+	// G2 is retained because it is a descendant of a retained AUM (G).
+	// F1 is retained because it is new enough by wall-clock time.
+	// F2 is retained because it is a descendant of a retained AUM (F1).
+	// C2 is retained because it is between an ancestor checkpoint and
+	// a retained AUM (F1).
+	// C is retained because it is the new lastActiveAncestor. It is the
+	// new lastActiveAncestor because it is the newest common checkpoint
+	// of all retained AUMs.
+	c := newTestchain(t, `
+        A -> B -> C -> C2 -> D -> E -> F -> G -> H
+                       |  -> F1 -> F2       | -> G2
+                       |  -> OLD
+
+        // make {A,B,C,D} compaction candidates
+        A.template = checkpoint
+        B.template = checkpoint
+        C.template = checkpoint
+        D.template = checkpoint
+
+        // tweak seeds of forks so hashes arent identical
+        F1.hashSeed = 1
+        OLD.hashSeed = 2
+        G2.hashSeed = 3
+    `, optTemplate("checkpoint", AUM{MessageKind: AUMCheckpoint, State: fakeState}))
+
+	storage := &compactingChonkFake{
+		Mem:        (*c.Chonk().(*Mem)),
+		aumAge:     map[AUMHash]time.Time{(c.AUMHashes["F1"]): time.Now()},
+		t:          t,
+		wantDelete: []AUMHash{c.AUMHashes["A"], c.AUMHashes["B"], c.AUMHashes["OLD"]},
+	}
+
+	lastActiveAncestor, err := Compact(storage, c.AUMHashes["H"], CompactionOptions{MinChain: 2, MinAge: time.Hour})
+	if err != nil {
+		t.Errorf("Compact() failed: %v", err)
+	}
+	if lastActiveAncestor != c.AUMHashes["C"] {
+		t.Errorf("last active ancestor = %v, want %v", lastActiveAncestor, c.AUMHashes["C"])
+	}
+
+	if t.Failed() {
+		for name, hash := range c.AUMHashes {
+			t.Logf("AUM[%q] = %v", name, hash)
+		}
+	}
+}