Previously there was a mismatch between how nodes store AUMs and what the control plane would offer during sync: - Client compaction: Nodes aggressively compact their TKA state -- they keep the last 24 AUMs, every AUM received in the last two weeks, and then everything from there back to the last checkpoint. Depending on when it compacts, a node may only have ~50 AUMs. - Exponential sampling: To save bandwidth, the control plane would send a SyncOffer containing ancestors at exponentially increasing intervals (4th, 16th, 64th, 256th...). If a node has been offline for too long, the exponential sampling skips the node's smaller window. When the SyncOffer and local state are disjoint, the node cannot find a common ancestor to use for synchronisation. It enters a failure loop where it keeps polling for new TKA state, but it cannot catch up and has an increasingly-outdated view of the tailnet. This patch replaces the exponential sampling with a SyncOffer that sends every checkpoint ancestor of the current HEAD. Since every node is guaranteed to keep at least one checkpoint after compaction, we're more likely to have an intersection for the sync process. This patch also increases `maxSyncHeadIntersectionIter`, which in practice means the control plane will send every checkpoint in the current chain. This means all affected nodes will be able to find an intersection and catch up immediately, without requiring a client update. It's still possible for a node to be unable to sync, but these edge cases become less likely with this change. (For example, if a node is 1000+ AUMs behind, or if it creates a local branch and then compacts away the intersection with the main chain.) This patch includes a regression test with synthetic data, and I verified the fix with customer data. Updates https://github.com/tailscale/corp/issues/40404 Change-Id: I2174011bb23a2b5972f6d1591aadcc016e3cba35 Signed-off-by: Alex Chan <alexc@tailscale.com>
332 lines
9.2 KiB
Go
332 lines
9.2 KiB
Go
// Copyright (c) Tailscale Inc & contributors
|
|
// SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
//go:build !ts_omit_tailnetlock
|
|
|
|
package tka
|
|
|
|
import (
|
|
"bytes"
|
|
"crypto/subtle"
|
|
"errors"
|
|
"fmt"
|
|
|
|
"golang.org/x/crypto/argon2"
|
|
"tailscale.com/types/tkatype"
|
|
"tailscale.com/util/testenv"
|
|
)
|
|
|
|
// ErrNoSuchKey is returned if the key referenced by a KeyID does not exist.
|
|
var ErrNoSuchKey = errors.New("key not found")
|
|
|
|
// State describes Tailnet Key Authority state at an instant in time.
|
|
//
|
|
// State is mutated by applying Authority Update Messages (AUMs), resulting
|
|
// in a new State.
|
|
type State struct {
|
|
// LastAUMHash is the blake2s digest of the last-applied AUM.
|
|
// Because AUMs are strictly ordered and form a hash chain, we
|
|
// check the previous AUM hash in an update we are applying
|
|
// is the same as the LastAUMHash.
|
|
LastAUMHash *AUMHash `cbor:"1,keyasint"`
|
|
|
|
// DisablementValues are KDF-derived values used to verify that a caller
|
|
// possesses a valid DisablementSecret. These values are used during the
|
|
// Tailnet Lock deactivation process.
|
|
//
|
|
// These are safe to share publicly or store in the clear. They cannot be
|
|
// used to derive the original DisablementSecret.
|
|
DisablementValues [][]byte `cbor:"2,keyasint"`
|
|
|
|
// Keys are the public keys of either:
|
|
//
|
|
// 1. The signing nodes currently trusted by the TKA.
|
|
// 2. Ephemeral keys that were used to generate pre-signed auth keys.
|
|
Keys []Key `cbor:"3,keyasint"`
|
|
|
|
// StateID's are nonce's, generated on enablement and fixed for
|
|
// the lifetime of the Tailnet Key Authority. We generate 16-bytes
|
|
// worth of keyspace here just in case we come up with a cool future
|
|
// use for this.
|
|
StateID1 uint64 `cbor:"4,keyasint,omitempty"`
|
|
StateID2 uint64 `cbor:"5,keyasint,omitempty"`
|
|
}
|
|
|
|
// GetKey returns the trusted key with the specified KeyID.
|
|
func (s State) GetKey(key tkatype.KeyID) (Key, error) {
|
|
for _, k := range s.Keys {
|
|
keyID, err := k.ID()
|
|
if err != nil {
|
|
return Key{}, err
|
|
}
|
|
|
|
if bytes.Equal(keyID, key) {
|
|
return k, nil
|
|
}
|
|
}
|
|
|
|
return Key{}, ErrNoSuchKey
|
|
}
|
|
|
|
// Clone makes an independent copy of State.
|
|
//
|
|
// NOTE: There is a difference between a nil slice and an empty
|
|
// slice for encoding purposes, so an implementation of Clone()
|
|
// must take care to preserve this.
|
|
func (s State) Clone() State {
|
|
out := State{
|
|
StateID1: s.StateID1,
|
|
StateID2: s.StateID2,
|
|
}
|
|
|
|
if s.LastAUMHash != nil {
|
|
dupe := *s.LastAUMHash
|
|
out.LastAUMHash = &dupe
|
|
}
|
|
|
|
if s.DisablementValues != nil {
|
|
out.DisablementValues = make([][]byte, len(s.DisablementValues))
|
|
for i := range s.DisablementValues {
|
|
out.DisablementValues[i] = make([]byte, len(s.DisablementValues[i]))
|
|
copy(out.DisablementValues[i], s.DisablementValues[i])
|
|
}
|
|
}
|
|
|
|
if s.Keys != nil {
|
|
out.Keys = make([]Key, len(s.Keys))
|
|
for i := range s.Keys {
|
|
out.Keys[i] = s.Keys[i].Clone()
|
|
}
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
// cloneForUpdate is like Clone, except LastAUMHash is set based
|
|
// on the hash of the given update.
|
|
func (s State) cloneForUpdate(update *AUM) State {
|
|
out := s.Clone()
|
|
aumHash := update.Hash()
|
|
out.LastAUMHash = &aumHash
|
|
return out
|
|
}
|
|
|
|
const disablementLength = 32
|
|
|
|
var disablementSalt = []byte("tailscale network-lock disablement salt")
|
|
|
|
// DisablementKDF computes a public value which can be stored in a
|
|
// key authority, but cannot be reversed to find the input secret.
|
|
//
|
|
// When the output of this function is stored in tka state (i.e. in
|
|
// tka.State.DisablementValues) a call to Authority.ValidDisablement()
|
|
// with the input of this function as the argument will return true.
|
|
func DisablementKDF(secret []byte) []byte {
|
|
// time = 4 (3 recommended, booped to 4 to compensate for less memory)
|
|
// memory = 16 (32 recommended)
|
|
// threads = 4
|
|
// keyLen = 32 (256 bits)
|
|
return argon2.Key(secret, disablementSalt, 4, 16*1024, 4, disablementLength)
|
|
}
|
|
|
|
// checkDisablement returns true for a valid disablement secret.
|
|
func (s State) checkDisablement(secret []byte) bool {
|
|
derived := DisablementKDF(secret)
|
|
for _, candidate := range s.DisablementValues {
|
|
if subtle.ConstantTimeCompare(derived, candidate) == 1 {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// parentMatches returns true if an AUM can chain to (be applied)
|
|
// to the current state.
|
|
//
|
|
// Specifically, the rules are:
|
|
// - The last AUM hash must match (transitively, this implies that this
|
|
// update follows the last update message applied to the state machine)
|
|
// - Or, the state machine knows no parent (it's brand new).
|
|
func (s State) parentMatches(update AUM) bool {
|
|
if s.LastAUMHash == nil {
|
|
return true
|
|
}
|
|
return bytes.Equal(s.LastAUMHash[:], update.PrevAUMHash)
|
|
}
|
|
|
|
// applyVerifiedAUM computes a new state based on the update provided.
|
|
//
|
|
// The provided update MUST be verified: That is, the AUM must be well-formed
|
|
// (as defined by StaticValidate()), and signatures over the AUM must have
|
|
// been verified.
|
|
func (s State) applyVerifiedAUM(update AUM) (State, error) {
|
|
// Validate that the update message has the right parent.
|
|
if !s.parentMatches(update) {
|
|
return State{}, errors.New("parent AUMHash mismatch")
|
|
}
|
|
|
|
switch update.MessageKind {
|
|
case AUMNoOp:
|
|
out := s.cloneForUpdate(&update)
|
|
return out, nil
|
|
|
|
case AUMCheckpoint:
|
|
if update.State == nil {
|
|
return State{}, errors.New("missing checkpoint state")
|
|
}
|
|
id1Match, id2Match := update.State.StateID1 == s.StateID1, update.State.StateID2 == s.StateID2
|
|
if !id1Match || !id2Match {
|
|
return State{}, errors.New("checkpointed state has an incorrect stateID")
|
|
}
|
|
return update.State.cloneForUpdate(&update), nil
|
|
|
|
case AUMAddKey:
|
|
if update.Key == nil {
|
|
return State{}, errors.New("no key to add provided")
|
|
}
|
|
keyID, err := update.Key.ID()
|
|
if err != nil {
|
|
return State{}, err
|
|
}
|
|
if _, err := s.GetKey(keyID); err == nil {
|
|
return State{}, errors.New("key already exists")
|
|
}
|
|
out := s.cloneForUpdate(&update)
|
|
out.Keys = append(out.Keys, *update.Key)
|
|
return out, nil
|
|
|
|
case AUMUpdateKey:
|
|
k, err := s.GetKey(update.KeyID)
|
|
if err != nil {
|
|
return State{}, err
|
|
}
|
|
if update.Votes != nil {
|
|
k.Votes = *update.Votes
|
|
}
|
|
if update.Meta != nil {
|
|
k.Meta = update.Meta
|
|
}
|
|
if err := k.StaticValidate(); err != nil {
|
|
return State{}, fmt.Errorf("updated key fails validation: %v", err)
|
|
}
|
|
out := s.cloneForUpdate(&update)
|
|
for i := range out.Keys {
|
|
keyID, err := out.Keys[i].ID()
|
|
if err != nil {
|
|
return State{}, err
|
|
}
|
|
if bytes.Equal(keyID, update.KeyID) {
|
|
out.Keys[i] = k
|
|
}
|
|
}
|
|
return out, nil
|
|
|
|
case AUMRemoveKey:
|
|
idx := -1
|
|
for i := range s.Keys {
|
|
keyID, err := s.Keys[i].ID()
|
|
if err != nil {
|
|
return State{}, err
|
|
}
|
|
if bytes.Equal(update.KeyID, keyID) {
|
|
idx = i
|
|
break
|
|
}
|
|
}
|
|
if idx < 0 {
|
|
return State{}, ErrNoSuchKey
|
|
}
|
|
out := s.cloneForUpdate(&update)
|
|
out.Keys = append(out.Keys[:idx], out.Keys[idx+1:]...)
|
|
return out, nil
|
|
|
|
default:
|
|
// An AUM with an unknown message kind was received! That means
|
|
// that a future version of tailscaled added some feature we don't
|
|
// understand.
|
|
//
|
|
// The future-compatibility contract for AUM message types is that
|
|
// they must only add new features, not change the semantics of existing
|
|
// mechanisms or features. As such, old clients can safely ignore them.
|
|
out := s.cloneForUpdate(&update)
|
|
return out, nil
|
|
}
|
|
}
|
|
|
|
// staticValidateCheckpoint validates that the state is well-formed for
|
|
// inclusion in a checkpoint AUM.
|
|
func (s *State) staticValidateCheckpoint() error {
|
|
if s.LastAUMHash != nil {
|
|
return errors.New("cannot specify a parent AUM")
|
|
}
|
|
if len(s.DisablementValues) == 0 {
|
|
return errors.New("at least one disablement secret required")
|
|
}
|
|
if numDS := len(s.DisablementValues); numDS > maxDisablementValues {
|
|
return fmt.Errorf("too many disablement values (%d, max %d)", numDS, maxDisablementValues)
|
|
}
|
|
for i, ds := range s.DisablementValues {
|
|
if len(ds) != disablementLength {
|
|
return fmt.Errorf("disablement[%d]: invalid length (got %d, want %d)", i, len(ds), disablementLength)
|
|
}
|
|
for j, ds2 := range s.DisablementValues {
|
|
if i == j {
|
|
continue
|
|
}
|
|
if bytes.Equal(ds, ds2) {
|
|
return fmt.Errorf("disablement[%d]: duplicates disablement[%d]", i, j)
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(s.Keys) == 0 {
|
|
return errors.New("at least one key is required")
|
|
}
|
|
if numKeys := len(s.Keys); numKeys > maxKeys {
|
|
return fmt.Errorf("too many keys (%d, max %d)", numKeys, maxKeys)
|
|
}
|
|
for i, k := range s.Keys {
|
|
if err := k.StaticValidate(); err != nil {
|
|
return fmt.Errorf("key[%d]: %v", i, err)
|
|
}
|
|
}
|
|
// NOTE: The max number of keys is constrained (512), so
|
|
// O(n^2) is fine.
|
|
for i, k := range s.Keys {
|
|
for j, k2 := range s.Keys {
|
|
if i == j {
|
|
continue
|
|
}
|
|
|
|
id1, err := k.ID()
|
|
if err != nil {
|
|
return fmt.Errorf("key[%d]: %w", i, err)
|
|
}
|
|
id2, err := k2.ID()
|
|
if err != nil {
|
|
return fmt.Errorf("key[%d]: %w", j, err)
|
|
}
|
|
|
|
if bytes.Equal(id1, id2) {
|
|
return fmt.Errorf("key[%d]: duplicates key[%d]", i, j)
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CreateStateForTest creates a [State] that marks the given keys as trusted
|
|
// with an arbitrary disablement value.
|
|
//
|
|
// This is only for use in tests, and will panic if called outside a test.
|
|
func CreateStateForTest(keys ...Key) State {
|
|
testenv.AssertInTest()
|
|
|
|
disablementSecret := bytes.Repeat([]byte{0xa5}, 32)
|
|
|
|
return State{
|
|
Keys: keys,
|
|
DisablementValues: [][]byte{DisablementKDF(disablementSecret)},
|
|
}
|
|
}
|