tsconsensus: enable writing state to disk

The comments in the raft code say to only use the InMemStore for tests.

Updates #16027

Signed-off-by: Fran Bull <fran@tailscale.com>

go.mod

@@ -51,6 +51,7 @@ require (
 	github.com/goreleaser/nfpm/v2 v2.33.1
 	github.com/hashicorp/go-hclog v1.6.2
 	github.com/hashicorp/raft v1.7.2
+	github.com/hashicorp/raft-boltdb/v2 v2.3.1
 	github.com/hdevalence/ed25519consensus v0.2.0
 	github.com/illarion/gonotify/v3 v3.0.2
 	github.com/inetaf/tcpproxy v0.0.0-20250203165043-ded522cbd03f
@@ -135,6 +136,7 @@ require (
 	github.com/alecthomas/go-check-sumtype v0.1.4 // indirect
 	github.com/alexkohler/nakedret/v2 v2.0.4 // indirect
 	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bombsimon/wsl/v4 v4.2.1 // indirect
 	github.com/butuzov/mirror v1.1.0 // indirect
 	github.com/catenacyber/perfsprint v0.7.1 // indirect
@@ -166,6 +168,7 @@ require (
 	github.com/ykadowak/zerologlint v0.1.5 // indirect
 	go-simpler.org/musttag v0.9.0 // indirect
 	go-simpler.org/sloglint v0.5.0 // indirect
+	go.etcd.io/bbolt v1.3.11 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
 	go.opentelemetry.io/otel v1.33.0 // indirect

go.sum

@@ -180,6 +180,8 @@ github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb h1:m935MPodAbYS46DG4
 github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI=
 github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M=
 github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k=
+github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
+github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bombsimon/wsl/v4 v4.2.1 h1:Cxg6u+XDWff75SIFFmNsqnIOgob+Q9hG6y/ioKbRFiM=
 github.com/bombsimon/wsl/v4 v4.2.1/go.mod h1:Xu/kDxGZTofQcDGCtQe9KCzhHphIe0fDuyWTxER9Feo=
 github.com/bramvdbogaerde/go-scp v1.4.0 h1:jKMwpwCbcX1KyvDbm/PDJuXcMuNVlLGi0Q0reuzjyKY=
@@ -555,6 +557,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ
 github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-metrics v0.5.4 h1:8mmPiIJkTPPEbAiV97IxdAGNdRdaWwVap1BU6elejKY=
 github.com/hashicorp/go-metrics v0.5.4/go.mod h1:CG5yz4NZ/AI/aQt9Ucm/vdBnbh7fvmv4lxZ350i+QQI=
+github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
+github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-msgpack/v2 v2.1.2 h1:4Ee8FTp834e+ewB71RDrQ0VKpyFdrKOjvYtnQ/ltVj0=
 github.com/hashicorp/go-msgpack/v2 v2.1.2/go.mod h1:upybraOAblm4S7rx0+jeNy+CWWhzywQsSRV5033mMu4=
 github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
@@ -571,6 +575,10 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hashicorp/raft v1.7.2 h1:pyvxhfJ4R8VIAlHKvLoKQWElZspsCVT6YWuxVxsPAgc=
 github.com/hashicorp/raft v1.7.2/go.mod h1:DfvCGFxpAUPE0L4Uc8JLlTPtc3GzSbdH0MTJCLgnmJQ=
+github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702 h1:RLKEcCuKcZ+qp2VlaaZsYZfLOmIiuJNpEi48Rl8u9cQ=
+github.com/hashicorp/raft-boltdb v0.0.0-20230125174641-2a8082862702/go.mod h1:nTakvJ4XYq45UXtn0DbwR4aU9ZdjlnIenpbs6Cd+FM0=
+github.com/hashicorp/raft-boltdb/v2 v2.3.1 h1:ackhdCNPKblmOhjEU9+4lHSJYFkJd6Jqyvj6eW9pwkc=
+github.com/hashicorp/raft-boltdb/v2 v2.3.1/go.mod h1:n4S+g43dXF1tqDT+yzcXHhXM6y7MrlUd3TTwGRcUvQE=
 github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU=
 github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
@@ -1046,6 +1054,8 @@ go-simpler.org/musttag v0.9.0 h1:Dzt6/tyP9ONr5g9h9P3cnYWCxeBFRkd0uJL/w+1Mxos=
 go-simpler.org/musttag v0.9.0/go.mod h1:gA9nThnalvNSKpEoyp3Ko4/vCX2xTpqKoUtNqXOnVR4=
 go-simpler.org/sloglint v0.5.0 h1:2YCcd+YMuYpuqthCgubcF5lBSjb6berc5VMOYUHKrpY=
 go-simpler.org/sloglint v0.5.0/go.mod h1:EUknX5s8iXqf18KQxKnaBHUPVriiPnOrPjjJcsaTcSQ=
+go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0=
+go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=

tsconsensus/bolt_store.go

@@ -0,0 +1,19 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build !loong64
+
+package tsconsensus
+
+import (
+	"github.com/hashicorp/raft"
+	raftboltdb "github.com/hashicorp/raft-boltdb/v2"
+)
+
+func boltStore(path string) (raft.StableStore, raft.LogStore, error) {
+	store, err := raftboltdb.NewBoltStore(path)
+	if err != nil {
+		return nil, nil, err
+	}
+	return store, store, nil
+}

tsconsensus/bolt_store_no_bolt.go

@@ -0,0 +1,18 @@
+// Copyright (c) Tailscale Inc & AUTHORS
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build loong64
+
+package tsconsensus
+
+import (
+	"errors"
+
+	"github.com/hashicorp/raft"
+)
+
+func boltStore(path string) (raft.StableStore, raft.LogStore, error) {
+	// "github.com/hashicorp/raft-boltdb/v2" doesn't build on loong64
+	// see https://github.com/hashicorp/raft-boltdb/issues/27
+	return nil, nil, errors.New("not implemented")
+}

tsconsensus/tsconsensus.go

@@ -32,6 +32,7 @@ import (
 	"net"
 	"net/http"
 	"net/netip"
+	"path/filepath"
 	"time"
 
 	"github.com/hashicorp/go-hclog"
@@ -71,6 +72,7 @@ type Config struct {
 	MaxConnPool       int
 	ConnTimeout       time.Duration
 	ServeDebugMonitor bool
+	StateDirPath      string
 }
 
 // DefaultConfig returns a Config populated with default values ready for use.
@@ -223,10 +225,31 @@ func Start(ctx context.Context, ts *tsnet.Server, fsm raft.FSM, clusterTag strin
 func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, self selfRaftNode, auth *authorization, cfg Config) (*raft.Raft, error) {
 	cfg.Raft.LocalID = raft.ServerID(self.id)
 
-	// no persistence (for now?)
-	logStore := raft.NewInmemStore()
-	stableStore := raft.NewInmemStore()
-	snapshots := raft.NewInmemSnapshotStore()
+	var logStore raft.LogStore
+	var stableStore raft.StableStore
+	var snapStore raft.SnapshotStore
+
+	if cfg.StateDirPath == "" {
+		// comments in raft code say to only use for tests
+		logStore = raft.NewInmemStore()
+		stableStore = raft.NewInmemStore()
+		snapStore = raft.NewInmemSnapshotStore()
+	} else {
+		var err error
+		stableStore, logStore, err = boltStore(filepath.Join(cfg.StateDirPath, "store"))
+		if err != nil {
+			return nil, err
+		}
+		snaplogger := hclog.New(&hclog.LoggerOptions{
+			Name:   "raft-snap",
+			Output: cfg.Raft.LogOutput,
+			Level:  hclog.LevelFromString(cfg.Raft.LogLevel),
+		})
+		snapStore, err = raft.NewFileSnapshotStoreWithLogger(filepath.Join(cfg.StateDirPath, "snapstore"), 2, snaplogger)
+		if err != nil {
+			return nil, err
+		}
+	}
 
 	// opens the listener on the raft port, raft will close it when it thinks it's appropriate
 	ln, err := ts.Listen("tcp", raftAddr(self.hostAddr, cfg))
@@ -234,7 +257,7 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel
 		return nil, err
 	}
 
-	logger := hclog.New(&hclog.LoggerOptions{
+	transportLogger := hclog.New(&hclog.LoggerOptions{
 		Name:   "raft-net",
 		Output: cfg.Raft.LogOutput,
 		Level:  hclog.LevelFromString(cfg.Raft.LogLevel),
@@ -248,9 +271,9 @@ func startRaft(shutdownCtx context.Context, ts *tsnet.Server, fsm *raft.FSM, sel
 	},
 		cfg.MaxConnPool,
 		cfg.ConnTimeout,
-		logger)
+		transportLogger)
 
-	return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapshots, transport)
+	return raft.NewRaft(cfg.Raft, *fsm, logStore, stableStore, snapStore, transport)
 }
 
 // A Consensus is the consensus algorithm for a tsnet.Server