cmd/natc: use new on disk state store for consensus

Fixes #16027

Signed-off-by: Fran Bull <fran@tailscale.com>

cmd/natc/ippool/consensusippool.go

@@ -10,6 +10,8 @@ import (
 	"fmt"
 	"log"
 	"net/netip"
+	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/hashicorp/raft"
@@ -150,9 +152,14 @@ func (ipp *ConsensusIPPool) domainLookup(from tailcfg.NodeID, addr netip.Addr) (
 }
 
 // StartConsensus is part of the IPPool interface. It starts the raft background routines that handle consensus.
-func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string) error {
+func (ipp *ConsensusIPPool) StartConsensus(ctx context.Context, ts *tsnet.Server, clusterTag string, clusterStateDir string) error {
 	cfg := tsconsensus.DefaultConfig()
 	cfg.ServeDebugMonitor = true
+	var err error
+	cfg.StateDirPath, err = getStatePath(clusterStateDir)
+	if err != nil {
+		return err
+	}
 	cns, err := tsconsensus.Start(ctx, ts, ipp, clusterTag, cfg)
 	if err != nil {
 		return err
@@ -204,6 +211,30 @@ func (ps *consensusPerPeerState) unusedIPV4(ipset *netipx.IPSet, reuseDeadline t
 	return netip.Addr{}, false, "", errors.New("ip pool exhausted")
 }
 
+func getStatePath(pathFromFlag string) (string, error) {
+	var dirPath string
+	if pathFromFlag != "" {
+		dirPath = pathFromFlag
+	} else {
+		confDir, err := os.UserConfigDir()
+		if err != nil {
+			return "", err
+		}
+		dirPath = filepath.Join(confDir, "nat-connector-cluster-state")
+	}
+
+	if err := os.MkdirAll(dirPath, 0700); err != nil {
+		return "", err
+	}
+	if fi, err := os.Stat(dirPath); err != nil {
+		return "", err
+	} else if !fi.IsDir() {
+		return "", fmt.Errorf("%v is not a directory", dirPath)
+	}
+
+	return dirPath, nil
+}
+
 // isCloseToExpiry returns true if the lastUsed and now times are more than
 // half the lifetime apart
 func isCloseToExpiry(lastUsed, now time.Time, lifetime time.Duration) bool {

cmd/natc/natc.go

@@ -59,6 +59,7 @@ func main() {
 		wgPort          = fs.Uint("wg-port", 0, "udp port for wireguard and peer to peer traffic")
 		clusterTag      = fs.String("cluster-tag", "", "optionally run in a consensus cluster with other nodes with this tag")
 		server          = fs.String("login-server", ipn.DefaultControlURL, "the base URL of control server")
+		clusterStateDir = fs.String("cluster-state-dir", "", "path to directory in which to store raft state")
 	)
 	ff.Parse(fs, os.Args[1:], ff.WithEnvVarPrefix("TS_NATC"))
 
@@ -155,7 +156,7 @@ func main() {
 	var ipp ippool.IPPool
 	if *clusterTag != "" {
 		cipp := ippool.NewConsensusIPPool(addrPool)
-		err = cipp.StartConsensus(ctx, ts, *clusterTag)
+		err = cipp.StartConsensus(ctx, ts, *clusterTag, *clusterStateDir)
 		if err != nil {
 			log.Fatalf("StartConsensus: %v", err)
 		}