tstest: extend node key expiration integration test.

Can produce the problem in #2515, preparing to test a fix.
Marked as t.Skip() until we have a fix.

Updates https://github.com/tailscale/tailscale/issues/2515

Signed-off-by: Denton Gentry <dgentry@tailscale.com>

tstest/integration/integration_test.go

@@ -84,6 +84,48 @@ func TestOneNodeUp_NoAuth(t *testing.T) {
 	t.Logf("number of HTTP logcatcher requests: %v", env.LogCatcher.numRequests())
 }
 
+func TestOneNodeExpiredKey(t *testing.T) {
+	t.Skip("Test to exercise a problem which is not fixed yet.")
+	t.Parallel()
+	bins := BuildTestBinaries(t)
+
+	env := newTestEnv(t, bins)
+	defer env.Close()
+
+	n1 := newTestNode(t, env)
+
+	d1 := n1.StartDaemon(t)
+	defer d1.Kill()
+	n1.AwaitResponding(t)
+	n1.MustUp()
+	n1.AwaitRunning(t)
+
+	nodes := env.Control.AllNodes()
+	if len(nodes) != 1 {
+		t.Fatalf("expected 1 node, got %d nodes", len(nodes))
+	}
+
+	nodeKey := nodes[0].Key
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	env.Control.SetExpireAllNodes(true)
+	n1.AwaitNeedsLogin(t)
+	ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second)
+	if err := env.Control.AwaitNodeInMapRequest(ctx, nodeKey); err != nil {
+		t.Fatal(err)
+	}
+	cancel()
+
+	env.Control.SetExpireAllNodes(false)
+	n1.AwaitRunning(t)
+
+	d1.MustCleanShutdown(t)
+}
+
 func TestCollectPanic(t *testing.T) {
 	t.Parallel()
 	bins := BuildTestBinaries(t)
@@ -780,6 +822,23 @@ func (n *testNode) AwaitRunning(t testing.TB) {
 	}
 }
 
+// AwaitNeedsLogin waits for n to reach the IPN state "NeedsLogin".
+func (n *testNode) AwaitNeedsLogin(t testing.TB) {
+	t.Helper()
+	if err := tstest.WaitFor(20*time.Second, func() error {
+		st, err := n.Status()
+		if err != nil {
+			return err
+		}
+		if st.BackendState != "NeedsLogin" {
+			return fmt.Errorf("in state %q", st.BackendState)
+		}
+		return nil
+	}); err != nil {
+		t.Fatalf("failure/timeout waiting for transition to NeedsLogin status: %v", err)
+	}
+}
+
 // Tailscale returns a command that runs the tailscale CLI with the provided arguments.
 // It does not start the process.
 func (n *testNode) Tailscale(arg ...string) *exec.Cmd {

tstest/integration/testcontrol/testcontrol.go

@@ -65,6 +65,7 @@ type Server struct {
 	authPath      map[string]*AuthPath
 	nodeKeyAuthed map[key.NodePublic]bool // key => true once authenticated
 	pingReqsToAdd map[key.NodePublic]*tailcfg.PingRequest
+	allExpired    bool // All nodes will be told their node key is expired.
 }
 
 // BaseURL returns the server's base URL, without trailing slash.
@@ -153,6 +154,17 @@ func (s *Server) AddPingRequest(nodeKeyDst key.NodePublic, pr *tailcfg.PingReque
 	return sendUpdate(oldUpdatesCh, updateDebugInjection)
 }
 
+// Mark the Node key of every node as expired
+func (s *Server) SetExpireAllNodes(expired bool) {
+	s.mu.Lock()
+	s.allExpired = expired
+	s.mu.Unlock()
+
+	for _, node := range s.AllNodes() {
+		sendUpdate(s.updates[node.ID], updateSelfChanged)
+	}
+}
+
 type AuthPath struct {
 	nodeKey key.NodePublic
 
@@ -467,6 +479,7 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 	if requireAuth && s.nodeKeyAuthed[nk] {
 		requireAuth = false
 	}
+	allExpired := s.allExpired
 	s.mu.Unlock()
 
 	authURL := ""
@@ -481,7 +494,7 @@ func (s *Server) serveRegister(w http.ResponseWriter, r *http.Request, mkey key.
 	res, err := s.encode(mkey, false, tailcfg.RegisterResponse{
 		User:              *user,
 		Login:             *login,
-		NodeKeyExpired:    false,
+		NodeKeyExpired:    allExpired,
 		MachineAuthorized: machineAuthorized,
 		AuthURL:           authURL,
 	})
@@ -642,6 +655,13 @@ func (s *Server) serveMap(w http.ResponseWriter, r *http.Request, mkey key.Machi
 		if res == nil {
 			return // done
 		}
+
+		s.mu.Lock()
+		allExpired := s.allExpired
+		s.mu.Unlock()
+		if allExpired {
+			res.Node.KeyExpiry = time.Now().Add(-1 * time.Minute)
+		}
 		// TODO: add minner if/when needed
 		resBytes, err := json.Marshal(res)
 		if err != nil {