webnet/tailscale
1
0
Fork 0
Code Issues 1 Pull Requests 1 Packages Projects Releases Wiki Activity

tstest/natlab/vmtest: retry SSHExec on transient SSH failures

Browse Source 
Add a retry loop with BatchMode=yes to absorb the race window
between Env.Start() returning (when tta reports the tailscale
backend as Running) and cloud-init finishing the user/SSH-key
setup. In CI, the second VM's tta agent has been observed
connecting only a few hundred milliseconds before the test SSHes
in, which is inside the window where /root/.ssh/authorized_keys
hasn't fully landed yet. SSH key auth then fails and ssh(1) falls
back to interactive password prompts (3x), wasting time and
producing a confusing "Permission denied (publickey,password)"
error.

BatchMode=yes makes the client fail fast on auth failure instead
of prompting, and the retry loop handles SSH transport-level
errors (exit code 255) for up to 30 seconds with 500ms backoff.
Remote command non-zero exits still pass through unchanged.

Fixes #20228

Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>
Change-Id: I17f7422e9e27bf7b995f505c0184cbb2b230ed81
This commit is contained in:
Brad Fitzpatrick
2026-06-24 09:22:28 -07:00
committed by Brad Fitzpatrick
parent 281404e9e3
commit 0bc0cb8131
1 changed files with 33 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tstest/natlab/vmtest/vmtest.go
+33 -11
View File
@@ -20,6 +20,7 @@ import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
@@ -1302,21 +1303,42 @@ func (e *Env) AddRoute(n *Node, prefix, via string) {
// SSHExec runs a command on a cloud VM via its debug SSH NIC.
// Only works for cloud VMs that have the debug NIC and SSH key configured.
// Returns stdout and any error.
//
// SSH transport-level errors (exit code 255: connection refused, auth
// failure, etc.) are retried for up to ~30s to absorb the race window
// between Env.Start() returning (when tta reports the tailscale backend
// as Running) and cloud-init finishing the user/SSH-key setup. The remote
// command's own non-zero exit codes are returned to the caller without
// retry.
func (e *Env) SSHExec(n *Node, cmd string) (string, error) {
if n.sshPort == 0 {
return "", fmt.Errorf("node %s has no SSH debug port", n.name)
}
sshCmd := exec.Command("ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=5",
"-o", "LogLevel=ERROR",
"-i", "/tmp/vmtest_key",
"-p", fmt.Sprintf("%d", n.sshPort),
"root@127.0.0.1",
cmd)
out, err := sshCmd.CombinedOutput()
return string(out), err
deadline := time.Now().Add(30 * time.Second)
for {
sshCmd := exec.Command("ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "BatchMode=yes",
"-o", "ConnectTimeout=5",
"-o", "LogLevel=ERROR",
"-i", "/tmp/vmtest_key",
"-p", fmt.Sprintf("%d", n.sshPort),
"root@127.0.0.1",
cmd)
out, err := sshCmd.CombinedOutput()
if err == nil {
return string(out), nil
}
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) || exitErr.ExitCode() != 255 {
return string(out), err
}
if time.Now().After(deadline) {
return string(out), err
}
time.Sleep(500 * time.Millisecond)
}
}
// DumpStatus logs the tailscale status of a node, including its peers and their