tstest/integration: build binaries only once

The existing code relied on the Go build cache to avoid
needless work when obtaining the tailscale binaries.

For non-obvious reasons, the binaries were getting re-linked
every time, which added 600ms or so on my machine to every test.

Instead, build the binaries exactly once, on demand.
This reduces the time to run 'go test -count=5' from 34s to 10s
on my machine.

Signed-off-by: Josh Bleecher Snyder <josh@tailscale.com>

tstest/integration/integration.go

@@ -43,38 +43,68 @@ import (
 	"tailscale.com/version"
 )
 
-// Binaries are the paths to a tailscaled and tailscale binary.
-// These can be shared by multiple nodes.
-type Binaries struct {
-	Dir    string // temp dir for tailscale & tailscaled
-	Daemon string // tailscaled
-	CLI    string // tailscale
-}
-
-// BuildTestBinaries builds tailscale and tailscaled, failing the test
-// if they fail to compile.
-func BuildTestBinaries(t testing.TB) *Binaries {
-	td := t.TempDir()
-	build(t, td, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
-	return &Binaries{
-		Dir:    td,
-		Daemon: filepath.Join(td, "tailscaled"+exe()),
-		CLI:    filepath.Join(td, "tailscale"+exe()),
+// CleanupBinaries cleans up any resources created by calls to BinaryDir, TailscaleBinary, or TailscaledBinary.
+// It should be called from TestMain after all tests have completed.
+func CleanupBinaries() {
+	buildOnce.Do(func() {})
+	if binDir != "" {
+		os.RemoveAll(binDir)
 	}
 }
 
-// buildMu limits our use of "go build" to one at a time, so we don't
-// fight Go's built-in caching trying to do the same build concurrently.
-var buildMu sync.Mutex
+// BinaryDir returns a directory containing test tailscale and tailscaled binaries.
+// If any test calls BinaryDir, there must be a TestMain function that calls
+// CleanupBinaries after all tests are complete.
+func BinaryDir(tb testing.TB) string {
+	buildOnce.Do(func() {
+		binDir, buildErr = buildTestBinaries()
+	})
+	if buildErr != nil {
+		tb.Fatal(buildErr)
+	}
+	return binDir
+}
 
-func build(t testing.TB, outDir string, targets ...string) {
-	buildMu.Lock()
-	defer buildMu.Unlock()
+// TailscaleBinary returns the path to the test tailscale binary.
+// If any test calls TailscaleBinary, there must be a TestMain function that calls
+// CleanupBinaries after all tests are complete.
+func TailscaleBinary(tb testing.TB) string {
+	return filepath.Join(BinaryDir(tb), "tailscale"+exe())
+}
 
-	t0 := time.Now()
-	defer func() { t.Logf("built %s in %v", targets, time.Since(t0).Round(time.Millisecond)) }()
+// TailscaledBinary returns the path to the test tailscaled binary.
+// If any test calls TailscaleBinary, there must be a TestMain function that calls
+// CleanupBinaries after all tests are complete.
+func TailscaledBinary(tb testing.TB) string {
+	return filepath.Join(BinaryDir(tb), "tailscaled"+exe())
+}
 
-	goBin := findGo(t)
+var (
+	buildOnce sync.Once
+	buildErr  error
+	binDir    string
+)
+
+// buildTestBinaries builds tailscale and tailscaled.
+// It returns the dir containing the binaries.
+func buildTestBinaries() (string, error) {
+	bindir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return "", err
+	}
+	err = build(bindir, "tailscale.com/cmd/tailscaled", "tailscale.com/cmd/tailscale")
+	if err != nil {
+		os.RemoveAll(bindir)
+		return "", err
+	}
+	return bindir, nil
+}
+
+func build(outDir string, targets ...string) error {
+	goBin, err := findGo()
+	if err != nil {
+		return err
+	}
 	cmd := exec.Command(goBin, "install")
 	if version.IsRace() {
 		cmd.Args = append(cmd.Args, "-race")
@@ -83,7 +113,7 @@ func build(t testing.TB, outDir string, targets ...string) {
 	cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH, "GOBIN="+outDir)
 	errOut, err := cmd.CombinedOutput()
 	if err == nil {
-		return
+		return nil
 	}
 	if strings.Contains(string(errOut), "when GOBIN is set") {
 		// Fallback slow path for cross-compiled binaries.
@@ -92,25 +122,25 @@ func build(t testing.TB, outDir string, targets ...string) {
 			cmd := exec.Command(goBin, "build", "-o", outFile, target)
 			cmd.Env = append(os.Environ(), "GOARCH="+runtime.GOARCH)
 			if errOut, err := cmd.CombinedOutput(); err != nil {
-				t.Fatalf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
+				return fmt.Errorf("failed to build %v with %v: %v, %s", target, goBin, err, errOut)
 			}
 		}
-		return
+		return nil
 	}
-	t.Fatalf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
+	return fmt.Errorf("failed to build %v with %v: %v, %s", targets, goBin, err, errOut)
 }
 
-func findGo(t testing.TB) string {
+func findGo() (string, error) {
 	goBin := filepath.Join(runtime.GOROOT(), "bin", "go"+exe())
 	if fi, err := os.Stat(goBin); err != nil {
 		if os.IsNotExist(err) {
-			t.Fatalf("failed to find go at %v", goBin)
+			return "", fmt.Errorf("failed to find go at %v", goBin)
 		}
-		t.Fatalf("looking for go binary: %v", err)
+		return "", fmt.Errorf("looking for go binary: %v", err)
 	} else if !fi.Mode().IsRegular() {
-		t.Fatalf("%v is unexpected %v", goBin, fi.Mode())
+		return "", fmt.Errorf("%v is unexpected %v", goBin, fi.Mode())
 	}
-	return goBin
+	return goBin, nil
 }
 
 func exe() string {