cmd/testwrapper: exit code 1 when go build fails (#9276)

Fixes #9275
Fixes #8586
Fixes tailscale/corp#13115

Signed-off-by: Paul Scott <paul@tailscale.com>

cmd/testwrapper/testwrapper.go

@@ -8,6 +8,7 @@
 package main
 
 import (
+	"bufio"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -59,9 +60,10 @@ var debug = os.Getenv("TS_TESTWRAPPER_DEBUG") != ""
 
 // runTests runs the tests in pt and sends the results on ch. It sends a
 // testAttempt for each test and a final testAttempt per pkg with pkgFinished
-// set to true.
+// set to true. Package build errors will not emit a testAttempt (as no valid
+// JSON is produced) but the [os/exec.ExitError] will be returned.
 // It calls close(ch) when it's done.
-func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string, ch chan<- *testAttempt) {
+func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []string, ch chan<- *testAttempt) error {
 	defer close(ch)
 	args := []string{"test", "-json", pt.pattern}
 	args = append(args, otherArgs...)
@@ -87,17 +89,17 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
 		log.Printf("error starting test: %v", err)
 		os.Exit(1)
 	}
-	done := make(chan struct{})
+	cmdErr := make(chan error, 1)
 	go func() {
-		defer close(done)
-		cmd.Wait()
+		defer close(cmdErr)
+		cmdErr <- cmd.Wait()
 	}()
 
-	jd := json.NewDecoder(r)
+	s := bufio.NewScanner(r)
 	resultMap := make(map[string]map[string]*testAttempt) // pkg -> test -> testAttempt
-	for {
+	for s.Scan() {
 		var goOutput goTestOutput
-		if err := jd.Decode(&goOutput); err != nil {
+		if err := json.Unmarshal(s.Bytes(), &goOutput); err != nil {
 			if errors.Is(err, io.EOF) || errors.Is(err, os.ErrClosed) {
 				break
 			}
@@ -107,7 +109,7 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
 			// The build error will be printed to stderr.
 			// See: https://github.com/golang/go/issues/35169
 			if _, ok := err.(*json.SyntaxError); ok {
-				jd = json.NewDecoder(r)
+				fmt.Println(s.Text())
 				continue
 			}
 			panic(err)
@@ -162,7 +164,10 @@ func runTests(ctx context.Context, attempt int, pt *packageTests, otherArgs []st
 			}
 		}
 	}
-	<-done
+	if err := s.Err(); err != nil {
+		return err
+	}
+	return <-cmdErr
 }
 
 func main() {
@@ -244,12 +249,24 @@ func main() {
 			fmt.Printf("\n\nAttempt #%d: Retrying flaky tests:\n\n", thisRun.attempt)
 		}
 
-		failed := false
 		toRetry := make(map[string][]string) // pkg -> tests to retry
 		for _, pt := range thisRun.tests {
 			ch := make(chan *testAttempt)
-			go runTests(ctx, thisRun.attempt, pt, otherArgs, ch)
+			runErr := make(chan error, 1)
+			go func() {
+				defer close(runErr)
+				runErr <- runTests(ctx, thisRun.attempt, pt, otherArgs, ch)
+			}()
+
+			var failed bool
 			for tr := range ch {
+				// Go assigns the package name "command-line-arguments" when you
+				// `go test FILE` rather than `go test PKG`. It's more
+				// convenient for us to to specify files in tests, so fix tr.pkg
+				// so that subsequent testwrapper attempts run correctly.
+				if tr.pkg == "command-line-arguments" {
+					tr.pkg = pattern
+				}
 				if tr.pkgFinished {
 					if tr.outcome == "fail" && len(toRetry[tr.pkg]) == 0 {
 						// If a package fails and we don't have any tests to
@@ -272,10 +289,21 @@ func main() {
 					failed = true
 				}
 			}
-		}
-		if failed {
-			fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
-			os.Exit(1)
+			if failed {
+				fmt.Println("\n\nNot retrying flaky tests because non-flaky tests failed.")
+				os.Exit(1)
+			}
+
+			// If there's nothing to retry and no non-retryable tests have
+			// failed then we've probably hit a build error.
+			if err := <-runErr; len(toRetry) == 0 && err != nil {
+				var exit *exec.ExitError
+				if errors.As(err, &exit) {
+					if code := exit.ExitCode(); code > -1 {
+						os.Exit(exit.ExitCode())
+					}
+				}
+			}
 		}
 		if len(toRetry) == 0 {
 			continue