cmd/testwrapper: surface race reports and skip retries when detected

A data race in a package matters more than any individual test
result. Two related problems:

1. Where go test's race detector text ("WARNING: DATA RACE" plus
   the goroutine stack traces) lands in JSON output is timing-
   dependent: it can be attributed to a test that ends up reporting
   PASS (e.g. when the racing goroutines outlive the test that
   spawned them and TSan prints during a different test's window).
   testwrapper's main loop only flushes the logs of failed tests,
   so the race report ends up stuck in a passing test's buffer and
   is silently dropped. The race builders just see a bare
   "FAIL\nFAIL\tpkg\ttime".

2. If the failing test in such a package happens to be marked flaky,
   testwrapper retries it. That is the worst possible response to a
   race: the flaky test might not even be the racy code, and a
   second run without the racy goroutines could "succeed" while
   hiding the real bug.

Address both: scan every output line for the race detector's first-
line marker. Track whether the package observed a race at all, on
the pkgFinished testAttempt. When a race was seen, fold every per-
test log buffer into the package-level logs (so the full report
surfaces from the existing pkg-fail flush path), and drop any
flaky-test retry plans for that package so we fail immediately
instead of running another attempt.

Two new tests:
- TestRaceSuppressesFlakyRetry verifies that a flaky test alongside
  a racy test does NOT get retried.
- TestRaceAttributedToPassingTest verifies that a race attributed by
  test2json to a passing test still surfaces in the output.

Also add a corpus of captured raw test binary outputs under
cmd/testwrapper/testdata/, with one subdirectory per scenario,
documenting the six representative shapes that go test -race can
emit (race in test body, race in goroutines that outlive a test,
race forced into a later test, race in TestMain post-m.Run, and a
parallel-tests split-attribution case via a "=== NAME" redirect
line). See its README.md for details.

Fixes #19603

Change-Id: Ifbfcd67fb3b1882c4907bd9cb2d68a8b5a91dd54
Signed-off-by: Brad Fitzpatrick <bradfitz@tailscale.com>

cmd/testwrapper/testwrapper_test.go

@@ -145,6 +145,118 @@ func TestAlwaysError(t *testing.T) {
 	}
 }
 
+// TestRaceSuppressesFlakyRetry verifies that detecting a data race
+// in a package's output stops testwrapper from retrying any flaky
+// test in that package. Races are too serious to paper over: the
+// flaky test might not even be the one whose code is racy, and a
+// retry without the racy goroutine could silently hide the bug.
+func TestRaceSuppressesFlakyRetry(t *testing.T) {
+	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
+		t.Skip("test requires the race detector, which needs linux/amd64")
+	}
+	t.Parallel()
+
+	testfile := filepath.Join(t.TempDir(), "raceretry_test.go")
+	code := []byte(`package raceretry_test
+
+import (
+	"sync"
+	"testing"
+	"tailscale.com/cmd/testwrapper/flakytest"
+)
+
+var counter int
+
+func TestRace(t *testing.T) {
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() { defer wg.Done(); counter++ }()
+	go func() { defer wg.Done(); counter++ }()
+	wg.Wait()
+}
+
+func TestFlaky(t *testing.T) {
+	flakytest.Mark(t, "https://github.com/tailscale/tailscale/issues/0")
+	t.Fatal("flaky test failing; would normally be retried")
+}
+`)
+	if err := os.WriteFile(testfile, code, 0o644); err != nil {
+		t.Fatalf("writing package: %s", err)
+	}
+
+	out, err := cmdTestwrapper(t, testfile, "-race").CombinedOutput()
+	if code, ok := errExitCode(err); !ok || code != 1 {
+		t.Fatalf("testwrapper %s -race: expected exit code 1, got %v; output was:\n%s", testfile, err, out)
+	}
+	if want := "WARNING: DATA RACE"; !bytes.Contains(out, []byte(want)) {
+		t.Fatalf("expected race report in output, got:\n%s", out)
+	}
+	if want := "Not retrying flaky tests"; !bytes.Contains(out, []byte(want)) {
+		t.Fatalf("expected no-retry message in output, got:\n%s", out)
+	}
+	if got := bytes.Count(out, []byte("Attempt #")); got != 0 {
+		t.Fatalf("expected no retry attempts to be made, but %d were:\n%s", got, out)
+	}
+	if got := bytes.Count(out, []byte("=== RUN   TestFlaky")); got != 1 {
+		t.Fatalf("expected TestFlaky to be run exactly once, ran %d times:\n%s", got, out)
+	}
+
+	if testing.Verbose() {
+		t.Logf("success - output:\n%s", out)
+	}
+}
+
+// TestRaceAttributedToPassingTest covers the case where go test
+// attributes a data race report to a test that itself reports PASS
+// (e.g. when the racing goroutines outlive the test that spawned
+// them and TSan prints during a different test's window). Without
+// the race-detection fix, the WARNING: DATA RACE block would be
+// stuck in a passing test's log buffer and dropped on the floor.
+// See https://github.com/tailscale/tailscale/issues/19603.
+func TestRaceAttributedToPassingTest(t *testing.T) {
+	if runtime.GOOS != "linux" || runtime.GOARCH != "amd64" {
+		t.Skip("test requires the race detector, which needs linux/amd64")
+	}
+	t.Parallel()
+
+	testfile := filepath.Join(t.TempDir(), "race_test.go")
+	code := []byte(`package race_test
+
+import (
+	"sync"
+	"testing"
+)
+
+var counter int
+var wg sync.WaitGroup
+
+func TestSpawn(t *testing.T) {
+	wg.Add(2)
+	go func() { defer wg.Done(); counter++ }()
+	go func() { defer wg.Done(); counter++ }()
+}
+
+func TestWait(t *testing.T) {
+	wg.Wait()
+}
+`)
+	if err := os.WriteFile(testfile, code, 0o644); err != nil {
+		t.Fatalf("writing package: %s", err)
+	}
+
+	out, err := cmdTestwrapper(t, testfile, "-race").CombinedOutput()
+	if code, ok := errExitCode(err); !ok || code != 1 {
+		t.Fatalf("testwrapper %s -race: expected exit code 1, got %v; output was:\n%s", testfile, err, out)
+	}
+	if want := "WARNING: DATA RACE"; !bytes.Contains(out, []byte(want)) {
+		t.Fatalf("expected race report in output, got:\n%s", out)
+	}
+
+	if testing.Verbose() {
+		t.Logf("success - output:\n%s", out)
+	}
+}
+
 func TestBuildError(t *testing.T) {
 	t.Parallel()