diff --git a/cmd/testwrapper/testwrapper.go b/cmd/testwrapper/testwrapper.go
index d392a070c..3c8661e46 100644
--- a/cmd/testwrapper/testwrapper.go
+++ b/cmd/testwrapper/testwrapper.go
@@ -247,6 +247,12 @@ func main() {
 			go runTests(ctx, thisRun.attempt, pt, otherArgs, ch)
 			for tr := range ch {
 				if tr.pkgFinished {
+					if tr.outcome == "fail" && len(toRetry[tr.name.pkg]) == 0 {
+						// If a package fails and we don't have any tests to
+						// retry, then we should fail. This typically happens
+						// when a package times out.
+						failed = true
+					}
 					printPkgOutcome(tr.name.pkg, tr.outcome, thisRun.attempt)
 					continue
 				}