cmd/cigocacher,go.mod: add logging for canceled PUTs (#20584)

Pulls in bradfitz/go-tool-cache#43 and:

* Add logging for canceled PUTs to ensure we have some visibility.
* Scale PUT timeouts with object size.
* Control the Shutdown timeout separately from the PUT timeout.

Updates tailscale/corp#45334

Signed-off-by: Tom Proctor <tomhjp@users.noreply.github.com>

cmd/cigocacher/cigocacher.go

@@ -149,7 +149,7 @@ func main() {
 			AccessToken:           *token,
 			Verbose:               *verbose,
 			BestEffortHTTP:        true,
-			AsyncPutTimeout:       10 * time.Second, // Generous enough to allow for big objects ~100MiB.
+			AsyncPutTimeout:       asyncPutTimeout,
 			AsyncPutMaxConcurrent: 10,
 		}
 	}
@@ -157,9 +157,15 @@ func main() {
 	p = &cacheproc.Process{
 		Close: func() error {
 			if c.remote != nil {
-				if !c.remote.Shutdown() {
+				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+				defer cancel()
+				if !c.remote.Shutdown(ctx) {
 					log.Printf("cigocacher: timed out waiting for background PUTs to drain")
 				}
+				// Always surface dropped PUTs.
+				if timedOut, canceled := c.remote.PutsTimedOut.Load(), c.remote.PutsCanceled.Load(); timedOut+canceled > 0 {
+					log.Printf("cigocacher: %d background PUTs timed out, %d canceled", timedOut, canceled)
+				}
 			}
 			if c.verbose {
 				log.Printf("cigocacher: closing; %d gets (%d hits, %d misses, %d errors); %d puts (%d errors)",
@@ -345,3 +351,23 @@ func fetchStats(cl *http.Client, baseURL, accessToken string) (string, error) {
 	}
 	return string(b), nil
 }
+
+const (
+	// minPutTimeout is the floor we clamp to for small objects where the time is
+	// dominated by fixed overheads like connection establishment, waiting for a
+	// busy server to service the request etc.
+	minPutTimeout = 5 * time.Second
+	// maxPutTimeout is the ceiling we clamp to for large objects.
+	maxPutTimeout = 30 * time.Second
+	// minAverageBandwidth is the minimum average bandwidth (2MiB/s) we require
+	// for PUTs to complete within the timeout in its linear scaling region.
+	minAverageBandwidth = 2 * 1 << 20 / float64(time.Second)
+)
+
+// asyncPutTimeout returns a size-dependent timeout for async PUTs to the remote
+// gocached server. It returns 5s for size <= 10MiB, 30s for size >= 60MiB and
+// scales linearly in between.
+func asyncPutTimeout(size int64) time.Duration {
+	timeout := time.Duration(float64(size) / minAverageBandwidth)
+	return min(max(minPutTimeout, timeout), maxPutTimeout)
+}

cmd/cigocacher/cigocacher_test.go

@@ -0,0 +1,25 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+func TestAsyncPutTimeout(t *testing.T) {
+	for size, expected := range map[int64]time.Duration{
+		0:            5 * time.Second,
+		10:           5 * time.Second,
+		10 * 1 << 20: 5 * time.Second,
+		20 * 1 << 20: 10 * time.Second,
+		40 * 1 << 20: 20 * time.Second,
+		60 * 1 << 20: 30 * time.Second,
+		10 * 1 << 30: 30 * time.Second,
+	} {
+		if actual := asyncPutTimeout(size); actual != expected {
+			t.Errorf("for size %d, expected %v, but got %v", size, expected, actual)
+		}
+	}
+}