control/controlclient: handle 429 responses during node registration
If we get a 429 response during node registration, use the `Retry-After` header for backoff instead of the regular exponential backoff. The rate limiter error is propagated to the user, just like other registration errors are, e.g. ``` $ tailscale up backend error: node registration rate limited; will retry after 57s exit status 1 ``` Updates tailscale/corp#39533 Signed-off-by: Anton Tolchanov <anton@tailscale.com>
This commit is contained in:
committed by
Anton Tolchanov
parent
d8190e0de5
commit
958bcda5bf
@@ -5,9 +5,11 @@ package controlclient
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -126,6 +128,109 @@ func fakeEndpoints(ports ...uint16) (ret []tailcfg.Endpoint) {
|
||||
return
|
||||
}
|
||||
|
||||
func TestParseRateLimitError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
statusCode int
|
||||
body string
|
||||
retryAfter string // Retry-After header value
|
||||
wantMsg string
|
||||
wantMin time.Duration // minimum expected retryAfter
|
||||
wantMax time.Duration // maximum expected retryAfter
|
||||
}{
|
||||
{
|
||||
name: "retry-after-seconds",
|
||||
statusCode: 429,
|
||||
body: "too many requests",
|
||||
retryAfter: "30",
|
||||
wantMsg: "too many requests",
|
||||
wantMin: 30 * time.Second,
|
||||
wantMax: 30 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "no-retry-after-header",
|
||||
statusCode: 429,
|
||||
body: "slow down",
|
||||
retryAfter: "",
|
||||
wantMsg: "slow down",
|
||||
wantMin: 5 * time.Second,
|
||||
wantMax: 10 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "unparseable-retry-after",
|
||||
statusCode: 429,
|
||||
body: "rate limited",
|
||||
retryAfter: "not-a-number",
|
||||
wantMsg: "rate limited",
|
||||
wantMin: 5 * time.Second,
|
||||
wantMax: 10 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "empty-body",
|
||||
statusCode: 429,
|
||||
body: "",
|
||||
retryAfter: "5",
|
||||
wantMsg: "",
|
||||
wantMin: 5 * time.Second,
|
||||
wantMax: 5 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "body-with-whitespace",
|
||||
statusCode: 429,
|
||||
body: " too many requests \n",
|
||||
retryAfter: "10",
|
||||
wantMsg: "too many requests",
|
||||
wantMin: 10 * time.Second,
|
||||
wantMax: 10 * time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
if tt.retryAfter != "" {
|
||||
rec.Header().Set("Retry-After", tt.retryAfter)
|
||||
}
|
||||
rec.WriteHeader(tt.statusCode)
|
||||
rec.Body.WriteString(tt.body)
|
||||
res := rec.Result()
|
||||
|
||||
err := parseRateLimitError(res)
|
||||
if err == nil {
|
||||
t.Fatal("expected non-nil error")
|
||||
}
|
||||
|
||||
var rle *rateLimitError
|
||||
if !errors.As(err, &rle) {
|
||||
t.Fatalf("error is not a *rateLimitError: %T", err)
|
||||
}
|
||||
if rle.msg != tt.wantMsg {
|
||||
t.Errorf("msg = %q, want %q", rle.msg, tt.wantMsg)
|
||||
}
|
||||
if rle.retryAfter < tt.wantMin || rle.retryAfter > tt.wantMax {
|
||||
t.Errorf("retryAfter = %v, want between %v and %v", rle.retryAfter, tt.wantMin, tt.wantMax)
|
||||
}
|
||||
|
||||
// Verify the Error() string contains useful information.
|
||||
errStr := err.Error()
|
||||
if !strings.Contains(errStr, "rate limited") {
|
||||
t.Errorf("Error() = %q, want it to contain 'rate limited'", errStr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRateLimitErrorIsError(t *testing.T) {
|
||||
err := &rateLimitError{msg: "test", retryAfter: 5 * time.Second}
|
||||
var target *rateLimitError
|
||||
if !errors.As(err, &target) {
|
||||
t.Fatal("errors.As should match *rateLimitError")
|
||||
}
|
||||
if target.retryAfter != 5*time.Second {
|
||||
t.Errorf("retryAfter = %v, want 5s", target.retryAfter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTsmpPing(t *testing.T) {
|
||||
hi := hostinfo.New()
|
||||
ni := tailcfg.NetInfo{LinkType: "wired"}
|
||||
|
||||
Reference in New Issue
Block a user