// HTTP client with exponential backoff retry. // // Handles transient network failures, timeouts, and rate limiting. // - Backoff: 500ms → 1s → 2s → 4s max // - Jitter prevents thundering herd // - Respects 429 Retry-After header package core import ( "context" "errors" "fmt" "math/rand" "net" "net/http" "os" "strconv" "strings" "time" ) // ============================================================================ // ╻ ╻╺┳╸╺┳╸┏━┓ ┏━┓┏━╸╺┳╸┏━┓╻ ╻ // ┣━┫ ┃ ┃ ┣━┛ ┣┳┛┣╸ ┃ ┣┳┛┗┳┛ // ╹ ╹ ╹ ╹ ╹ ╹┗╸┗━╸ ╹ ╹┗╸ ╹ // ============================================================================ const PoliteUserAgent = "scholscan/1.0 (https://github.com/mrichman/scholscan; mailto:matt@mrichman.net)" var DefaultHTTPClient = &http.Client{ Timeout: 30 * time.Second, } var ( retryMaxAttempts = 4 retryInitialBackoff = 500 * time.Millisecond retryMaxBackoff = 5 * time.Second ) // Makes HTTP request with exponential backoff retry func DoRequestWithRetry( ctx context.Context, client *http.Client, req *http.Request, ) (*http.Response, error) { if client == nil { client = DefaultHTTPClient } var lastErr error backoff := retryInitialBackoff for attempt := 1; attempt <= retryMaxAttempts; attempt++ { // Make the request cancellable reqWithCtx := req.WithContext(ctx) resp, err := client.Do(reqWithCtx) if err == nil { if isRetriableStatus(resp.StatusCode) { retryAfter := parseRetryAfter(resp.Header.Get("Retry-After")) _ = resp.Body.Close() sleep := backoff if retryAfter > sleep { sleep = retryAfter } // Add jitter to avoid thundering herd. jitter := time.Duration(rand.Intn(int(backoff / 2))) sleep += jitter // Make sleep cancellable timer := time.NewTimer(sleep) select { case <-ctx.Done(): timer.Stop() return nil, ctx.Err() case <-timer.C: } backoff = minDuration(backoff*2, retryMaxBackoff) continue } return resp, nil } // Check for context cancellation if ctx.Err() != nil { return nil, ctx.Err() } // Network error: retry on timeouts, context deadline, transient net errors, and HTTP/2 stream errors if os.IsTimeout(err) || errors.Is(err, context.DeadlineExceeded) || isTransientNetError(err) || isHTTP2StreamErr(err) { lastErr = err // Add jitter to avoid thundering herd. jitter := time.Duration(rand.Intn(int(backoff / 2))) sleep := backoff + jitter // Make sleep cancellable timer := time.NewTimer(sleep) select { case <-ctx.Done(): timer.Stop() return nil, ctx.Err() case <-timer.C: } backoff = minDuration(backoff*2, retryMaxBackoff) continue } // Non-retriable error return nil, err } if lastErr == nil { lastErr = fmt.Errorf("request retries exhausted") } return nil, lastErr } // ============================================================================ // ╻ ╻┏━╸╻ ┏━┓┏━╸┏━┓┏━┓ // ┣━┫┣╸ ┃ ┣━┛┣╸ ┣┳┛┗━┓ // ╹ ╹┗━╸┗━╸╹ ┗━╸╹┗╸┗━┛ // ============================================================================ func isRetriableStatus(code int) bool { if code == http.StatusTooManyRequests { return true } return code >= 500 && code != http.StatusNotImplemented } func parseRetryAfter(v string) time.Duration { if v == "" { return 0 } if secs, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && secs > 0 { return time.Duration(secs) * time.Second } if t, err := http.ParseTime(v); err == nil { if d := time.Until(t); d > 0 { return d } } return 0 } func minDuration(a, b time.Duration) time.Duration { if a < b { return a } return b } // isTransientNetError returns true for network errors which are commonly transient, // such as timeouts and common connection reset/closed cases. func isTransientNetError(err error) bool { if err == nil { return false } var ne net.Error if errors.As(err, &ne) { if ne.Timeout() { return true } } msg := strings.ToLower(err.Error()) switch { case strings.Contains(msg, "use of closed network connection"): return true case strings.Contains(msg, "connection reset by peer"): return true case strings.Contains(msg, "connection aborted"): return true case strings.Contains(msg, "broken pipe"): return true case strings.Contains(msg, "eof"): // Treat unexpected EOFs as transient when occurring at transport level. return true default: return false } } // isHTTP2StreamErr detects HTTP/2 stream-level errors which are often transient. func isHTTP2StreamErr(err error) bool { if err == nil { return false } msg := strings.ToLower(err.Error()) return strings.Contains(msg, "stream error") || strings.Contains(msg, "internal_error") || strings.Contains(msg, "rst_stream") || strings.Contains(msg, "goaway") || strings.Contains(msg, "http2:") }