fix: add retry with exponential backoff to schema fetch for transient HTTP errors (#2582)

Gateway startup fails when `raw.githubusercontent.com` rate-limits the
JSON config schema fetch (HTTP 429). Any transient error was treated as
fatal with no retry.

## Changes

**`internal/config/validation_schema.go`**
- Added `isTransientHTTPError()` identifying 429, 503, and all 5xx codes
- `fetchAndFixSchema()` now retries up to `maxSchemaFetchRetries` (3)
times with exponential backoff (1s → 2s) for transient errors; permanent
errors (4xx except 429) fail immediately
- `schemaFetchRetryDelay` and `schemaHTTPClientTimeout` are
package-level vars so tests can zero them out without sleeps

**`internal/config/validation_schema_fetch_test.go`**
- `init()` sets zero delay + 200ms HTTP timeout for all tests in the
file
- `TestFetchAndFixSchema_HTTPError` extended with 429 case and
per-error-type request-count assertions (1 request for permanent, 3 for
transient)
- Added `TestFetchAndFixSchema_RetrySucceedsAfterTransientError`: 2×429
→ 200 succeeds
- Added `TestFetchAndFixSchema_ExponentialBackoffDelays`: verifies
exactly `maxSchemaFetchRetries` requests are made on persistent
transient failure

**Note**: The secrecy-vs-integrity notice labeling fix (Problem 2 from
the issue) is already present in the codebase (`difcPolicyLabel()` +
`IsSecrecyViolation` in `difc_log.go` / `difc/resource.go`).

> [!WARNING]
>
> <details>
> <summary>Firewall rules blocked me from connecting to one or more
addresses (expand for details)</summary>
>
> #### I tried to connect to the following addresses, but was blocked by
firewall rules:
>
> - `example.com`
> - Triggering command: `/tmp/go-build3732201295/b330/launcher.test
/tmp/go-build3732201295/b330/launcher.test
-test.testlogfile=/tmp/go-build3732201295/b330/testlog.txt
-test.paniconexit0 -test.timeout=10m0s -test.v=true g_.a --global git
pull.rebase abis` (dns block)
> - Triggering command: `/tmp/go-build63582137/b334/launcher.test
/tmp/go-build63582137/b334/launcher.test
-test.testlogfile=/tmp/go-build63582137/b334/testlog.txt
-test.paniconexit0 -test.timeout=10m0s` (dns block)
> - `invalid-host-that-does-not-exist-12345.com`
> - Triggering command: `/tmp/go-build3732201295/b315/config.test
/tmp/go-build3732201295/b315/config.test
-test.testlogfile=/tmp/go-build3732201295/b315/testlog.txt
-test.paniconexit0 -test.timeout=10m0s -test.v=true -c=4 -nolocalimports
-importcfg /tmp/go-build3732201295/b287/importcfg -pack
/home/REDACTED/go/pkg/mod/github.com/modelcontextprotocol/go-sdk@v1.4.1/auth/auth.go
/home/REDACTED/go/pkg/mod/github.com/modelcontextprotocol/go-sdk@v1.4.1/auth/client.go
ortc�� go 9p-W5oE9v ache/go/1.25.8/x64/pkg/tool/linu-Werror
credential.helpe/opt/hostedtoolcache/go/1.25.8/x64/pkg/tool/linux_amd64/compile`
(dns block)
> - Triggering command: `/tmp/go-build2566417517/b001/config.test
/tmp/go-build2566417517/b001/config.test
-test.testlogfile=/tmp/go-build2566417517/b001/testlog.txt
-test.paniconexit0 -test.timeout=10m0s -test.run=TestFetchAndFixSchema
-test.v=true ache/go/1.25.8/x64/src/crypto/ec-ifaceassert x_amd64/vet -p
internal/reflectdocker-cli-plugin-metadata -lang=go1.25 x_amd64/vet ap
2201295/b171=/tmp/go-build .cfg 64/pkg/tool/linux_amd64/vet -p
ache/go/1.25.8/x-c -lang=go1.25 64/pkg/tool/linux_amd64/vet` (dns block)
> - Triggering command: `/tmp/go-build1075949314/b001/config.test
/tmp/go-build1075949314/b001/config.test
-test.testlogfile=/tmp/go-build1075949314/b001/testlog.txt
-test.paniconexit0 -test.run=TestFetchAndFixSchema -test.v=true
-test.timeout=30s SkhS/x3QNW0bkE1zmain 64/pkg/tool/linu-lang=go1.25 -I
/tmp/go-build373docker-cli-plugin-metadata -I
64/pkg/tool/linu-dwarf=false 2201�� azero@v1.11.0/ingo1.25.8
azero@v1.11.0/in-c=4 64/pkg/tool/linu-nolocalimports 2201295/b165/
/dev/null ctor
64/pkg/tool/linu/tmp/go-build3732201295/b345/_testmain.go` (dns block)
> - `nonexistent.local`
> - Triggering command: `/tmp/go-build3732201295/b330/launcher.test
/tmp/go-build3732201295/b330/launcher.test
-test.testlogfile=/tmp/go-build3732201295/b330/testlog.txt
-test.paniconexit0 -test.timeout=10m0s -test.v=true g_.a --global git
pull.rebase abis` (dns block)
> - Triggering command: `/tmp/go-build63582137/b334/launcher.test
/tmp/go-build63582137/b334/launcher.test
-test.testlogfile=/tmp/go-build63582137/b334/testlog.txt
-test.paniconexit0 -test.timeout=10m0s` (dns block)
> - `slow.example.com`
> - Triggering command: `/tmp/go-build3732201295/b330/launcher.test
/tmp/go-build3732201295/b330/launcher.test
-test.testlogfile=/tmp/go-build3732201295/b330/testlog.txt
-test.paniconexit0 -test.timeout=10m0s -test.v=true g_.a --global git
pull.rebase abis` (dns block)
> - Triggering command: `/tmp/go-build63582137/b334/launcher.test
/tmp/go-build63582137/b334/launcher.test
-test.testlogfile=/tmp/go-build63582137/b334/testlog.txt
-test.paniconexit0 -test.timeout=10m0s` (dns block)
> - `this-host-does-not-exist-12345.com`
> - Triggering command: `/tmp/go-build3732201295/b339/mcp.test
/tmp/go-build3732201295/b339/mcp.test
-test.testlogfile=/tmp/go-build3732201295/b339/testlog.txt
-test.paniconexit0 -test.timeout=10m0s -test.v=true 64/src/runtime/cgo
committer.name x_amd64/vet` (dns block)
> - Triggering command: `/tmp/go-build63582137/b343/mcp.test
/tmp/go-build63582137/b343/mcp.test
-test.testlogfile=/tmp/go-build63582137/b343/testlog.txt
-test.paniconexit0 -test.timeout=10m0s ortc�� TOKEN&#34;; }; f get
TOKEN&#34;; }; f get ache/go/1.25.8/x64/pkg/tool/linux_amd64/vet
-pthread telabs/wazero -fmessage-length=0
ache/go/1.25.8/x64/pkg/tool/linux_amd64/vet -I eNqb/Hpbq2t4bcSrSRPw9eNqb
-I x_amd64/vet --gdwarf-5 --64 -o x_amd64/vet` (dns block)
>
> If you need me to access, download, or install something from one of
these locations, you can either:
>
> - Configure [Actions setup
steps](https://gh.io/copilot/actions-setup-steps) to set up my
environment, which run before the firewall is enabled
> - Add the appropriate URLs or hosts to the custom allowlist in this
repository's [Copilot coding agent
settings](https://github.com/github/gh-aw-mcpg/settings/copilot/coding_agent)
(admins only)
>
> </details>

<!-- START COPILOT CODING AGENT TIPS -->
---

📍 Connect Copilot coding agent with [Jira](https://gh.io/cca-jira-docs),
[Azure Boards](https://gh.io/cca-azure-boards-docs) or
[Linear](https://gh.io/cca-linear-docs) to delegate work to Copilot in
one click without leaving your project management tool.

Author: lpcox

internal/config/validation_schema.go

@@ -16,6 +16,28 @@ import (
 	"github.com/santhosh-tekuri/jsonschema/v5"
 )
 
+const (
+	// maxSchemaFetchRetries is the number of fetch attempts before giving up.
+	maxSchemaFetchRetries = 3
+)
+
+// schemaFetchRetryDelay is the base delay between retry attempts using exponential
+// backoff (1×, 2×, 4×, …). It is a variable so tests can override it to zero for
+// fast execution.
+var schemaFetchRetryDelay = time.Second
+
+// schemaHTTPClientTimeout is the per-attempt HTTP request timeout. It is a variable
+// so tests can shorten it to avoid long waits when testing timeout behaviour.
+var schemaHTTPClientTimeout = 10 * time.Second
+
+// isTransientHTTPError returns true for status codes that indicate a temporary
+// server-side condition (rate-limiting or transient failure) worth retrying.
+func isTransientHTTPError(statusCode int) bool {
+	return statusCode == http.StatusTooManyRequests ||
+		statusCode == http.StatusServiceUnavailable ||
+		(statusCode >= 500 && statusCode < 600)
+}
+
 var (
 	// Compile regex patterns from schema for additional validation
 	containerPattern = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9./_-]*(:([a-zA-Z0-9._-]+|latest))?$`)
@@ -83,22 +105,56 @@ func fetchAndFixSchema(url string) ([]byte, error) {
 	logSchema.Printf("Fetching schema from URL: %s", url)
 
 	client := &http.Client{
-		Timeout: 10 * time.Second,
+		Timeout: schemaHTTPClientTimeout,
 	}
 
-	fetchStart := time.Now()
-	resp, err := client.Get(url)
-	if err != nil {
-		logSchema.Printf("Schema fetch failed after %v: %v", time.Since(fetchStart), err)
-		return nil, fmt.Errorf("failed to fetch schema from %s: %w", url, err)
+	var resp *http.Response
+	var lastErr error
+
+	for attempt := 1; attempt <= maxSchemaFetchRetries; attempt++ {
+		if attempt > 1 {
+			delay := schemaFetchRetryDelay << uint(attempt-2) // 1×, 2×, 4× base delay
+			logSchema.Printf("Retrying schema fetch (attempt %d/%d) after %v: %v", attempt, maxSchemaFetchRetries, delay, lastErr)
+			time.Sleep(delay)
+		}
+
+		fetchStart := time.Now()
+		var err error
+		resp, err = client.Get(url)
+		if err != nil {
+			logSchema.Printf("Schema fetch attempt %d failed after %v: %v", attempt, time.Since(fetchStart), err)
+			lastErr = fmt.Errorf("failed to fetch schema from %s: %w", url, err)
+			resp = nil
+			continue
+		}
+		logSchema.Printf("HTTP request attempt %d completed in %v with status %d", attempt, time.Since(fetchStart), resp.StatusCode)
+
+		if resp.StatusCode == http.StatusOK {
+			lastErr = nil
+			break
+		}
+
+		if isTransientHTTPError(resp.StatusCode) {
+			lastErr = fmt.Errorf("failed to fetch schema: HTTP %d", resp.StatusCode)
+			logSchema.Printf("Schema fetch attempt %d returned transient error: HTTP %d, will retry", attempt, resp.StatusCode)
+			resp.Body.Close()
+			resp = nil
+			continue
+		}
+
+		// Permanent HTTP error (404, 403, 401, etc.) — do not retry.
+		lastErr = fmt.Errorf("failed to fetch schema: HTTP %d", resp.StatusCode)
+		logSchema.Printf("Schema fetch returned permanent error: HTTP %d", resp.StatusCode)
+		resp.Body.Close()
+		resp = nil
+		break
 	}
-	defer resp.Body.Close()
-	logSchema.Printf("HTTP request completed in %v", time.Since(fetchStart))
 
-	if resp.StatusCode != http.StatusOK {
-		logSchema.Printf("Schema fetch returned non-OK status: %d", resp.StatusCode)
-		return nil, fmt.Errorf("failed to fetch schema: HTTP %d", resp.StatusCode)
+	if resp == nil {
+		return nil, lastErr
 	}
+	defer resp.Body.Close()
+	logSchema.Printf("HTTP request completed in %v", time.Since(startTime))
 
 	readStart := time.Now()
 	schemaBytes, err := io.ReadAll(resp.Body)

internal/config/validation_schema_fetch_test.go

@@ -6,13 +6,21 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func init() {
+	// Speed up all retry-related tests by disabling inter-attempt delays and
+	// shortening the per-attempt HTTP timeout so timeout tests don't take 30 s.
+	schemaFetchRetryDelay = 0
+	schemaHTTPClientTimeout = 200 * time.Millisecond
+}
+
 // TestFetchAndFixSchema_SuccessfulFetch tests the happy path where schema is fetched successfully
 func TestFetchAndFixSchema_SuccessfulFetch(t *testing.T) {
 	// Create a minimal valid schema for testing
@@ -52,35 +60,48 @@ func TestFetchAndFixSchema_SuccessfulFetch(t *testing.T) {
 // TestFetchAndFixSchema_HTTPError tests handling of HTTP error responses
 func TestFetchAndFixSchema_HTTPError(t *testing.T) {
 	tests := []struct {
-		name       string
-		statusCode int
-		wantErr    string
+		name         string
+		statusCode   int
+		wantErr      string
+		wantRequests int // expected number of HTTP requests (1 = no retry, 3 = full retry)
 	}{
 		{
-			name:       "404 Not Found",
-			statusCode: http.StatusNotFound,
-			wantErr:    "failed to fetch schema: HTTP 404",
+			name:         "404 Not Found",
+			statusCode:   http.StatusNotFound,
+			wantErr:      "failed to fetch schema: HTTP 404",
+			wantRequests: 1, // permanent error, no retry
 		},
 		{
-			name:       "500 Internal Server Error",
-			statusCode: http.StatusInternalServerError,
-			wantErr:    "failed to fetch schema: HTTP 500",
+			name:         "500 Internal Server Error",
+			statusCode:   http.StatusInternalServerError,
+			wantErr:      "failed to fetch schema: HTTP 500",
+			wantRequests: maxSchemaFetchRetries, // transient, retried
 		},
 		{
-			name:       "403 Forbidden",
-			statusCode: http.StatusForbidden,
-			wantErr:    "failed to fetch schema: HTTP 403",
+			name:         "403 Forbidden",
+			statusCode:   http.StatusForbidden,
+			wantErr:      "failed to fetch schema: HTTP 403",
+			wantRequests: 1, // permanent error, no retry
 		},
 		{
-			name:       "503 Service Unavailable",
-			statusCode: http.StatusServiceUnavailable,
-			wantErr:    "failed to fetch schema: HTTP 503",
+			name:         "503 Service Unavailable",
+			statusCode:   http.StatusServiceUnavailable,
+			wantErr:      "failed to fetch schema: HTTP 503",
+			wantRequests: maxSchemaFetchRetries, // transient, retried
+		},
+		{
+			name:         "429 Too Many Requests",
+			statusCode:   http.StatusTooManyRequests,
+			wantErr:      "failed to fetch schema: HTTP 429",
+			wantRequests: maxSchemaFetchRetries, // transient, retried
 		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			var requestCount atomic.Int32
 			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				requestCount.Add(1)
 				w.WriteHeader(tt.statusCode)
 			}))
 			defer server.Close()
@@ -90,6 +111,8 @@ func TestFetchAndFixSchema_HTTPError(t *testing.T) {
 			assert.Error(t, err)
 			assert.Nil(t, result)
 			assert.Contains(t, err.Error(), tt.wantErr)
+			assert.Equal(t, int32(tt.wantRequests), requestCount.Load(),
+				"expected %d HTTP request(s) for status %d", tt.wantRequests, tt.statusCode)
 		})
 	}
 }
@@ -108,9 +131,11 @@ func TestFetchAndFixSchema_NetworkError(t *testing.T) {
 
 // TestFetchAndFixSchema_Timeout tests handling of request timeouts
 func TestFetchAndFixSchema_Timeout(t *testing.T) {
-	// Create a server that delays longer than the client timeout
+	// Create a server that delays longer than the configured client timeout.
+	// The init() in this test file sets schemaHTTPClientTimeout = 200ms, so any
+	// delay > 200ms will trigger a timeout on each attempt.
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		time.Sleep(15 * time.Second) // fetchAndFixSchema has 10 second timeout
+		time.Sleep(500 * time.Millisecond)
 		w.WriteHeader(http.StatusOK)
 	}))
 	defer server.Close()
@@ -589,3 +614,53 @@ func TestFetchAndFixSchema_LargeSchema(t *testing.T) {
 	require.True(t, ok)
 	assert.Equal(t, 100, len(properties), "Should preserve all 100 properties")
 }
+
+// TestFetchAndFixSchema_RetrySucceedsAfterTransientError verifies that a transient
+// error on the first attempt is retried and the eventual success is returned.
+func TestFetchAndFixSchema_RetrySucceedsAfterTransientError(t *testing.T) {
+	validSchema := map[string]interface{}{
+		"$schema": "http://json-schema.org/draft-07/schema#",
+		"type":    "object",
+	}
+	schemaJSON, err := json.Marshal(validSchema)
+	require.NoError(t, err)
+
+	var requestCount atomic.Int32
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		n := requestCount.Add(1)
+		if n < 3 {
+			// First two attempts return 429
+			w.WriteHeader(http.StatusTooManyRequests)
+			return
+		}
+		w.WriteHeader(http.StatusOK)
+		w.Write(schemaJSON)
+	}))
+	defer server.Close()
+
+	result, err := fetchAndFixSchema(server.URL)
+
+	require.NoError(t, err)
+	assert.NotNil(t, result)
+	assert.Equal(t, int32(3), requestCount.Load(), "should have made 3 requests (2 failures + 1 success)")
+}
+
+// TestFetchAndFixSchema_ExponentialBackoffDelays verifies that all retries are attempted
+// when transient errors occur, and that the function eventually gives up after
+// maxSchemaFetchRetries attempts.
+func TestFetchAndFixSchema_ExponentialBackoffDelays(t *testing.T) {
+	var requestCount atomic.Int32
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		requestCount.Add(1)
+		w.WriteHeader(http.StatusTooManyRequests)
+	}))
+	defer server.Close()
+
+	_, err := fetchAndFixSchema(server.URL)
+
+	require.Error(t, err)
+	assert.Equal(t, int32(maxSchemaFetchRetries), requestCount.Load(),
+		"should make exactly maxSchemaFetchRetries requests before giving up")
+	assert.Contains(t, err.Error(), "failed to fetch schema: HTTP 429")
+}