feat(gateway): IPIP-0524 + AllowCodecConversion config option

.github/workflows/gateway-conformance.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
         with:
           output: fixtures
           merged: true
@@ -47,7 +47,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.8
+        uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
         with:
           gateway-url: http://127.0.0.1:8040
           subdomain-url: http://example.net:8040
@@ -84,7 +84,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
         with:
           output: fixtures
           merged: true
@@ -114,7 +114,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.8
+        uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
         with:
           gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
           subdomain-url: http://example.net:8040
@@ -152,7 +152,7 @@ jobs:
     steps:
       # 1. Download the gateway-conformance fixtures
       - name: Download gateway-conformance fixtures
-        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.8
+        uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
         with:
           output: fixtures
           merged: true
@@ -182,7 +182,7 @@ jobs:
 
       # 4. Run the gateway-conformance tests
       - name: Run gateway-conformance tests without IPNS and DNSLink
-        uses: ipfs/gateway-conformance/.github/actions/test@v0.8
+        uses: ipfs/gateway-conformance/.github/actions/test@376504c31aae5e2d47c23cb1e131a7573a7e3a7f # TODO: switch to release tag once https://github.com/ipfs/gateway-conformance/pull/254 ships
         with:
           gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
           subdomain-url: http://example.net:8040

CHANGELOG.md

@@ -22,6 +22,8 @@ The following emojis are used to highlight certain changes:
 
 ### Changed
 
+- `gateway`: 🛠 Codec conversions (e.g., dag-pb to dag-json, dag-json to dag-cbor) are no longer performed by default per [IPIP-0524](https://github.com/ipfs/specs/pull/524). Requesting a format that differs from the block's codec now returns HTTP 406 Not Acceptable. Clients should fetch raw blocks (`?format=raw`) and convert in userland. Set `Config.AllowCodecConversion` to `true` to restore the old behavior.
+
 ### Removed
 
 ### Fixed

examples/gateway/car-file/main_test.go

@@ -1,15 +1,14 @@
 package main
 
 import (
+	"bytes"
 	"io"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/ipfs/boxo/examples/gateway/common"
 	"github.com/ipfs/boxo/gateway"
-	"github.com/ipld/go-ipld-prime/codec/dagjson"
-	"github.com/ipld/go-ipld-prime/node/basicnode"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -62,48 +61,35 @@ func TestFile(t *testing.T) {
 	assert.EqualValues(t, string(body), "hello world\n")
 }
 
-func TestDirectoryAsDAG(t *testing.T) {
+func TestDirectoryAsRawBlock(t *testing.T) {
 	ts, f, err := newTestServer()
 	assert.NoError(t, err)
 	defer f.Close()
 
-	res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=dag-json")
+	res, err := http.Get(ts.URL + "/ipfs/" + BaseCID + "?format=raw")
 	assert.NoError(t, err)
 	defer res.Body.Close()
 
-	contentType := res.Header.Get("Content-Type")
-	assert.EqualValues(t, contentType, "application/vnd.ipld.dag-json")
-
-	// Parses the DAG-JSON response.
-	dag := basicnode.Prototype.Any.NewBuilder()
-	err = dagjson.Decode(dag, res.Body)
-	assert.NoError(t, err)
-
-	// Checks for the links inside the logical model.
-	links, err := dag.Build().LookupByString("Links")
-	assert.NoError(t, err)
-
-	// Checks if there are 2 links.
-	assert.EqualValues(t, links.Length(), 2)
-
-	// Check if the first item is correct.
-	n, err := links.LookupByIndex(0)
-	assert.NoError(t, err)
-	assert.NotNil(t, n)
+	assert.Equal(t, http.StatusOK, res.StatusCode)
 
-	nameNode, err := n.LookupByString("Name")
-	assert.NoError(t, err)
-	assert.NotNil(t, nameNode)
-
-	name, err := nameNode.AsString()
-	assert.NoError(t, err)
-	assert.EqualValues(t, name, "eye.png")
+	contentType := res.Header.Get("Content-Type")
+	assert.Equal(t, "application/vnd.ipld.raw", contentType)
 
-	hashNode, err := n.LookupByString("Hash")
+	body, err := io.ReadAll(res.Body)
 	assert.NoError(t, err)
-	assert.NotNil(t, hashNode)
 
-	hash, err := hashNode.AsLink()
-	assert.NoError(t, err)
-	assert.EqualValues(t, hash.String(), "bafybeigmlfksb374fdkxih4urny2yiyazyra2375y2e4a72b3jcrnthnau")
+	// Raw bytes of the dag-pb directory block
+	expected := []byte{
+		0x12, 0x33, 0x0a, 0x24, 0x01, 0x70, 0x12, 0x20, 0xcc, 0x59, 0x55, 0x20,
+		0xef, 0xfc, 0x28, 0xd5, 0x74, 0x1f, 0x94, 0x8b, 0x71, 0xac, 0x23, 0x00,
+		0xce, 0x22, 0x0d, 0x6f, 0xfd, 0xc6, 0x89, 0xc0, 0x7f, 0x41, 0xda, 0x45,
+		0x16, 0xcc, 0xed, 0x05, 0x12, 0x07, 0x65, 0x79, 0x65, 0x2e, 0x70, 0x6e,
+		0x67, 0x18, 0xd0, 0xc8, 0x10, 0x12, 0x33, 0x0a, 0x24, 0x01, 0x55, 0x12,
+		0x20, 0xa9, 0x48, 0x90, 0x4f, 0x2f, 0x0f, 0x47, 0x9b, 0x8f, 0x81, 0x97,
+		0x69, 0x4b, 0x30, 0x18, 0x4b, 0x0d, 0x2e, 0xd1, 0xc1, 0xcd, 0x2a, 0x1e,
+		0xc0, 0xfb, 0x85, 0xd2, 0x99, 0xa1, 0x92, 0xa4, 0x47, 0x12, 0x09, 0x68,
+		0x65, 0x6c, 0x6c, 0x6f, 0x2e, 0x74, 0x78, 0x74, 0x18, 0x0c, 0x0a, 0x02,
+		0x08, 0x01,
+	}
+	assert.True(t, bytes.Equal(body, expected), "raw block bytes should match")
 }

examples/go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/ipfs/go-cid v0.6.0
 	github.com/ipfs/go-datastore v0.9.0
 	github.com/ipld/go-car/v2 v2.16.0
-	github.com/ipld/go-ipld-prime v0.21.0
 	github.com/libp2p/go-libp2p v0.45.0
 	github.com/multiformats/go-multiaddr v0.16.1
 	github.com/multiformats/go-multicodec v0.10.0
@@ -63,6 +62,7 @@ require (
 	github.com/ipfs/go-peertaskqueue v0.8.2 // indirect
 	github.com/ipfs/go-unixfsnode v1.10.2 // indirect
 	github.com/ipld/go-codec-dagpb v1.7.0 // indirect
+	github.com/ipld/go-ipld-prime v0.21.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect

gateway/gateway.go

@@ -52,6 +52,19 @@ type Config struct {
 	// [Trustless Gateway]: https://specs.ipfs.tech/http-gateways/trustless-gateway/
 	DeserializedResponses bool
 
+	// AllowCodecConversion enables automatic conversion between codecs when
+	// the requested format differs from the block's native codec. For example,
+	// converting dag-pb (UnixFS) to dag-json.
+	//
+	// When false (default), the gateway returns 406 Not Acceptable if the
+	// requested format doesn't match the block's codec. This follows the
+	// behavior specified in IPIP-0524.
+	//
+	// When true, the gateway attempts to convert between legacy IPLD formats.
+	// This is provided for backwards compatibility but is not required by
+	// the gateway specification.
+	AllowCodecConversion bool
+
 	// NoDNSLink configures the gateway to _not_ perform DNS TXT record lookups in
 	// response to requests with values in `Host` HTTP header. This flag can be
 	// overridden per FQDN in PublicGateways. To be used with WithHostname.

gateway/gateway_test.go

@@ -521,6 +521,7 @@ func TestHeaders(t *testing.T) {
 				},
 			},
 			DeserializedResponses: true,
+			AllowCodecConversion:  true, // Test tests various format conversions
 		})
 
 		runTest := func(name, path, accept, host, expectedContentLocationHdr string) {
@@ -1073,7 +1074,8 @@ func TestDeserializedResponses(t *testing.T) {
 		backend, root := newMockBackend(t, "fixtures.car")
 
 		ts := newTestServerWithConfig(t, backend, Config{
-			NoDNSLink: false,
+			NoDNSLink:            false,
+			AllowCodecConversion: true, // Test expects codec conversions to work
 			PublicGateways: map[string]*PublicGateway{
 				"trustless.com": {
 					Paths: []string{"/ipfs", "/ipns"},
@@ -1152,7 +1154,8 @@ func TestDeserializedResponses(t *testing.T) {
 		backend.namesys["/ipns/trusted.com"] = newMockNamesysItem(path.FromCid(root), 0)
 
 		ts := newTestServerWithConfig(t, backend, Config{
-			NoDNSLink: false,
+			NoDNSLink:            false,
+			AllowCodecConversion: true, // Test expects codec conversions to work
 			PublicGateways: map[string]*PublicGateway{
 				"trustless.com": {
 					Paths: []string{"/ipfs", "/ipns"},
@@ -1186,6 +1189,58 @@ func TestDeserializedResponses(t *testing.T) {
 	})
 }
 
+func TestAllowCodecConversion(t *testing.T) {
+	t.Parallel()
+
+	// Use dag-cbor fixture
+	backend, dagCborRoot := newMockBackend(t, "path_gateway_dag/dag-cbor-traversal.car")
+
+	t.Run("AllowCodecConversion=false returns 406 for codec mismatch", func(t *testing.T) {
+		t.Parallel()
+
+		ts := newTestServerWithConfig(t, backend, Config{
+			DeserializedResponses: true,
+			AllowCodecConversion:  false, // IPIP-0524 behavior
+		})
+
+		// Request dag-json for a dag-cbor block - should return 406
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil)
+		res := mustDoWithoutRedirect(t, req)
+		defer res.Body.Close()
+		assert.Equal(t, http.StatusNotAcceptable, res.StatusCode)
+	})
+
+	t.Run("AllowCodecConversion=false allows matching codec", func(t *testing.T) {
+		t.Parallel()
+
+		ts := newTestServerWithConfig(t, backend, Config{
+			DeserializedResponses: true,
+			AllowCodecConversion:  false, // IPIP-0524 behavior
+		})
+
+		// Request dag-cbor for a dag-cbor block - should return 200
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-cbor", nil)
+		res := mustDoWithoutRedirect(t, req)
+		defer res.Body.Close()
+		assert.Equal(t, http.StatusOK, res.StatusCode)
+	})
+
+	t.Run("AllowCodecConversion=true allows codec conversion", func(t *testing.T) {
+		t.Parallel()
+
+		ts := newTestServerWithConfig(t, backend, Config{
+			DeserializedResponses: true,
+			AllowCodecConversion:  true, // Legacy behavior
+		})
+
+		// Request dag-json for a dag-cbor block - should return 200 with conversion
+		req := mustNewRequest(t, http.MethodGet, ts.URL+"/ipfs/"+dagCborRoot.String()+"?format=dag-json", nil)
+		res := mustDoWithoutRedirect(t, req)
+		defer res.Body.Close()
+		assert.Equal(t, http.StatusOK, res.StatusCode)
+	})
+}
+
 type errorMockBackend struct {
 	err error
 }

gateway/handler_codec.go

@@ -148,7 +148,20 @@ func (i *handler) renderCodec(ctx context.Context, w http.ResponseWriter, r *htt
 		return false
 	}
 
-	// This handles DAG-* conversions and validations.
+	// IPIP-0524: Check if codec conversion is allowed
+	if !i.config.AllowCodecConversion && toCodec != cidCodec {
+		// Conversion not allowed and codecs don't match - return 406
+		err := fmt.Errorf("format %q requested but block has codec %q: codec conversion is not supported", rq.responseFormat, cidCodec.String())
+		i.webError(w, r, err, http.StatusNotAcceptable)
+		return false
+	}
+
+	// If codecs match, serve raw (no conversion needed)
+	if toCodec == cidCodec {
+		return i.serveCodecRaw(ctx, w, r, blockSize, blockData, rq.contentPath, modtime, rq.begin)
+	}
+
+	// AllowCodecConversion is true - perform DAG-* conversion
 	return i.serveCodecConverted(ctx, w, r, blockCid, blockData, rq.contentPath, toCodec, modtime, rq.begin)
 }
 

gateway/utilities_test.go

@@ -234,6 +234,7 @@ func newTestServerAndNode(t *testing.T, fixturesFile string) (*httptest.Server,
 func newTestServer(t *testing.T, backend IPFSBackend) *httptest.Server {
 	return newTestServerWithConfig(t, backend, Config{
 		DeserializedResponses: true,
+		AllowCodecConversion:  true, // Enable for backwards compatibility in tests
 		MetricsRegistry:       prometheus.NewRegistry(),
 	})
 }