feat: support simple JSON API for PyPI package registry (#12095)

This PR extends Forĝejo's PyPI package index to support [the simple JSON repository API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization). Since the existing implementation was for the HTML serialization of the same simple API, no new endpoint has been added. Instead, Forĝejo chooses between serialization schemes based on the "Accept" header in the request. This, together with CORS, will make Forĝejo compatible with [micropip](https://github.com/pyodide/micropip).

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. All work and communication must conform to Forgejo's [AI Agreement](https://codeberg.org/forgejo/governance/src/branch/main/AIAgreement.md). There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests for Go changes

(can be removed for JavaScript changes)

- I added test coverage for Go changes...
  - [x] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I ran...
  - [x] `make pr-go` before pushing

### Documentation

- [x] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [x] This change will be noticed by a Forgejo user or admin (feature, bug fix, performance, etc.). I suggest to include a release note for this change.
- [ ] This change is not visible to a Forgejo user or admin (refactor, dependency upgrade, etc.). I think there is no need to add a release note for this change.

*The decision if the pull request will be shown in the release notes is up to the mergers / release team.*

The content of the `release-notes/<pull request number>.md` file will serve as the basis for the release notes. If the file does not exist, the title of the pull request will be used instead.

<!--start release-notes-assistant-->

## Release notes
<!--URL:https://codeberg.org/forgejo/forgejo-->
- Features
  - [PR](https://codeberg.org/forgejo/forgejo/pulls/12095): <!--number 12095 --><!--line 0 --><!--description SG9zdGVkIFB5UEkgcGFja2FnZXMgbWF5IGJlIGFjY2Vzc2VkIHZpYSB0aGUgW3NpbXBsZSBKU09OIEFQSV0oaHR0cHM6Ly9wYWNrYWdpbmcucHl0aG9uLm9yZy9lbi9sYXRlc3Qvc3BlY2lmaWNhdGlvbnMvc2ltcGxlLXJlcG9zaXRvcnktYXBpLyNqc29uLXNlcmlhbGl6YXRpb24pIGluIGFkZGl0aW9uIHRvIHRoZSBzaW1wbGUgSFRNTCBBUEkgYWxyZWFkeSBhdmFpbGFibGUu-->Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available.<!--description-->
<!--end release-notes-assistant-->

Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/12095
Reviewed-by: Mathieu Fenniak <mfenniak@noreply.codeberg.org>

Author: clayote

modules/packages/pypi/metadata.go

@@ -13,3 +13,26 @@ type Metadata struct {
 	License         string `json:"license,omitempty"`
 	RequiresPython  string `json:"requires_python,omitempty"`
 }
+
+type FileHashesJSON struct {
+	SHA256 string `json:"sha256"`
+}
+
+type FileJSON struct {
+	Filename       string         `json:"filename"`
+	URL            string         `json:"url"`
+	Hashes         FileHashesJSON `json:"hashes"`
+	RequiresPython string         `json:"requires-python"`
+	Size           int64          `json:"size"`
+}
+
+type PackageMetaJSON struct {
+	APIVersion string `json:"api-version"`
+}
+
+type PackageJSON struct {
+	Name     string          `json:"name"`
+	Meta     PackageMetaJSON `json:"meta"`
+	Versions []string        `json:"versions"`
+	Files    []FileJSON      `json:"files"`
+}

release-notes/12095.md

@@ -0,0 +1 @@
+Hosted PyPI packages may be accessed via the [simple JSON API](https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-serialization) in addition to the simple HTML API already available.

routers/api/packages/pypi/pypi.go

@@ -8,11 +8,14 @@ import (
 	"io"
 	"net/http"
 	"regexp"
+	"slices"
 	"sort"
 	"strings"
 	"unicode"
 
 	packages_model "forgejo.org/models/packages"
+	"forgejo.org/modules/json"
+	"forgejo.org/modules/log"
 	packages_module "forgejo.org/modules/packages"
 	pypi_module "forgejo.org/modules/packages/pypi"
 	"forgejo.org/modules/setting"
@@ -44,8 +47,14 @@ func apiError(ctx *context.Context, status int, obj any) {
 	})
 }
 
-// PackageMetadata returns the metadata for a single package
-func PackageMetadata(ctx *context.Context) {
+func contentTypeSupported(ctyps []string, v string) bool {
+	return slices.ContainsFunc(ctyps, func(ctyp string) bool {
+		return strings.HasPrefix(ctyp, v)
+	})
+}
+
+// HTMLPackageMetadata returns the metadata for a single package in Simple HTML per PEP691
+func HTMLPackageMetadata(ctx *context.Context) {
 	packageName := normalizer.Replace(ctx.Params("id"))
 
 	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName)
@@ -72,9 +81,82 @@ func PackageMetadata(ctx *context.Context) {
 	ctx.Data["RegistryURL"] = setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi"
 	ctx.Data["PackageDescriptor"] = pds[0]
 	ctx.Data["PackageDescriptors"] = pds
+	// Content-Type headers need to be in this order for the page to show in the browser
+	ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+html")
+	ctx.Resp.Header().Add("Content-Type", "text/html")
 	ctx.HTML(http.StatusOK, "api/packages/pypi/simple")
 }
 
+// JSONPackageMetadata returns the metadata for a single package in Simple JSON per PEP691
+func JSONPackageMetadata(ctx *context.Context) {
+	packageName := normalizer.Replace(ctx.Params("id"))
+
+	pvs, err := packages_model.GetVersionsByPackageName(ctx, ctx.Package.Owner.ID, packages_model.TypePyPI, packageName)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+	if len(pvs) == 0 {
+		apiError(ctx, http.StatusNotFound, err)
+		return
+	}
+
+	pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
+	if err != nil {
+		apiError(ctx, http.StatusInternalServerError, err)
+		return
+	}
+
+	// sort package descriptors by version to mimic PyPI format
+	slices.SortFunc(pds, func(a, b *packages_model.PackageDescriptor) int {
+		return strings.Compare(a.Version.Version, b.Version.Version)
+	})
+	registryURL := setting.AppURL + "api/packages/" + ctx.Package.Owner.Name + "/pypi"
+	versions := make([]string, len(pvs))
+	for i, pv := range pvs {
+		versions[i] = pv.Version
+	}
+	var fileCounter int
+	for _, pd := range pds {
+		fileCounter += len(pd.Files)
+	}
+	files := make([]pypi_module.FileJSON, fileCounter)
+	var i int
+	for _, pd := range pds {
+		for _, file := range pd.Files {
+			files[i] = pypi_module.FileJSON{
+				Filename:       file.File.Name,
+				URL:            registryURL + "/files/" + pd.Package.LowerName + "/" + pd.Version.Version + "/" + file.File.Name,
+				RequiresPython: pd.Metadata.(*pypi_module.Metadata).RequiresPython,
+				Hashes:         pypi_module.FileHashesJSON{SHA256: file.Blob.HashSHA256},
+				Size:           file.Blob.Size,
+			}
+			i++
+		}
+	}
+	content := pypi_module.PackageJSON{
+		Name:     pds[0].Package.Name,
+		Meta:     pypi_module.PackageMetaJSON{APIVersion: "1.4"},
+		Versions: versions,
+		Files:    files,
+	}
+	ctx.Resp.Header().Set("Content-Type", "application/vnd.pypi.simple.v1+json")
+	ctx.Resp.Header().Add("Content-Type", "application/json")
+	if err := json.NewEncoder(ctx.Resp).Encode(content); err != nil {
+		log.Error("Render JSON failed: %v", err)
+		apiError(ctx, http.StatusInternalServerError, err)
+	}
+}
+
+func PackageMetadata(ctx *context.Context) {
+	ctyp := ctx.Req.Header["Accept"]
+	if contentTypeSupported(ctyp, "application/vnd.pypi.simple.v1+json") {
+		JSONPackageMetadata(ctx)
+	} else {
+		HTMLPackageMetadata(ctx)
+	}
+}
+
 // DownloadPackageFile serves the content of a package
 func DownloadPackageFile(ctx *context.Context) {
 	packageName := normalizer.Replace(ctx.Params("id"))

tests/integration/api_packages_pypi_test.go

@@ -17,6 +17,7 @@ import (
 	"forgejo.org/models/packages"
 	"forgejo.org/models/unittest"
 	user_model "forgejo.org/models/user"
+	"forgejo.org/modules/json"
 	"forgejo.org/modules/packages/pypi"
 	"forgejo.org/tests"
 
@@ -211,19 +212,20 @@ func TestPackagePyPI(t *testing.T) {
 		assert.Equal(t, int64(2), pvs[0].DownloadCount)
 	})
 
-	t.Run("PackageMetadata", func(t *testing.T) {
+	hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256))
+
+	t.Run("PackageMetadataHTML", func(t *testing.T) {
 		defer tests.PrintCurrentTest(t)()
 
 		req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)).
 			AddBasicAuth(user.Name)
+		req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+html"}
 		resp := MakeRequest(t, req, http.StatusOK)
 
 		htmlDoc := NewHTMLParser(t, resp.Body)
 		nodes := htmlDoc.doc.Find("a").Nodes
 		assert.Len(t, nodes, 2)
 
-		hrefMatcher := regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\..+#sha256=%s`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion), hashSHA256))
-
 		for _, a := range nodes {
 			for _, att := range a.Attr {
 				switch att.Key {
@@ -237,4 +239,24 @@ func TestPackagePyPI(t *testing.T) {
 			}
 		}
 	})
+
+	t.Run("PackageMetadataJSON", func(t *testing.T) {
+		defer tests.PrintCurrentTest(t)()
+
+		req := NewRequest(t, "GET", fmt.Sprintf("%s/simple/%s", root, packageName)).
+			AddBasicAuth(user.Name)
+		req.Header["Accept"] = []string{"application/vnd.pypi.simple.v1+json"}
+		resp := MakeRequest(t, req, http.StatusOK)
+		assert.Greater(t, resp.Body.Len(), 3)
+		txt := make([]byte, resp.Body.Len())
+		resp.Body.Read(txt)
+		var obj pypi.PackageJSON
+		require.NoError(t, json.Unmarshal(txt, &obj))
+		assert.Equal(t, packageName, obj.Name)
+		assert.Equal(t, pypi.PackageMetaJSON{APIVersion: "1.4"}, obj.Meta)
+		for _, filed := range obj.Files {
+			hrefMatcher = regexp.MustCompile(fmt.Sprintf(`%s/files/%s/%s/test\.(tar\.gz)|(whl)`, root, regexp.QuoteMeta(packageName), regexp.QuoteMeta(packageVersion)))
+			assert.Regexp(t, hrefMatcher, filed.URL[21:])
+		}
+	})
 }