From d9c510e1e764130d398c429d5fc150f219467e18 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Fri, 5 Apr 2024 09:12:55 +0200 Subject: [PATCH 1/4] feat(nuget): support nuspec manifest download --- modules/packages/nuget/metadata.go | 21 ++++++----- routers/api/packages/nuget/nuget.go | 32 +++++++++++++++-- tests/integration/api_packages_nuget_test.go | 37 ++++++++++++++++++-- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/modules/packages/nuget/metadata.go b/modules/packages/nuget/metadata.go index 6769c514cc2ff..1e98ddffde40e 100644 --- a/modules/packages/nuget/metadata.go +++ b/modules/packages/nuget/metadata.go @@ -48,10 +48,11 @@ const maxNuspecFileSize = 3 * 1024 * 1024 // Package represents a Nuget package type Package struct { - PackageType PackageType - ID string - Version string - Metadata *Metadata + PackageType PackageType + ID string + Version string + Metadata *Metadata + NuspecContent *bytes.Buffer } // Metadata represents the metadata of a Nuget package @@ -138,8 +139,9 @@ func ParsePackageMetaData(r io.ReaderAt, size int64) (*Package, error) { // ParseNuspecMetaData parses a Nuspec file to retrieve the metadata of a Nuget package func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { + var nuspecBuf bytes.Buffer var p nuspecPackage - if err := xml.NewDecoder(r).Decode(&p); err != nil { + if err := xml.NewDecoder(io.TeeReader(r, &nuspecBuf)).Decode(&p); err != nil { return nil, err } @@ -212,10 +214,11 @@ func ParseNuspecMetaData(archive *zip.Reader, r io.Reader) (*Package, error) { } } return &Package{ - PackageType: packageType, - ID: p.Metadata.ID, - Version: toNormalizedVersion(v), - Metadata: m, + PackageType: packageType, + ID: p.Metadata.ID, + Version: toNormalizedVersion(v), + Metadata: m, + NuspecContent: &nuspecBuf, }, nil } diff --git a/routers/api/packages/nuget/nuget.go b/routers/api/packages/nuget/nuget.go index c28bc6c9d9233..09156ece6b9a9 100644 --- a/routers/api/packages/nuget/nuget.go +++ b/routers/api/packages/nuget/nuget.go @@ -388,7 +388,8 @@ func EnumeratePackageVersionsV3(ctx *context.Context) { ctx.JSON(http.StatusOK, resp) } -// https://docs.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg +// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-manifest-nuspec +// https://learn.microsoft.com/en-us/nuget/api/package-base-address-resource#download-package-content-nupkg func DownloadPackageFile(ctx *context.Context) { packageName := ctx.Params("id") packageVersion := ctx.Params("version") @@ -431,7 +432,7 @@ func UploadPackage(ctx *context.Context) { return } - _, _, err := packages_service.CreatePackageAndAddFile( + pv, _, err := packages_service.CreatePackageAndAddFile( ctx, &packages_service.PackageCreationInfo{ PackageInfo: packages_service.PackageInfo{ @@ -465,6 +466,33 @@ func UploadPackage(ctx *context.Context) { return } + nuspecBuf, err := packages_module.CreateHashedBufferFromReaderWithSize(np.NuspecContent, np.NuspecContent.Len()) + if err != nil { + apiError(ctx, http.StatusInternalServerError, err) + return + } + defer nuspecBuf.Close() + + _, err = packages_service.AddFileToPackageVersionInternal( + ctx, + pv, + &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: strings.ToLower(fmt.Sprintf("%s.nuspec", np.ID)), + }, + Data: nuspecBuf, + }, + ) + if err != nil { + switch err { + case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize: + apiError(ctx, http.StatusForbidden, err) + default: + apiError(ctx, http.StatusInternalServerError, err) + } + return + } + ctx.Status(http.StatusCreated) } diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 20dafd5cc799f..991f37fe742dc 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -112,6 +112,20 @@ func TestPackageNuGet(t *testing.T) { return &buf } + nuspec := ` + + + ` + packageName + ` + ` + packageVersion + ` + ` + packageAuthors + ` + ` + packageDescription + ` + + + + + + + ` content, _ := io.ReadAll(createPackage(packageName, packageVersion)) url := fmt.Sprintf("/api/packages/%s/nuget", user.Name) @@ -224,7 +238,7 @@ func TestPackageNuGet(t *testing.T) { pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeNuGet) assert.NoError(t, err) - assert.Len(t, pvs, 1) + assert.Len(t, pvs, 1, "Should have one version") pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0]) assert.NoError(t, err) @@ -235,7 +249,7 @@ func TestPackageNuGet(t *testing.T) { pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 1) + assert.Len(t, pfs, 2, "Should have 2 files: nuget and nuspec") assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) assert.True(t, pfs[0].IsLead) @@ -302,16 +316,27 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) - assert.Len(t, pfs, 3) + assert.Len(t, pfs, 4, "Should have 4 files: nupkg, snupkg, nuspec and pdb") for _, pf := range pfs { switch pf.Name { case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): + assert.True(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(414), pb.Size) case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion): assert.False(t, pf.IsLead) pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) assert.NoError(t, err) assert.Equal(t, int64(616), pb.Size) + case fmt.Sprintf("%s.nuspec", packageName): + assert.False(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(453), pb.Size) case symbolFilename: assert.False(t, pf.IsLead) @@ -353,6 +378,12 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) assert.Equal(t, content, resp.Body.Bytes()) + req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.nuspec", url, packageName, packageVersion, packageName)). + AddBasicAuth(user.Name) + resp = MakeRequest(t, req, http.StatusOK) + + assert.Equal(t, nuspec, resp.Body.String()) + checkDownloadCount(1) req = NewRequest(t, "GET", fmt.Sprintf("%s/package/%s/%s/%s.%s.snupkg", url, packageName, packageVersion, packageName, packageVersion)). From 52e70baa5fad04314b742e8a72f111aacfa60b60 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 17 Apr 2024 08:23:55 +0200 Subject: [PATCH 2/4] Update api_packages_nuget_test.go --- tests/integration/api_packages_nuget_test.go | 72 ++++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 991f37fe742dc..7aaa83063592f 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -90,43 +90,33 @@ func TestPackageNuGet(t *testing.T) { symbolFilename := "test.pdb" symbolID := "d910bb6948bd4c6cb40155bcf52c3c94" - createPackage := func(id, version string) io.Reader { + createNuspec := func(id, version string) string { + return ` + + + ` + id + ` + ` + version + ` + ` + packageAuthors + ` + ` + packageDescription + ` + + + + + + +` + } + + createPackage := func(id, version string) *bytes.Buffer { var buf bytes.Buffer archive := zip.NewWriter(&buf) w, _ := archive.Create("package.nuspec") - w.Write([]byte(` - - - ` + id + ` - ` + version + ` - ` + packageAuthors + ` - ` + packageDescription + ` - - - - - - - `)) + w.Write([]byte(createNuspec(id, version))) archive.Close() return &buf } - nuspec := ` - - - ` + packageName + ` - ` + packageVersion + ` - ` + packageAuthors + ` - ` + packageDescription + ` - - - - - - - ` - content, _ := io.ReadAll(createPackage(packageName, packageVersion)) + content := createPackage(packageName, packageVersion).Bytes() url := fmt.Sprintf("/api/packages/%s/nuget", user.Name) @@ -250,12 +240,20 @@ func TestPackageNuGet(t *testing.T) { pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID) assert.NoError(t, err) assert.Len(t, pfs, 2, "Should have 2 files: nuget and nuspec") - assert.Equal(t, fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion), pfs[0].Name) - assert.True(t, pfs[0].IsLead) - - pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID) - assert.NoError(t, err) - assert.Equal(t, int64(len(content)), pb.Size) + for _, pf := range pfs { + switch pf.Name { + case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): + assert.True(t, pf.IsLead) + + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) + assert.NoError(t, err) + assert.Equal(t, int64(len(content)), pb.Size) + case fmt.Sprintf("%s.nuspec", packageName): + assert.False(t, pf.IsLead) + default: + assert.Fail(t, "unexpected filename: %v", pf.Name) + } + } req = NewRequestWithBody(t, "PUT", url, bytes.NewReader(content)). AddBasicAuth(user.Name) @@ -382,7 +380,7 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) AddBasicAuth(user.Name) resp = MakeRequest(t, req, http.StatusOK) - assert.Equal(t, nuspec, resp.Body.String()) + assert.Equal(t, createNuspec(packageName, packageVersion), resp.Body.String()) checkDownloadCount(1) From 3845d9cf758ff12f9c65d99e28b90ef8d2196179 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 17 Apr 2024 14:08:28 +0000 Subject: [PATCH 3/4] fmt --- tests/integration/api_packages_nuget_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 7aaa83063592f..2e78ed6a22a63 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -244,7 +244,7 @@ func TestPackageNuGet(t *testing.T) { switch pf.Name { case fmt.Sprintf("%s.%s.nupkg", packageName, packageVersion): assert.True(t, pf.IsLead) - + pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) assert.NoError(t, err) assert.Equal(t, int64(len(content)), pb.Size) From 7550c5495232d3b039c24274e78ffb83e510fa03 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 17 Apr 2024 15:02:57 +0000 Subject: [PATCH 4/4] fix tests --- tests/integration/api_packages_nuget_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/api_packages_nuget_test.go b/tests/integration/api_packages_nuget_test.go index 2e78ed6a22a63..83947ff9671ec 100644 --- a/tests/integration/api_packages_nuget_test.go +++ b/tests/integration/api_packages_nuget_test.go @@ -322,7 +322,7 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) assert.NoError(t, err) - assert.Equal(t, int64(414), pb.Size) + assert.Equal(t, int64(412), pb.Size) case fmt.Sprintf("%s.%s.snupkg", packageName, packageVersion): assert.False(t, pf.IsLead) @@ -334,7 +334,7 @@ AAAjQmxvYgAAAGm7ENm9SGxMtAFVvPUsPJTF6PbtAAAAAFcVogEJAAAAAQAAAA==`) pb, err := packages.GetBlobByID(db.DefaultContext, pf.BlobID) assert.NoError(t, err) - assert.Equal(t, int64(453), pb.Size) + assert.Equal(t, int64(427), pb.Size) case symbolFilename: assert.False(t, pf.IsLead)