Shaded namespace detection (#2311)

Signed-off-by: Prabhu Subramanian <[email protected]>

Author: prabhu

lib/helpers/utils.js

@@ -5138,7 +5138,12 @@ export async function getMvnMetadata(
         p.publisher = bodyJson?.organization?.name
           ? bodyJson?.organization.name._
           : "";
-        p.description = bodyJson?.description ? bodyJson.description._ : "";
+        p.description = bodyJson?.description
+          ? bodyJson.description._.replace(/[ \t]+/g, " ")
+              .replace(/^[ \t]+|[ \t]+$/gm, "")
+              .replace(/\n\s*\n/g, "\n")
+              .trim()
+          : "";
         if (bodyJson?.scm?.url) {
           p.repository = { url: bodyJson.scm.url._ };
         }
@@ -12961,7 +12966,7 @@ export async function extractJarArchive(jarFile, tempDir, jarNSMapping = {}) {
         let group = pomProperties["groupId"];
         let name = pomProperties["artifactId"];
         let version = pomProperties["version"];
-        let confidence = 1;
+        let confidence = 0.5;
         let technique = "manifest-analysis";
         if (
           (!group || !name || !version) &&
@@ -12970,7 +12975,7 @@ export async function extractJarArchive(jarFile, tempDir, jarNSMapping = {}) {
         ) {
           try {
             const sha = await checksumFile("sha1", jf);
-            const searchurl = `https://search.maven.org/solrsearch/select?q=1:%22${sha}%22&rows=20&wt=json`;
+            const searchurl = `https://central.sonatype.com/solrsearch/select?q=1:%22${sha}%22&rows=20&wt=json`;
             const res = await cdxgenAgent.get(searchurl, {
               responseType: "json",
               timeout: {
@@ -12992,12 +12997,13 @@ export async function extractJarArchive(jarFile, tempDir, jarNSMapping = {}) {
             }
           } catch (err) {
             if (err?.message && !err.message.includes("404")) {
-              if (err.message.includes("Timeout")) {
+              if (
+                err.message.includes("Timeout") ||
+                err.message.includes("429")
+              ) {
                 console.log(
                   "Maven search appears to be unavailable. Search will be skipped for all remaining packages.",
                 );
-              } else if (DEBUG_MODE) {
-                console.log(err);
               }
               search_maven_org_errors++;
             }
@@ -13083,18 +13089,74 @@ export async function extractJarArchive(jarFile, tempDir, jarNSMapping = {}) {
           if (!version) {
             confidence = 0;
           }
+          const properties = [
+            {
+              name: "SrcFile",
+              value: jf,
+            },
+          ];
+          const purl = new PackageURL(
+            "maven",
+            group,
+            name,
+            version,
+            { type: "jar" },
+            null,
+          ).toString();
+          let namespaceValues;
+          let namespaceList;
+          if (jarNSMapping?.[purl]?.namespaces) {
+            namespaceList = jarNSMapping[purl].namespaces;
+            namespaceValues = namespaceList.join("\n");
+            properties.push({
+              name: "Namespaces",
+              value: namespaceValues,
+            });
+          } else {
+            const tmpJarNSMapping = await collectJarNS(jf);
+            if (tmpJarNSMapping?.[jf]?.namespaces?.length) {
+              namespaceList = tmpJarNSMapping[jf].namespaces;
+              namespaceValues = namespaceList.join("\n");
+              properties.push({
+                name: "Namespaces",
+                value: namespaceValues,
+              });
+            }
+          }
+          // Are there any shaded classes
+          if (
+            namespaceValues?.includes(".shaded.") ||
+            namespaceValues?.includes(".thirdparty.com.")
+          ) {
+            properties.push({
+              name: "cdx:maven:shaded",
+              value: "true",
+            });
+            confidence = 0;
+            const unshadedNS = new Set();
+            for (const ans of namespaceList) {
+              let tmpns;
+              if (ans.includes(".shaded.")) {
+                tmpns = ans.split(".shaded.").pop();
+              } else if (ans.includes(".thirdparty.")) {
+                tmpns = ans.split(".thirdparty.").pop();
+              }
+              if (tmpns?.search("[.]") > 3) {
+                unshadedNS.add(tmpns.split("$")[0]);
+              }
+            }
+            if (unshadedNS.size) {
+              properties.push({
+                name: "cdx:maven:unshadedNamespaces",
+                value: Array.from(unshadedNS).join("\n"),
+              });
+            }
+          }
           const apkg = {
             group: group ? encodeForPurl(group) : "",
             name: name ? encodeForPurl(name) : "",
             version,
-            purl: new PackageURL(
-              "maven",
-              group,
-              name,
-              version,
-              { type: "jar" },
-              null,
-            ).toString(),
+            purl,
             evidence: {
               identity: {
                 field: "purl",
@@ -13108,27 +13170,8 @@ export async function extractJarArchive(jarFile, tempDir, jarNSMapping = {}) {
                 ],
               },
             },
-            properties: [
-              {
-                name: "SrcFile",
-                value: jf,
-              },
-            ],
+            properties,
           };
-          if (jarNSMapping?.[apkg.purl] && jarNSMapping[apkg.purl].namespaces) {
-            apkg.properties.push({
-              name: "Namespaces",
-              value: jarNSMapping[apkg.purl].namespaces.join("\n"),
-            });
-          } else {
-            const tmpJarNSMapping = await collectJarNS(jf);
-            if (tmpJarNSMapping?.[jf]?.namespaces?.length) {
-              apkg.properties.push({
-                name: "Namespaces",
-                value: tmpJarNSMapping[jf].namespaces.join("\n"),
-              });
-            }
-          }
           pkgList.push(apkg);
         } else {
           if (DEBUG_MODE) {