Expand optional tree (CycloneDX#1523)

* Expand optional tree

Signed-off-by: Prabhu Subramanian <[email protected]>

* Collect more properties

Signed-off-by: Prabhu Subramanian <[email protected]>

* Bump version

Signed-off-by: Prabhu Subramanian <[email protected]>

* More juice-shop edge cases

Signed-off-by: Prabhu Subramanian <[email protected]>

---------

Signed-off-by: Prabhu Subramanian <[email protected]>

Author: prabhu

.github/workflows/snapshot-tests.yml

@@ -2,19 +2,12 @@ name: Test BOM Snapshots
 
 on:
   workflow_dispatch:
-  pull_request:
-    branches:
-      - master
-  push:
-    branches:
-      - master
 
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.head_ref || github.run_id }}"
   cancel-in-progress: true
 
-
 jobs:
 
   test_non_dotnet:

bin/cdxgen.js

@@ -510,6 +510,15 @@ const applyAdvancedOptions = (options) => {
       options.installDeps = true;
       break;
   }
+  // When the user specifies source-code-analysis as a technique, then enable deep and evidence mode.
+  if (
+    options?.technique &&
+    Array.isArray(options.technique) &&
+    options?.technique?.includes("source-code-analysis")
+  ) {
+    options.deep = true;
+    options.evidence = true;
+  }
   return options;
 };
 applyAdvancedOptions(options);

deno.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "11.0.8",
+  "version": "11.0.9",
   "exports": "./lib/cli/index.js",
   "compilerOptions": {
     "lib": ["deno.window"],

jsr.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "11.0.8",
+  "version": "11.0.9",
   "exports": "./lib/cli/index.js",
   "include": ["*.js", "lib/**", "bin/**", "data/**", "types/**"],
   "exclude": [

lib/helpers/utils.js

@@ -1038,7 +1038,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
   if (!options) {
     options = {};
   }
-
+  const pkgSpecVersionCache = {};
   if (!existsSync(pkgLockFile)) {
     return {
       pkgList,
@@ -1051,6 +1051,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
     rootNode,
     parentRef = null,
     visited = new Set(),
+    pkgSpecVersionCache = {},
     options = {},
   ) => {
     if (visited.has(node)) {
@@ -1119,6 +1120,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         author: authorString,
         scope: scope,
         _integrity: integrity,
+        externalReferences: [],
         properties: [
           {
             name: "SrcFile",
@@ -1155,11 +1157,88 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
           value: node.location,
         });
       }
+      if (node?.installLinks) {
+        pkg.properties.push({
+          name: "cdx:npm:installLinks",
+          value: "true",
+        });
+      }
+      if (node?.binPaths?.length) {
+        pkg.properties.push({
+          name: "cdx:npm:binPaths",
+          value: node.binPaths.join(", "),
+        });
+      }
+      if (node?.hasInstallScript) {
+        pkg.properties.push({
+          name: "cdx:npm:hasInstallScript",
+          value: "true",
+        });
+      }
+      if (node?.isLink) {
+        pkg.properties.push({
+          name: "cdx:npm:isLink",
+          value: "true",
+        });
+      }
+      // This getter method could fail with errors at times.
+      // Example Error: Invalid tag name "^>=6.0.0" of package "^>=6.0.0": Tags may not have any characters that encodeURIComponent encodes.
+      try {
+        if (!node?.isRegistryDependency) {
+          pkg.properties.push({
+            name: "cdx:npm:isRegistryDependency",
+            value: "false",
+          });
+        }
+      } catch (err) {
+        // ignore
+      }
+      if (node?.isWorkspace) {
+        pkg.properties.push({
+          name: "cdx:npm:isWorkspace",
+          value: "true",
+        });
+      }
+      if (node?.inBundle) {
+        pkg.properties.push({
+          name: "cdx:npm:inBundle",
+          value: "true",
+        });
+      }
+      if (node?.inDepBundle) {
+        pkg.properties.push({
+          name: "cdx:npm:inDepBundle",
+          value: "true",
+        });
+      }
+      if (node.package?.repository?.url) {
+        pkg.externalReferences.push({
+          type: "vcs",
+          url: node.package.repository.url,
+        });
+      }
+      if (node.package?.bugs?.url) {
+        pkg.externalReferences.push({
+          type: "issue-tracker",
+          url: node.package.bugs.url,
+        });
+      }
+      if (node?.package?.keywords?.length) {
+        pkg.tags = Array.isArray(node.package.keywords)
+          ? node.package.keywords.sort()
+          : node.package.keywords.split(",");
+      }
     }
-    const packageLicense = node.package.license;
-    if (packageLicense) {
+    if (node.package?.license) {
       // License will be overridden if shouldFetchLicense() is enabled
-      pkg.license = packageLicense;
+      pkg.license = node.package.license;
+    }
+    const deprecatedMessage = node.package?.deprecated;
+    if (deprecatedMessage) {
+      pkg.properties.push({
+        name: "cdx:npm:deprecated",
+        value: deprecatedMessage,
+      });
     }
     pkgList.push(pkg);
 
@@ -1170,7 +1249,14 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
         const {
           pkgList: childPkgList,
           dependenciesList: childDependenciesList,
-        } = parseArboristNode(workspaceNode, rootNode, purlString, visited);
+        } = parseArboristNode(
+          workspaceNode,
+          rootNode,
+          purlString,
+          visited,
+          pkgSpecVersionCache,
+          options,
+        );
         pkgList = pkgList.concat(childPkgList);
         dependenciesList = dependenciesList.concat(childDependenciesList);
         const depWorkspacePurlString = decodeURIComponent(
@@ -1193,8 +1279,9 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
 
     // this handles the case when a node has ["dependencies"] key in a package-lock.json
     // for a node. We exclude the root node because it's already been handled
+    // If the node has "requires", we don't have to track the "dependencies"
     const childrenDependsOn = [];
-    if (node !== rootNode) {
+    if (node !== rootNode && !node.edgesOut.size) {
       for (const child of node.children) {
         const childNode = child[1];
         const {
@@ -1205,6 +1292,8 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
           rootNode,
           decodeURIComponent(purlString),
           visited,
+          pkgSpecVersionCache,
+          options,
         );
         pkgList = pkgList.concat(childPkgList);
         dependenciesList = dependenciesList.concat(childDependenciesList);
@@ -1232,6 +1321,10 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
       let targetVersion;
       let targetName;
       let foundMatch = false;
+      // This cache is required to help us down the line.
+      if (edge?.to?.version && edge?.spec) {
+        pkgSpecVersionCache[`${edge.name}-${edge.spec}`] = edge.to.version;
+      }
       // if the edge doesn't have an integrity, it's likely a peer dependency
       // which isn't installed
       // Bug #795. At times, npm loses the integrity node completely and such packages are getting missed out
@@ -1285,11 +1378,43 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
             break;
           }
         }
+        if (!targetVersion || !targetName) {
+          if (pkgSpecVersionCache[`${edge.name}-${edge.spec}`]) {
+            targetVersion = pkgSpecVersionCache[`${edge.name}-${edge.spec}`];
+            targetName = edge.name;
+          }
+        }
       }
 
       // if we can't find the version of the edge, continue
       // it may be an optional peer dependency
       if (!targetVersion || !targetName) {
+        if (
+          DEBUG_MODE &&
+          !options.deep &&
+          !["optional", "peer", "peerOptional"].includes(edge?.type)
+        ) {
+          if (!targetVersion) {
+            console.log(
+              `Unable to determine the version for the dependency ${edge.name} from the path ${edge?.from?.path}. This is likely an edge case that is not handled.`,
+              edge,
+            );
+          } else if (!targetName) {
+            console.log(
+              `Unable to determine the name for the dependency from the edge from the path ${edge?.from?.path}. This is likely an edge case that is not handled.`,
+              edge,
+            );
+          }
+        }
+        // juice-shop
+        // Lock files created with --legacy-peer-deps will have certain peer dependencies missing
+        // This flags any non-missing peers
+        if (DEBUG_MODE && edge?.type === "peer" && edge?.error !== "MISSING") {
+          console.log(
+            `Unable to determine the version for the dependency ${edge.name} from the path ${edge?.from?.path}. This is likely an edge case that is not handled.`,
+            edge,
+          );
+        }
         continue;
       }
       const depPurlString = decodeURIComponent(
@@ -1309,6 +1434,8 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
           rootNode,
           decodeURIComponent(purlString),
           visited,
+          pkgSpecVersionCache,
+          options,
         );
       pkgList = pkgList.concat(childPkgList);
       dependenciesList = dependenciesList.concat(childDependenciesList);
@@ -1332,7 +1459,24 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
   });
   let tree = undefined;
   try {
-    tree = await arb.loadVirtual();
+    const rootNodeModulesDir = join(path.dirname(pkgLockFile), "node_modules");
+    if (existsSync(rootNodeModulesDir)) {
+      if (options.deep) {
+        console.log(
+          `Constructing the actual dependency hierarchy from ${rootNodeModulesDir}.`,
+        );
+        tree = await arb.loadActual();
+      } else {
+        if (DEBUG_MODE) {
+          console.log(
+            "Constructing virtual dependency tree based on the lock file. Pass --deep argument to construct the actual dependency tree from disk.",
+          );
+        }
+        tree = await arb.loadVirtual();
+      }
+    } else {
+      tree = await arb.loadVirtual();
+    }
   } catch (e) {
     console.log(
       `Unable to parse ${pkgLockFile} without legacy peer dependencies. Retrying ...`,
@@ -1364,6 +1508,7 @@ export async function parsePkgLock(pkgLockFile, options = {}) {
     tree,
     null,
     new Set(),
+    pkgSpecVersionCache,
     options,
   ));
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cyclonedx/cdxgen",
-  "version": "11.0.8",
+  "version": "11.0.9",
   "description": "Creates CycloneDX Software Bill of Materials (SBOM) from source or container image",
   "homepage": "http://github.com/cyclonedx/cdxgen",
   "author": "Prabhu Subramanian <[email protected]>",