bugfix(cli): parse requirements.txt first and map packages to technique (CycloneDX#2030)

* parse requirements.txt first and map packages to technique

Signed-off-by: Omri Yoffe <[email protected]>

* lint

Signed-off-by: Omri Yoffe <[email protected]>

* review changes

Signed-off-by: Omri Yoffe <[email protected]>

* change confidence

Signed-off-by: Omri Yoffe <[email protected]>

* unit tests

Signed-off-by: Omri Yoffe <[email protected]>

* lint

Signed-off-by: Omri Yoffe <[email protected]>

---------

Signed-off-by: Omri Yoffe <[email protected]>

Author: omriyoffe-panw

lib/cli/index.js

@@ -3620,6 +3620,7 @@ export async function createPythonBom(path, options) {
   let dependencies = [];
   let pkgList = [];
   let formulationList = [];
+  const packageTechniqueMap = new Map();
   const tempDir = mkdtempSync(join(getTmpDir(), "cdxgen-venv-"));
   let parentComponent = createDefaultParentComponent(path, "pypi", options);
   // We are checking only the root here for pipenv
@@ -3851,6 +3852,10 @@ export async function createPythonBom(path, options) {
           const basePath = dirname(f);
           let reqData;
           let frozen = false;
+
+          reqData = readFileSync(f, { encoding: "utf-8" });
+          await parseReqFile(reqData, true, packageTechniqueMap);
+
           // Attempt to pip freeze in a virtualenv to improve precision
           if (options.installDeps) {
             // If there are multiple requirements files then the tree is getting constructed for each one
@@ -3862,6 +3867,35 @@ export async function createPythonBom(path, options) {
               parentComponent,
             );
             if (pkgMap.pkgList?.length) {
+              pkgMap.pkgList.forEach((pkg) => {
+                const existingTechnique = packageTechniqueMap.get(
+                  pkg.name.toLowerCase(),
+                );
+                if (existingTechnique) {
+                  // Update evidence to preserve original technique
+                  if (pkg.evidence?.identity?.methods) {
+                    pkg.evidence.identity.methods =
+                      pkg.evidence.identity.methods.map((method) => ({
+                        ...method,
+                        technique: existingTechnique,
+                      }));
+                  }
+                } else {
+                  // New transitive dependency - mark as manifest-analysis derived
+                  packageTechniqueMap.set(
+                    pkg.name.toLowerCase(),
+                    "manifest-analysis",
+                  );
+                  if (pkg.evidence?.identity?.methods) {
+                    pkg.evidence.identity.methods =
+                      pkg.evidence.identity.methods.map((method) => ({
+                        ...method,
+                        technique: "manifest-analysis",
+                      }));
+                  }
+                }
+              });
+
               pkgList = pkgList.concat(pkgMap.pkgList);
               frozen = pkgMap.frozen;
             }

lib/helpers/utils.js

@@ -5628,9 +5628,14 @@ export async function parsePyLockData(lockData, lockFile, pyProjectFile) {
  *
  * @param {Object} reqData Requirements.txt data
  * @param {Boolean} fetchDepsInfo Fetch dependencies info from pypi
+ * @param {Object} packageTechniqueMap Mapping of package techniques
  */
-export async function parseReqFile(reqData, fetchDepsInfo) {
-  const pkgList = [];
+export async function parseReqFile(
+  reqData,
+  fetchDepsInfo,
+  packageTechniqueMap = null,
+) {
+  let pkgList = [];
   let compScope;
   reqData
     .replace(/\r/g, "")
@@ -5788,7 +5793,33 @@ export async function parseReqFile(reqData, fetchDepsInfo) {
         }
       }
     });
-  return await getPyMetadata(pkgList, fetchDepsInfo);
+  const directDependencies = await getPyMetadata(pkgList, fetchDepsInfo);
+  if (packageTechniqueMap && directDependencies?.length) {
+    // Mark direct dependencies from requirements.txt as manifest-analysis
+    directDependencies.forEach((pkg) => {
+      packageTechniqueMap.set(pkg.name.toLowerCase(), "manifest-analysis");
+      // Also mark the evidence
+      if (!pkg.evidence) {
+        pkg.evidence = {
+          identity: {
+            field: "purl",
+            confidence: pkg.version ? 0.5 : 0.3,
+            methods: [],
+          },
+        };
+      }
+      if (!pkg.evidence.identity.methods) {
+        pkg.evidence.identity.methods = [];
+      }
+      pkg.evidence.identity.methods.push({
+        technique: "manifest-analysis",
+        confidence: pkg.version ? 0.5 : 0.3,
+        value: pkg["bom-ref"] || pkg.purl,
+      });
+    });
+    pkgList = pkgList.concat(directDependencies);
+  }
+  return directDependencies;
 }
 
 /**

lib/helpers/utils.test.js

@@ -4780,36 +4780,69 @@ test("parseGemspecData", async () => {
 });
 
 test("parse requirements.txt", async () => {
+  const packageTechniqueMap1 = new Map();
   let deps = await parseReqFile(
     readFileSync("./test/data/requirements.comments.txt", {
       encoding: "utf-8",
     }),
     false,
+    packageTechniqueMap1,
   );
   expect(deps.length).toEqual(31);
+  expect(packageTechniqueMap1.size).toEqual(31);
+  const packageTechniqueMap2 = new Map();
   deps = await parseReqFile(
     readFileSync("./test/data/requirements.freeze.txt", {
       encoding: "utf-8",
     }),
     false,
+    packageTechniqueMap2,
   );
   expect(deps.length).toEqual(113);
+  expect(packageTechniqueMap2.size).toEqual(113);
   expect(deps[0]).toEqual({
     name: "elasticsearch",
     version: "8.6.2",
     scope: "required",
+    evidence: {
+      identity: {
+        field: "purl",
+        confidence: 0.5,
+        methods: [
+          {
+            technique: "manifest-analysis",
+            confidence: 0.5,
+          },
+        ],
+      },
+    },
   });
+  const packageTechniqueMap3 = new Map();
   deps = await parseReqFile(
     readFileSync("./test/data/chen-science-requirements.txt", {
       encoding: "utf-8",
     }),
     false,
+    packageTechniqueMap3,
   );
   expect(deps.length).toEqual(87);
+  expect(packageTechniqueMap3.size).toEqual(87);
   expect(deps[0]).toEqual({
     name: "aiofiles",
     version: "23.2.1",
     scope: undefined,
+    evidence: {
+      identity: {
+        field: "purl",
+        confidence: 0.5,
+        methods: [
+          {
+            technique: "manifest-analysis",
+            confidence: 0.5,
+          },
+        ],
+      },
+    },
     properties: [
       {
         name: "cdx:pip:markers",
@@ -4818,22 +4851,49 @@ test("parse requirements.txt", async () => {
       },
     ],
   });
+  const packageTechniqueMap4 = new Map();
   deps = await parseReqFile(
     readFileSync("./test/data/requirements-lock.linux_py3.txt", {
       encoding: "utf-8",
     }),
     false,
+    packageTechniqueMap4,
   );
   expect(deps.length).toEqual(375);
+  expect(packageTechniqueMap4.size).toEqual(375);
   expect(deps[0]).toEqual({
     name: "adal",
     scope: undefined,
     version: "1.2.2",
+    evidence: {
+      identity: {
+        field: "purl",
+        confidence: 0.5,
+        methods: [
+          {
+            technique: "manifest-analysis",
+            confidence: 0.5,
+          },
+        ],
+      },
+    },
   });
   expect(deps[deps.length - 1]).toEqual({
     name: "zipp",
     scope: undefined,
     version: "0.6.0",
+    evidence: {
+      identity: {
+        field: "purl",
+        confidence: 0.5,
+        methods: [
+          {
+            technique: "manifest-analysis",
+            confidence: 0.5,
+          },
+        ],
+      },
+    },
   });
 });