feat(merge): Add filter for intermediate reports

README.md

@@ -54,6 +54,8 @@ which is useful if you're trying the latest code from
       -t TAG, --tag TAG     provide a tag to differentiate between multiple intermediate reports
       -m INTERMEDIATE_REPORTS, --merge INTERMEDIATE_REPORTS           
                             comma separated intermediate reports path for merging
+      -F TAGS, --filter TAGS           
+                            comma separated tags to filter out intermediate reports
       --html-theme HTML_THEME
                             provide custom theme directory for HTML Report
       -f {csv,json,console,html,pdf}, --format {csv,json,console,html,pdf}
@@ -83,7 +85,8 @@ Note that you can use `-i` or `--input-file` option to produce list of CVEs foun
 
 `-n` or `--nvd` allows you to use between the default `json` which fetches the yearly JSON feeds or you can use the NVD REST-based CVE Retrieval Interface using `api`
 
-You can also use `-m` or `--merge` along with `-f --format` and `-o --output-file` to generate output from intermediate reports in different formats.
+You can also use `-m` or `--merge` along with `-f --format` and `-o --output-file` to generate output from intermediate reports in different formats. 
+Use `-F --filter` along with `-m --merge`to filter out intermediate reports based on tag.
 
 > Note: For backward compatibility, we still support `csv2cve` command for producing CVEs from csv but we recommend using new `--input-file` command instead.
 

cve_bin_tool/cli.py

@@ -198,6 +198,12 @@ def main(argv=None):
         action=StringToListAction,
         help="comma separated intermediate reports path for merging",
     )
+    parser.add_argument(
+        "-F",
+        "--filter",
+        action=StringToListAction,
+        help="comma separated tag string for filtering intermediate reports",
+    )
 
     checker_group = parser.add_argument_group("Checkers")
     checker_group.add_argument(
@@ -238,6 +244,7 @@ def main(argv=None):
         "tag": "",
         "merge": None,
         "nvd": "json",
+        "filter": [],
         "affected_versions": 0,
     }
 
@@ -290,13 +297,19 @@ def main(argv=None):
         LOGGER.info(
             "You can use -f --format and -o --output-file for saving merged intermediate reports in a file"
         )
-        merged_reports = MergeReports(merge_files=args["merge"], score=score)
+        merged_reports = MergeReports(
+            merge_files=args["merge"], score=score, filter_tag=args["filter"]
+        )
         if args["input_file"]:
             LOGGER.warning(
                 "Ignoring -i --input-file while merging intermediate reports"
             )
             args["input_file"] = None
         merge_cve_scanner = merged_reports.merge_intermediate()
+    elif args["filter"] and not args["merge"]:
+        LOGGER.warning(
+            "Use -F --filter only when you want to filter out intermediate reports on the basis of tag"
+        )
 
     # Database update related settings
     # Connect to the database

cve_bin_tool/merge.py

@@ -40,6 +40,7 @@ def __init__(
         error_mode=ErrorMode.TruncTrace,
         cache_dir=DISK_LOCATION_DEFAULT,
         score=0,
+        filter_tag=[],
     ):
         self.logger = logger or LOGGER.getChild(self.__class__.__name__)
         self.merge_files = merge_files
@@ -54,6 +55,7 @@ def __init__(
         self.cache_dir = cache_dir
         self.merged_files = ["tag"]
         self.score = score
+        self.filter_tag = filter_tag
 
         self.walker = DirWalk(
             pattern=";".join(
@@ -104,7 +106,6 @@ def scan_intermediate_file(self, filename):
                             self.logger.info(
                                 f"Adding data from {os.path.basename(filename)} with timestamp {inter_data['metadata']['timestamp']}"
                             )
-                            self.total_inter_files += 1
                             inter_data["metadata"]["severity"] = get_severity_count(
                                 inter_data["report"]
                             )
@@ -122,7 +123,13 @@ def merge_intermediate(self):
 
         for inter_file in self.recursive_scan(self.merge_files):
             # Create a list of intermediate files dictionary
-            self.intermediate_cve_data.append(self.scan_intermediate_file(inter_file))
+            intermediate_data = self.scan_intermediate_file(inter_file)
+            if (
+                self.filter_tag == []
+                or intermediate_data["metadata"]["tag"] in self.filter_tag
+            ):
+                self.intermediate_cve_data.append(intermediate_data)
+                self.total_inter_files += 1
 
         if self.intermediate_cve_data:
             # sort on basis of timestamp and scans

doc/MANUAL.md

@@ -26,6 +26,7 @@
     - [-a INTERMEDIATE_PATH, --append INTERMEDIATE_PATH](#a-intermediate-path-append-intermediate-path)
     - [-t TAG, --tag TAG](#t-tag-tag-tag)
     - [-m INTERMEDIATE_REPORTS, --merge INTERMEDIATE_REPORTS](#m-intermediate-reports-merge-intermediate-reports)
+    - [-F TAGS, --filter TAGS](#f-tags-filter-tags)
     - [-o OUTPUT_FILE, --output-file OUTPUT_FILE](#o-output-file-output-file-output-file)
     - [--html-theme HTML_THEME](#html-theme-html-theme)
     - [-f {csv,json,console,html}, --format {csv,json,console,html}](#f-csv-json-console-html-format-csv-json-console-html)
@@ -83,6 +84,8 @@ which is useful if you're trying the latest code from
       -t TAG, --tag TAG     provide a tag to differentiate between multiple intermediate reports
       -m INTERMEDIATE_REPORTS, --merge INTERMEDIATE_REPORTS           
                             comma separated intermediate reports path for merging
+      -F TAGS, --filter TAGS           
+                            comma separated tags to filter out intermediate reports
       --html-theme HTML_THEME
                             provide custom theme directory for HTML Report
       -f {csv,json,console,html}, --format {csv,json,console,html}
@@ -548,6 +551,10 @@ This option allows you to save a tag inside the metadata of intermediate reports
 
 This option allows you to merge intermediate reports created using `-a` or `--append`. The output from the merged report produces a report on the console. But you can also use it along with `-f --format` and `-o --output-file` to produce output in other formats. It takes a list of comma-separated filepath.
 
+### -F TAGS, --filter TAGS
+
+This allows you to filter out intermediate reports based on the tag. This can be useful while merging multiple intermediate reports from a single path. See detailed guide on [`filter intermediate reports`](how_to_guides/filter_intermediate_reports.md) for more information.
+
 ### -o OUTPUT_FILE, --output-file OUTPUT_FILE
 
 This option allows you to specify the filename for the report, rather than having CVE Binary Tool generate it by itself.

doc/how_to_guides/filter_intermediate_reports.md

@@ -0,0 +1,45 @@
+# Filter Intermediate Reports
+
+Let's consider a case where a user has multiple intermediate reports stored at a single path. But the user wants to merge only selected reports. We can do this by filtering intermediate reports based on the tag.
+
+## How to filter Intermediate reports?
+
+Suppose you have multiple intermediate reports stored at a single path.
+
+```bash
+path
+├── intermediate.cve-bin-tool.2021-06-02.01-17-48.json[weekly]
+├── intermediate.cve-bin-tool.2021-06-09.02-39-43.json[weekly]
+├── intermediate.cve-bin-tool.2021-06-16.23-44-35.json[weekly]
+├── intermediate.cve-bin-tool.2021-06-23.23-44-49.json[weekly]
+├── intermediate.cve-bin-tool.2021-06-24.23-45-08.json[daily]
+├── intermediate.cve-bin-tool.2021-06-25.23-45-08.json[daily]
+└── intermediate.cve-bin-tool.2021-06-26.23-45-08.json[daily]
+
+0 directories, 7 files
+
+```
+Note: You can specify tags while generating intermediate reports using `-t --tag`. For this example, we have assumed that the intermediate reports contain tag {weekly or daily}
+
+We want to merge the weekly generated report to plot the week-wise timeline trace as an HTML report. We can have some other use cases as well.      
+To filter out the intermediate reports, use:
+```
+python -m cve_bin_tool.cli -F weekly -m /path -f html
+```
+This will generate a merged report from these files -
+
+```
+path
+├── intermediate.cve-bin-tool.2021-06-02.01-17-48.json[weekly]
+├── intermediate.cve-bin-tool.2021-06-09.02-39-43.json[weekly]
+├── intermediate.cve-bin-tool.2021-06-16.23-44-35.json[weekly]
+└── intermediate.cve-bin-tool.2021-06-23.23-44-49.json[weekly]
+```
+
+Alternatively, users can filter out intermediate reports which are generated by multiple teams if they have specified distinguishable tags.
+If we have intermediate reports {backend and frontend}. We can simply use:
+
+```
+python -m cve_bin_tool.cli -F backend -m /path 
+```
+This will generate a vulnerability report for all the backend related binaries.

doc/how_to_guides/index.rst

@@ -16,3 +16,4 @@ How To Guides
    multiple_scans_at_once
    use_incremental_updates
    use_intermediate_reports
+   filter_intermediate_reports