Config finder make target (#1328)

* Add config-finder make target

* Add recursive functionality

* Add config finder to CI

* Workaround bash argument limit failures

Author: abejgonzalez

.github/workflows/chipyard-full-flow.yml

@@ -81,6 +81,19 @@ jobs:
           export MAKEFLAGS="-j32"
           ./build-setup.sh -f
 
+  run-cfg-finder:
+    name: run-cfg-finder
+    needs: [setup-repo]
+    runs-on: ferry
+    steps:
+      - name: Run config finder
+        run: |
+          cd ${{ env.REMOTE_WORK_DIR }}
+          eval "$(conda shell.bash hook)"
+          source env.sh
+          cd sims/verilator
+          make find-config-fragments
+
   run-tutorial:
     name: run-tutorial
     needs: [setup-repo]

common.mk

@@ -48,6 +48,7 @@ HELP_COMMANDS += \
 "   run-tests                   = run all assembly and benchmark tests" \
 "   launch-sbt                  = start sbt terminal" \
 "   {shutdown,start}-sbt-server = shutdown or start sbt server if using ENABLE_SBT_THIN_CLIENT" \
+"   find-config-fragments       = list all config. fragments and their locations (recursive up to CONFIG_FRAG_LEVELS=$(CONFIG_FRAG_LEVELS))"
 
 #########################################################################################
 # include additional subproject make fragments
@@ -388,8 +389,21 @@ start-sbt-server: check-thin-client
 	cd $(base_dir) && $(SBT) "exit"
 
 #########################################################################################
-# print help text
+# print help text (and other help)
 #########################################################################################
+# helper to add newlines (avoid bash argument too long)
+define \n
+
+
+endef
+
+CONFIG_FRAG_LEVELS ?= 3
+.PHONY: find-config-fragments
+find-config-fragments: $(SCALA_SOURCES)
+	rm -rf /tmp/scala_files.f
+	@$(foreach file,$(SCALA_SOURCES),echo $(file) >> /tmp/scala_files.f${\n})
+	$(base_dir)/scripts/config-finder.py -l $(CONFIG_FRAG_LEVELS) /tmp/scala_files.f
+
 .PHONY: help
 help:
 	@for line in $(HELP_LINES); do echo "$$line"; done

docs/Customization/Keys-Traits-Configs.rst

@@ -75,3 +75,9 @@ We can use this config fragment when composing our configs.
 
 .. note::
    Readers who want more information on the configuration system may be interested in reading :ref:`cdes`.
+
+Chipyard Config Fragments
+-------------------------
+
+For discoverability, users can run ``make find-config-fragments`` to see a list of config. fragments
+(config. fragments that match "class NAME extends CONFIG\n" on a single line and a subset of their children) and their file path in a fully initialized Chipyard repository.

scripts/config-finder.py

@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+from collections import defaultdict
+import re
+from copy import deepcopy
+import os
+
+cy_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+# from https://gist.github.com/angstwad/bf22d1822c38a92ec0a9
+def deep_merge(a: dict, b: dict) -> dict:
+    """Merge two dicts and return a singular dict"""
+    result = deepcopy(a)
+    for bk, bv in b.items():
+        av = result.get(bk)
+        if isinstance(av, dict) and isinstance(bv, dict):
+            result[bk] = deep_merge(av, bv)
+        else:
+            result[bk] = deepcopy(bv)
+    return result
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description='Pretty print all configs given a filelist of scala files')
+  parser.add_argument('FILE', type=str, help='Filelist of scala files to search within')
+  parser.add_argument('-l', '--levels', default=0, type=int, help='Number of levels to recursively look for configs')
+  args = parser.parse_args()
+
+  files = []
+  with open(args.FILE, 'r') as f:
+    files = f.read().splitlines()
+
+  cmd = ['grep', '-o', r"class \+.* \+extends \+Config"] + files
+  r = subprocess.run(cmd, check=True, capture_output=True)
+
+  base_file_path_dict = defaultdict(list)
+  for l in r.stdout.decode("UTF-8").splitlines():
+    match = re.match(r"^(.*):class +([a-zA-Z_$][a-zA-Z\d_$]*).* +extends", l)
+    if match:
+      base_file_path_dict[match.group(1)].append(match.group(2))
+
+  levels = []
+  for level in range(args.levels):
+    if level == 0:
+      # use the base
+      dict_to_use = base_file_path_dict
+    else:
+      # use the level-1 dict
+      assert len(levels) > 0
+      dict_to_use = levels[-1]
+
+    file_path_dict = defaultdict(list)
+
+    for configs in dict_to_use.values():
+      for config in configs:
+        cmd = ['grep', '-o', r"class \+.* \+extends \+" + f"{config}"] + files
+        r = subprocess.run(cmd, capture_output=True)
+
+        for l in r.stdout.decode("UTF-8").splitlines():
+          match = re.match(r"^(.*):class +([a-zA-Z_$][a-zA-Z\d_$]*).* +extends", l)
+          if match:
+            file_path_dict[match.group(1)].append(match.group(2))
+
+    levels.append(file_path_dict)
+
+  final_dict = base_file_path_dict
+  for dct in levels:
+    final_dict = deep_merge(final_dict, dct)
+
+  print(f"Finding all one-line config. fragments (up to {args.levels} levels)\n")
+  for k, v in final_dict.items():
+    print(f"{k.replace(cy_path, 'chipyard')}:")
+    for e in v:
+      print(f"  {e}")
+    print("")