From e6e3d08ea6eaa683b0c5d42e1ccad88dfaedf3d7 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Mon, 3 Apr 2017 18:48:03 +1000 Subject: [PATCH 1/8] Add extra_targets.json support to build tools If the file extra_targets.json exists in the working directory load it over the top of the built in targets.json for defining new and overriding built in mbed target definitions. --- tools/targets/__init__.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index f46907422e3..e133db8986b 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -129,8 +129,15 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or @cached def get_json_target_data(): """Load the description of JSON target data""" - return json_file_to_dict(Target.__targets_json_location or - Target.__targets_json_location_default) + targets = json_file_to_dict(Target.__targets_json_location or + Target.__targets_json_location_default) + + # If extra_targets.json exists in working directory load it over the top + extra = os.path.join('.', 'extra_targets.json') + if os.path.exists(extra): + targets.update(json_file_to_dict(extra)) + + return targets @staticmethod def set_targets_json_location(location=None): From 58c52fa2e7a2e1787f542ecbd570f29844c75f2f Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Tue, 4 Apr 2017 11:26:02 +1000 Subject: [PATCH 2/8] Recursively merge extra_targets into targets Recursively merge any target configs in extra_targets.json rather than completely replacing keys at the top level --- tools/targets/__init__.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index e133db8986b..cdd114aa254 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -22,7 +22,7 @@ import inspect import sys from copy import copy -from collections import namedtuple +from collections import namedtuple, Mapping from tools.targets.LPC import patch from tools.paths import TOOLS_BOOTLOADERS from tools.utils import json_file_to_dict @@ -125,6 +125,23 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or # Current/new location of the 'targets.json' file __targets_json_location = None + @staticmethod + def _merge_dict(dct, merge_dct): + """ Recursive dict merge. Inspired by `dict.update()` however instead of + updating only top-level keys, dict_merge recurses down into dicts nested + to an arbitrary depth, updating keys. + The provided ``merge_dct`` is merged into ``dct`` in place. + :param dct: dict onto which the merge is executed + :param merge_dct: dct merged into dct + :return: None + """ + for k, v in merge_dct.iteritems(): + if (k in dct and isinstance(dct[k], dict) + and isinstance(merge_dct[k], Mapping)): + Target._merge_dict(dct[k], merge_dct[k]) + else: + dct[k] = merge_dct[k] + @staticmethod @cached def get_json_target_data(): @@ -135,7 +152,7 @@ def get_json_target_data(): # If extra_targets.json exists in working directory load it over the top extra = os.path.join('.', 'extra_targets.json') if os.path.exists(extra): - targets.update(json_file_to_dict(extra)) + Target._merge_dict(targets, json_file_to_dict(extra)) return targets From 2c4475cacccd1ddca787d6558e1cece203becf91 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Thu, 6 Apr 2017 14:40:38 -0500 Subject: [PATCH 3/8] Find extra targets in all source folders --- tools/build.py | 3 ++- tools/get_config.py | 3 ++- tools/make.py | 3 ++- tools/options.py | 25 ++++++++++++++++++------ tools/project.py | 5 +++-- tools/targets/__init__.py | 41 +++++++++++++++++++++------------------ tools/test.py | 4 ++-- 7 files changed, 52 insertions(+), 32 deletions(-) diff --git a/tools/build.py b/tools/build.py index 52c034ca132..1d0282a973d 100644 --- a/tools/build.py +++ b/tools/build.py @@ -32,6 +32,7 @@ from tools.targets import TARGET_NAMES, TARGET_MAP from tools.options import get_default_options_parser from tools.options import extract_profile +from tools.options import extract_mcus from tools.build_api import build_library, build_mbed_libs, build_lib from tools.build_api import mcu_toolchain_matrix from tools.build_api import print_build_results @@ -134,7 +135,7 @@ # Get target list - targets = options.mcu if options.mcu else TARGET_NAMES + targets = extract_mcus(parser, options) if options.mcu else TARGET_NAMES # Get toolchains list toolchains = options.tool if options.tool else TOOLCHAINS diff --git a/tools/get_config.py b/tools/get_config.py index 9782fc16aeb..b3d4d7f3b50 100644 --- a/tools/get_config.py +++ b/tools/get_config.py @@ -26,6 +26,7 @@ from tools.utils import args_error from tools.options import get_default_options_parser +from tools.options import extract_mcus from tools.build_api import get_config from config import Config from utils import argparse_filestring_type @@ -49,7 +50,7 @@ # Target if options.mcu is None : args_error(parser, "argument -m/--mcu is required") - target = options.mcu[0] + target = extract_mcus(parser, options)[0] # Toolchain if options.tool is None: diff --git a/tools/make.py b/tools/make.py index 037bbdf2892..542db92de3f 100644 --- a/tools/make.py +++ b/tools/make.py @@ -42,6 +42,7 @@ from tools.targets import TARGET_MAP from tools.options import get_default_options_parser from tools.options import extract_profile +from tools.options import extract_mcus from tools.build_api import build_project from tools.build_api import mcu_toolchain_matrix from tools.build_api import mcu_toolchain_list @@ -200,7 +201,7 @@ # Target if options.mcu is None : args_error(parser, "argument -m/--mcu is required") - mcu = options.mcu[0] + mcu = extract_mcus(parser, options)[0] # Toolchain if options.tool is None: diff --git a/tools/options.py b/tools/options.py index b4f7c2d258c..642a5f4af02 100644 --- a/tools/options.py +++ b/tools/options.py @@ -17,9 +17,9 @@ from json import load from os.path import join, dirname from os import listdir -from argparse import ArgumentParser +from argparse import ArgumentParser, ArgumentTypeError from tools.toolchains import TOOLCHAINS -from tools.targets import TARGET_NAMES +from tools.targets import TARGET_NAMES, Target, update_target_data from tools.utils import argparse_force_uppercase_type, \ argparse_lowercase_hyphen_type, argparse_many, \ argparse_filestring_type, args_error, argparse_profile_filestring_type,\ @@ -47,10 +47,7 @@ def get_default_options_parser(add_clean=True, add_options=True, parser.add_argument("-m", "--mcu", help=("build for the given MCU (%s)" % ', '.join(targetnames)), - metavar="MCU", - type=argparse_many( - argparse_force_uppercase_type( - targetnames, "MCU"))) + metavar="MCU") parser.add_argument("-t", "--tool", help=("build using the given TOOLCHAIN (%s)" % @@ -130,3 +127,19 @@ def mcu_is_enabled(parser, mcu): "See https://developer.mbed.org/platforms/Renesas-GR-PEACH/#important-notice " "for more information") % (mcu, mcu)) return True + +def extract_mcus(parser, options): + try: + extra_targets = [join(src, "custom_targets.json") for src in options.source_dir] + for filename in extra_targets: + Target.add_extra_targets(filename) + update_target_data() + except KeyError: + pass + targetnames = TARGET_NAMES + targetnames.sort() + try: + return argparse_many(argparse_force_uppercase_type(targetnames, "MCU"))(options.mcu) + except ArgumentTypeError as exc: + args_error(parser, "argument -m/--mcu: {}".format(str(exc))) + diff --git a/tools/project.py b/tools/project.py index 2c14e62bde3..f9da9749851 100644 --- a/tools/project.py +++ b/tools/project.py @@ -20,7 +20,7 @@ from tools.utils import argparse_force_lowercase_type from tools.utils import argparse_force_uppercase_type from tools.utils import print_large_string -from tools.options import extract_profile, list_profiles +from tools.options import extract_profile, list_profiles, extract_mcus def setup_project(ide, target, program=None, source_dir=None, build=None, export_path=None): """Generate a name, if not provided, and find dependencies @@ -247,7 +247,8 @@ def main(): profile = extract_profile(parser, options, toolchain_name, fallback="debug") if options.clean: rmtree(BUILD_DIR) - export(options.mcu, options.ide, build=options.build, + mcu = extract_mcus(parser, options)[0] + export(mcu, options.ide, build=options.build, src=options.source_dir, macros=options.macros, project_id=options.program, zip_proj=zip_proj, build_profile=profile, app_config=options.app_config) diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index cdd114aa254..a1ab280b256 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -127,35 +127,35 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or @staticmethod def _merge_dict(dct, merge_dct): - """ Recursive dict merge. Inspired by `dict.update()` however instead of - updating only top-level keys, dict_merge recurses down into dicts nested - to an arbitrary depth, updating keys. - The provided ``merge_dct`` is merged into ``dct`` in place. - :param dct: dict onto which the merge is executed - :param merge_dct: dct merged into dct - :return: None - """ - for k, v in merge_dct.iteritems(): - if (k in dct and isinstance(dct[k], dict) - and isinstance(merge_dct[k], Mapping)): - Target._merge_dict(dct[k], merge_dct[k]) - else: - dct[k] = merge_dct[k] + """ Recursive dict merge. Inspired by `dict.update()` however instead of + updating only top-level keys, dict_merge recurses down into dicts nested + to an arbitrary depth, updating keys. + The provided ``merge_dct`` is merged into ``dct`` in place. + :param dct: dict onto which the merge is executed + :param merge_dct: dct merged into dct + :return: None + """ + for k, v in merge_dct.iteritems(): + if (k in dct and isinstance(dct[k], dict) + and isinstance(merge_dct[k], Mapping)): + Target._merge_dict(dct[k], merge_dct[k]) + else: + dct[k] = merge_dct[k] @staticmethod @cached def get_json_target_data(): """Load the description of JSON target data""" targets = json_file_to_dict(Target.__targets_json_location or - Target.__targets_json_location_default) + Target.__targets_json_location_default) + return targets - # If extra_targets.json exists in working directory load it over the top - extra = os.path.join('.', 'extra_targets.json') + @staticmethod + @cached + def add_extra_targets(extra): if os.path.exists(extra): Target._merge_dict(targets, json_file_to_dict(extra)) - return targets - @staticmethod def set_targets_json_location(location=None): """Set the location of the targets.json file""" @@ -561,6 +561,9 @@ def set_targets_json_location(location=None): # re-initialization does not create new variables, it keeps the old ones # instead. This ensures compatibility with code that does # "from tools.targets import TARGET_NAMES" + update_target_data() + +def update_target_data(): TARGETS[:] = [Target.get_target(tgt) for tgt, obj in Target.get_json_target_data().items() if obj.get("public", True)] diff --git a/tools/test.py b/tools/test.py index 0058c97d37c..88e87a17be7 100644 --- a/tools/test.py +++ b/tools/test.py @@ -28,7 +28,7 @@ from tools.config import ConfigException from tools.test_api import test_path_to_name, find_tests, print_tests, build_tests, test_spec_from_test_builds -from tools.options import get_default_options_parser, extract_profile +from tools.options import get_default_options_parser, extract_profile, extract_mcus from tools.build_api import build_project, build_library from tools.build_api import print_build_memory_usage from tools.build_api import merge_build_data @@ -114,7 +114,7 @@ # Target if options.mcu is None : args_error(parser, "argument -m/--mcu is required") - mcu = options.mcu[0] + mcu = extract_mcus(parser, options)[0] # Toolchain if options.tool is None: From bf08b108aa25f9a137dd00102873a03b292333c0 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Fri, 7 Apr 2017 11:31:44 +1000 Subject: [PATCH 4/8] Add custom_targets.json file contents to targets Avoid duplication of update_target_data() code Keep "custom_targets.json" filename definition in Targets() --- tools/options.py | 5 ++--- tools/targets/__init__.py | 41 +++++++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/tools/options.py b/tools/options.py index 642a5f4af02..a8f80f24ea6 100644 --- a/tools/options.py +++ b/tools/options.py @@ -130,9 +130,8 @@ def mcu_is_enabled(parser, mcu): def extract_mcus(parser, options): try: - extra_targets = [join(src, "custom_targets.json") for src in options.source_dir] - for filename in extra_targets: - Target.add_extra_targets(filename) + for source_dir in options.source_dir: + Target.add_extra_targets(source_dir) update_target_data() except KeyError: pass diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index a1ab280b256..7020ed52c8d 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -125,6 +125,9 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or # Current/new location of the 'targets.json' file __targets_json_location = None + # Extra custom targets files + __extra_target_json_files = [] + @staticmethod def _merge_dict(dct, merge_dct): """ Recursive dict merge. Inspired by `dict.update()` however instead of @@ -148,13 +151,18 @@ def get_json_target_data(): """Load the description of JSON target data""" targets = json_file_to_dict(Target.__targets_json_location or Target.__targets_json_location_default) + + for extra_target in Target.__extra_target_json_files: + Target._merge_dict(targets, json_file_to_dict(extra_target)) + return targets @staticmethod - @cached - def add_extra_targets(extra): - if os.path.exists(extra): - Target._merge_dict(targets, json_file_to_dict(extra)) + def add_extra_targets(source_dir): + extra_targets_file = os.path.join(source_dir, "custom_targets.json") + if os.path.exists(extra_targets_file): + Target.__extra_target_json_files.append(extra_targets_file) + CACHES.clear() @staticmethod def set_targets_json_location(location=None): @@ -531,14 +539,20 @@ def binary_hook(t_self, resources, elf, binf): ################################################################################ # Instantiate all public targets -TARGETS = [Target.get_target(name) for name, value - in Target.get_json_target_data().items() - if value.get("public", True)] +def update_target_data(): + TARGETS[:] = [Target.get_target(tgt) for tgt, obj + in Target.get_json_target_data().items() + if obj.get("public", True)] + # Map each target name to its unique instance + TARGET_MAP.clear() + TARGET_MAP.update(dict([(tgt.name, tgt) for tgt in TARGETS])) + TARGET_NAMES[:] = TARGET_MAP.keys() -# Map each target name to its unique instance -TARGET_MAP = dict([(t.name, t) for t in TARGETS]) +TARGETS = [] +TARGET_MAP = dict() +TARGET_NAMES = [] -TARGET_NAMES = TARGET_MAP.keys() +update_target_data() # Some targets with different name have the same exporters EXPORT_MAP = {} @@ -563,10 +577,3 @@ def set_targets_json_location(location=None): # "from tools.targets import TARGET_NAMES" update_target_data() -def update_target_data(): - TARGETS[:] = [Target.get_target(tgt) for tgt, obj - in Target.get_json_target_data().items() - if obj.get("public", True)] - TARGET_MAP.clear() - TARGET_MAP.update(dict([(tgt.name, tgt) for tgt in TARGETS])) - TARGET_NAMES[:] = TARGET_MAP.keys() From 2540f01d2633ae122317a2601f4de0939bc64270 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Fri, 7 Apr 2017 11:50:10 +1000 Subject: [PATCH 5/8] Handle situation when options.source_dir is None --- tools/options.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/options.py b/tools/options.py index a8f80f24ea6..8cfec663ac3 100644 --- a/tools/options.py +++ b/tools/options.py @@ -130,9 +130,10 @@ def mcu_is_enabled(parser, mcu): def extract_mcus(parser, options): try: - for source_dir in options.source_dir: - Target.add_extra_targets(source_dir) - update_target_data() + if options.source_dir: + for source_dir in options.source_dir: + Target.add_extra_targets(source_dir) + update_target_data() except KeyError: pass targetnames = TARGET_NAMES From 49645b44d871d70433f65cf9707579fbc1371f29 Mon Sep 17 00:00:00 2001 From: Jimmy Brisson Date: Fri, 7 Apr 2017 12:00:33 -0500 Subject: [PATCH 6/8] Clear custom-targets uppon setting new taregts.json location --- tools/targets/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index 7020ed52c8d..d1478c36db7 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -169,6 +169,7 @@ def set_targets_json_location(location=None): """Set the location of the targets.json file""" Target.__targets_json_location = (location or Target.__targets_json_location_default) + Target.__extra_target_json_files = [] # Invalidate caches, since the location of the JSON file changed CACHES.clear() From 6bd55a16fe5a10b400b3cee1cc2f93bf66147fc6 Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Tue, 9 May 2017 07:15:51 +1000 Subject: [PATCH 7/8] Add basic unit tests for custom_targets.json handling --- tools/test/targets/target_test.py | 101 +++++++++++++++++++++++++++++- 1 file changed, 99 insertions(+), 2 deletions(-) diff --git a/tools/test/targets/target_test.py b/tools/test/targets/target_test.py index 3227c925ad4..4a0f502a4c5 100644 --- a/tools/test/targets/target_test.py +++ b/tools/test/targets/target_test.py @@ -15,18 +15,23 @@ See the License for the specific language governing permissions and limitations under the License. """ - +import os import sys +import shutil +import tempfile from os.path import join, abspath, dirname +from contextlib import contextmanager import unittest # Be sure that the tools directory is in the search path + ROOT = abspath(join(dirname(__file__), "..", "..", "..")) sys.path.insert(0, ROOT) -from tools.targets import TARGETS +from tools.targets import TARGETS, TARGET_MAP, Target, update_target_data from tools.arm_pack_manager import Cache + class TestTargets(unittest.TestCase): def test_device_name(self): @@ -39,5 +44,97 @@ def test_device_name(self): "Target %s contains invalid device_name %s" % (target.name, target.device_name)) + @contextmanager + def temp_target_file(self, extra_target, json_filename='custom_targets.json'): + """Create an extra targets temp file in a context manager""" + tempdir = tempfile.mkdtemp() + try: + targetfile = os.path.join(tempdir, json_filename) + with open(targetfile, 'w') as f: + f.write(extra_target) + yield tempdir + finally: + # Reset extra targets + Target.set_targets_json_location() + # Delete temp files + shutil.rmtree(tempdir) + + def test_add_extra_targets(self): + """Search for extra targets json in a source folder""" + test_target_json = """ + { + "Test_Target": { + "inherits": ["Target"] + } + } + """ + with self.temp_target_file(test_target_json) as source_dir: + Target.add_extra_targets(source_dir=source_dir) + update_target_data() + + assert 'Test_Target' in TARGET_MAP + assert TARGET_MAP['Test_Target'].core is None, \ + "attributes should be inherited from Target" + + def test_modify_default_target(self): + """Set default targets file, then override base Target definition""" + initial_target_json = """ + { + "Target": { + "core": null, + "default_toolchain": "ARM", + "supported_toolchains": null, + "extra_labels": [], + "is_disk_virtual": false, + "macros": [], + "device_has": [], + "features": [], + "detect_code": [], + "public": false, + "default_lib": "std", + "bootloader_supported": false + }, + "Test_Target": { + "inherits": ["Target"], + "core": "Cortex-M4", + "supported_toolchains": ["ARM"] + } + }""" + + test_target_json = """ + { + "Target": { + "core": "Cortex-M0", + "default_toolchain": "GCC_ARM", + "supported_toolchains": null, + "extra_labels": [], + "is_disk_virtual": false, + "macros": [], + "device_has": [], + "features": [], + "detect_code": [], + "public": false, + "default_lib": "std", + "bootloader_supported": true + } + } + """ + + with self.temp_target_file(initial_target_json, json_filename="targets.json") as targets_dir: + Target.set_targets_json_location(os.path.join(targets_dir, "targets.json")) + update_target_data() + assert TARGET_MAP["Test_Target"].core == "Cortex-M4" + assert TARGET_MAP["Test_Target"].default_toolchain == 'ARM' + assert TARGET_MAP["Test_Target"].bootloader_supported == False + + with self.temp_target_file(test_target_json) as source_dir: + Target.add_extra_targets(source_dir=source_dir) + update_target_data() + + assert TARGET_MAP["Test_Target"].core == "Cortex-M4" + assert TARGET_MAP["Test_Target"].default_toolchain == 'GCC_ARM' + assert TARGET_MAP["Test_Target"].bootloader_supported == True + + if __name__ == '__main__': unittest.main() From 4491d2e3f710adbfd167b0b74bb5933af6ce268f Mon Sep 17 00:00:00 2001 From: Andrew Leech Date: Sat, 10 Jun 2017 21:45:14 +1000 Subject: [PATCH 8/8] Prevent modifying existing targets. A warning will be printed if it is attempted. --- tools/targets/__init__.py | 23 +++++------------------ tools/test/targets/target_test.py | 7 ++++--- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/tools/targets/__init__.py b/tools/targets/__init__.py index d1478c36db7..9d02174234f 100644 --- a/tools/targets/__init__.py +++ b/tools/targets/__init__.py @@ -128,23 +128,6 @@ class Target(namedtuple("Target", "name json_data resolution_order resolution_or # Extra custom targets files __extra_target_json_files = [] - @staticmethod - def _merge_dict(dct, merge_dct): - """ Recursive dict merge. Inspired by `dict.update()` however instead of - updating only top-level keys, dict_merge recurses down into dicts nested - to an arbitrary depth, updating keys. - The provided ``merge_dct`` is merged into ``dct`` in place. - :param dct: dict onto which the merge is executed - :param merge_dct: dct merged into dct - :return: None - """ - for k, v in merge_dct.iteritems(): - if (k in dct and isinstance(dct[k], dict) - and isinstance(merge_dct[k], Mapping)): - Target._merge_dict(dct[k], merge_dct[k]) - else: - dct[k] = merge_dct[k] - @staticmethod @cached def get_json_target_data(): @@ -153,7 +136,11 @@ def get_json_target_data(): Target.__targets_json_location_default) for extra_target in Target.__extra_target_json_files: - Target._merge_dict(targets, json_file_to_dict(extra_target)) + for k, v in json_file_to_dict(extra_target).iteritems(): + if k in targets: + print 'WARNING: Custom target "%s" cannot replace existing target.' % k + else: + targets[k] = v return targets diff --git a/tools/test/targets/target_test.py b/tools/test/targets/target_test.py index 4a0f502a4c5..0e555ca74b7 100644 --- a/tools/test/targets/target_test.py +++ b/tools/test/targets/target_test.py @@ -76,7 +76,7 @@ def test_add_extra_targets(self): assert TARGET_MAP['Test_Target'].core is None, \ "attributes should be inherited from Target" - def test_modify_default_target(self): + def test_modify_existing_target(self): """Set default targets file, then override base Target definition""" initial_target_json = """ { @@ -132,8 +132,9 @@ def test_modify_default_target(self): update_target_data() assert TARGET_MAP["Test_Target"].core == "Cortex-M4" - assert TARGET_MAP["Test_Target"].default_toolchain == 'GCC_ARM' - assert TARGET_MAP["Test_Target"].bootloader_supported == True + # The existing target should not be modified by custom targets + assert TARGET_MAP["Test_Target"].default_toolchain != 'GCC_ARM' + assert TARGET_MAP["Test_Target"].bootloader_supported != True if __name__ == '__main__':