Unify mbed OS tools testing style with what's used in mbed-ls, htrun, greentea

.travis.yml

@@ -19,14 +19,13 @@ script:
   - |
     find  -name "*.s" | tee BUILD/badasm | sed -e "s/^/Bad Assembler file name found: /" && [ ! -s BUILD/badasm ]
   - make -C events/equeue test clean
-  - PYTHONPATH=. python tools/test/config_test/config_test.py
-  - PYTHONPATH=. python tools/test/build_api/build_api_test.py
-  - PYTHONPATH=. python tools/test/targets/target_test.py
-  - python tools/test/pylint.py
-  - py.test tools/test/toolchains/api.py
-  - python tools/test/memap/memap_test.py
-  - python tools/project.py -S
-  - python tools/build_travis.py
+  - PYTHONPATH=. coverage run -a -m pytest tools/test
+  - python2 tools/test/pylint.py
+  - coverage run -a tools/project.py -S
+  - python2 tools/build_travis.py
+  - coverage html
+after_success:
+  - coveralls
 before_install:
   - sudo add-apt-repository -y ppa:team-gcc-arm-embedded/ppa
   - sudo add-apt-repository -y ppa:libreoffice/libreoffice-4-2
@@ -42,3 +41,5 @@ install:
   - pip install pylint
   - pip install hypothesis
   - pip install mock
+  - pip install coverage
+  - pip install coveralls

README.md

@@ -21,6 +21,8 @@ We run continuous integration on all of our branches and pull requests to verify
  - Master branch [![Master Branch CI Badge](https://travis-ci.org/ARMmbed/mbed-os.svg?branch=master)](https://travis-ci.org/ARMmbed/mbed-os)
  - Latest release [![Latest Tag CI Badge](https://travis-ci.org/ARMmbed/mbed-os.svg?branch=latest)](https://travis-ci.org/ARMmbed/mbed-os/branches)
 
+Tools coverage [![Coverage Status](https://coveralls.io/repos/github/ARMmbed/mbed-os/badge.svg?branch=master)](https://coveralls.io/github/ARMmbed/mbed-os?branch=master)
+
 ## Getting Started for Developers
 
 You need [mbed CLI](https://github.com/ARMmbed/mbed-cli) to build mbed OS. For more details, read the [mbed OS Handbook](https://docs.mbed.com/docs/mbed-os-handbook/en/latest/).

tools/test/config/app_override_libs/test_data.json

@@ -0,0 +1,9 @@
+{
+   "test_target": {
+        "lib1.p1": "v_p1_lib1_app",
+        "lib1.p2": "v_p2_lib1",
+        "lib1.p3": "v_p3_lib1",
+        "lib2.p1": "v_p1_lib2_app",
+        "lib2.p2": "v_p2_lib2"
+    }
+}

tools/test/config/bootloader_missing/test_data.json

@@ -0,0 +1,5 @@
+{
+    "K64F": {
+        "exception_msg": "not found"
+    }
+}

tools/test/config_test/test02/targets.json → tools/test/config/compound_inheritance/targets.json

@@ -1,5 +1,5 @@
 {
-    "b1": {
+    "base": {
         "extra_labels": [],
         "default_lib": "std",
         "core": "Cortex-M0",
@@ -9,8 +9,8 @@
             "base1_3": "v_base1_3_b1"
         }
     },
-    "d1": {
-        "inherits": ["b1"],
+    "left_middle": {
+        "inherits": ["base"],
         "config": {
             "derived1": "v_derived1_d1",
             "derived2": "v_derived2_d1"
@@ -20,8 +20,8 @@
             "base1_2": "v_base1_2_d1"
         }
     },
-    "b2": {
-        "inherits": ["b1"],
+    "right_middle": {
+        "inherits": ["base"],
         "config": {
             "base2_1": "v_base2_1_b2",
             "base2_2": "v_base2_2_b2"
@@ -30,8 +30,8 @@
             "base1_2": "v_base1_2_b2"
         }
     },
-    "f": {
-        "inherits": ["d1", "b2"],
+    "inherits_diamond": {
+        "inherits": ["left_middle", "right_middle"],
         "config": {
             "f1_1": "v_f1_1_f",
             "f1_2": "v_f1_2_f"

tools/test/config/compound_inheritance/test_data.json

@@ -0,0 +1,20 @@
+{
+    "inherits_diamond": {
+        "target.base1_1": "v_base1_1_f",
+        "target.base1_2": "v_base1_2_b2",
+        "target.base1_3": "v_base1_3_b1",
+        "target.derived1": "v_derived1_d1",
+        "target.derived2": "v_derived2_f",
+        "target.base2_1": "v_base2_1_f",
+        "target.base2_2": "v_base2_2_b2",
+        "target.f1_1": "v_f1_1_f_override",
+        "target.f1_2": "v_f1_2_f"
+    },
+    "right_middle": {
+        "target.base1_1": "v_base1_1_b1",
+        "target.base1_2": "v_base1_2_b2",
+        "target.base1_3": "v_base1_3_b1",
+        "target.base2_1": "v_base2_1_b2",
+        "target.base2_2": "v_base2_2_b2"
+    }
+}

tools/test/config/config_test.py

@@ -1,6 +1,6 @@
 """
 mbed SDK
-Copyright (c) 2016 ARM Limited
+Copyright (c) 2011-2017 ARM Limited
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -15,118 +15,161 @@
 limitations under the License.
 """
 
-import os.path
-import unittest
+import os
+import sys
+import json
+import pytest
 from mock import patch
-from tools.config import Config
-
-"""
-Tests for config.py
-"""
-
-class ConfigTests(unittest.TestCase):
+from hypothesis import given
+from hypothesis.strategies import sampled_from
+from os.path import join, isfile, dirname, abspath
+from tools.build_api import get_config
+from tools.targets import set_targets_json_location, Target, TARGET_NAMES
+from tools.config import ConfigException, Config
+
+def compare_config(cfg, expected):
+    """Compare the output of config against a dictionary of known good results
+
+    :param cfg: the configuration to check
+    :param expected: what to expect in that config
     """
-    Test cases for Config class
+    try:
+        for k in cfg:
+            if cfg[k].value != expected[k]:
+                return "'%s': expected '%s', got '%s'" % (k, expected[k], cfg[k].value)
+    except KeyError:
+        return "Unexpected key '%s' in configuration data" % k
+    for k in expected:
+        if k not in ["expected_macros", "expected_features"] + cfg.keys():
+            return "Expected key '%s' was not found in configuration data" % k
+    return ""
+
+def data_path(path):
+    """The expected data file for a particular test
+
+    :param path: the path to the test
     """
+    return join(path, "test_data.json")
 
-    def setUp(self):
-        """
-        Called before each test case
+def is_test(path):
+    """Does a directory represent a test?
 
-        :return:
-        """
-        self.target = "K64F"
+    :param path: the path to the test
+    """
+    return isfile(data_path(path))
 
-    def tearDown(self):
-        """
-        Called after each test case
+root_dir = abspath(dirname(__file__))
 
-        :return:
-        """
-        pass
+@pytest.mark.parametrize("name", filter(lambda d: is_test(join(root_dir, d)),
+                                        os.listdir(root_dir)))
+def test_config(name):
+    """Run a particular configuration test
 
-    @patch.object(Config, '_process_config_and_overrides')
-    @patch('tools.config.json_file_to_dict')
-    def test_init_app_config(self, mock_json_file_to_dict, _):
-        """
-        Test that the initialisation correctly uses app_config
+    :param name: test name (same as directory name)
+    """
+    test_dir = join(root_dir, name)
+    test_data = json.load(open(data_path(test_dir)))
+    targets_json = os.path.join(test_dir, "targets.json")
+    set_targets_json_location(targets_json if isfile(targets_json) else None)
+    for target, expected in test_data.items():
+        try:
+            cfg, macros, features = get_config(test_dir, target, "GCC_ARM")
+            res = compare_config(cfg, expected)
+            assert not(res), res
+            expected_macros = expected.get("expected_macros", None)
+            expected_features = expected.get("expected_features", None)
+
+            if expected_macros is not None:
+                macros = Config.config_macros_to_macros(macros)
+                assert sorted(expected_macros) == sorted(macros)
+            if expected_features is not None:
+                assert sorted(expected_features) == sorted(features)
+        except ConfigException as e:
+            err_msg = e.message
+            if "exception_msg" not in expected:
+                assert not(err_msg), "Unexpected Error: %s" % e
+            else:
+                assert expected["exception_msg"] in err_msg
+
+
+@pytest.mark.parametrize("target", ["K64F"])
+def test_init_app_config(target):
+    """
+    Test that the initialisation correctly uses app_config
 
-        :param mock_json_file_to_dict: mock of function json_file_to_dict
-        :param _: mock of function _process_config_and_overrides (not tested)
-        :return:
-        """
+    :param target: The target to use
+    """
+    set_targets_json_location()
+    with patch.object(Config, '_process_config_and_overrides'),\
+         patch('tools.config.json_file_to_dict') as mock_json_file_to_dict:
         app_config = "app_config"
-        mock_return = {'config': 'test'}
+        mock_return = {'config': {'test': False}}
         mock_json_file_to_dict.return_value = mock_return
 
-        config = Config(self.target, app_config=app_config)
+        config = Config(target, app_config=app_config)
 
         mock_json_file_to_dict.assert_called_with(app_config)
-        self.assertEqual(config.app_config_data, mock_return,
-                         "app_config_data should be set to the returned value")
+        assert config.app_config_data == mock_return
 
-    @patch.object(Config, '_process_config_and_overrides')
-    @patch('tools.config.json_file_to_dict')
-    def test_init_no_app_config(self, mock_json_file_to_dict, _):
-        """
-        Test that the initialisation works without app config
 
-        :param mock_json_file_to_dict: mock of function json_file_to_dict
-        :param _: patch of function _process_config_and_overrides (not tested)
-        :return:
-        """
-        config = Config(self.target)
+@pytest.mark.parametrize("target", ["K64F"])
+def test_init_no_app_config(target):
+    """
+    Test that the initialisation works without app config
+
+    :param target: The target to use
+    """
+    set_targets_json_location()
+    with patch.object(Config, '_process_config_and_overrides'),\
+         patch('tools.config.json_file_to_dict') as mock_json_file_to_dict:
+        config = Config(target)
 
         mock_json_file_to_dict.assert_not_called()
-        self.assertEqual(config.app_config_data, {},
-                         "app_config_data should be set an empty dictionary")
-
-    @patch.object(Config, '_process_config_and_overrides')
-    @patch('os.path.isfile')
-    @patch('tools.config.json_file_to_dict')
-    def test_init_no_app_config_with_dir(self, mock_json_file_to_dict, mock_isfile, _):
-        """
-        Test that the initialisation works without app config and with a
-        specified top level directory
-
-        :param mock_json_file_to_dict: mock of function json_file_to_dict
-        :param _: patch of function _process_config_and_overrides (not tested)
-        :return:
-        """
+        assert config.app_config_data == {}
+
+
+@pytest.mark.parametrize("target", ["K64F"])
+def test_init_no_app_config_with_dir(target):
+    """
+    Test that the initialisation works without app config and with a
+    specified top level directory
+
+    :param target: The target to use
+    """
+    set_targets_json_location()
+    with patch.object(Config, '_process_config_and_overrides'),\
+         patch('os.path.isfile') as mock_isfile, \
+         patch('tools.config.json_file_to_dict') as mock_json_file_to_dict:
         directory = '.'
         path = os.path.join('.', 'mbed_app.json')
-        mock_return = {'config': 'test'}
+        mock_return = {'config': {'test': False}}
         mock_json_file_to_dict.return_value = mock_return
         mock_isfile.return_value = True
 
-        config = Config(self.target, [directory])
+        config = Config(target, [directory])
 
         mock_isfile.assert_called_with(path)
         mock_json_file_to_dict.assert_called_once_with(path)
-        self.assertEqual(config.app_config_data, mock_return,
-                         "app_config_data should be set to the returned value")
-
-    @patch.object(Config, '_process_config_and_overrides')
-    @patch('tools.config.json_file_to_dict')
-    def test_init_override_app_config(self, mock_json_file_to_dict, _):
-        """
-        Test that the initialisation uses app_config instead of top_level_dir
-        when both are specified
-
-        :param mock_json_file_to_dict: mock of function json_file_to_dict
-        :param _: patch of function _process_config_and_overrides (not tested)
-        :return:
-        """
+        assert config.app_config_data == mock_return
+
+
+@pytest.mark.parametrize("target", ["K64F"])
+def test_init_override_app_config(target):
+    """
+    Test that the initialisation uses app_config instead of top_level_dir
+    when both are specified
+
+    :param target: The target to use
+    """
+    set_targets_json_location()
+    with patch.object(Config, '_process_config_and_overrides'),\
+         patch('tools.config.json_file_to_dict') as mock_json_file_to_dict:
         app_config = "app_config"
         directory = '.'
-        mock_return = {'config': 'test'}
+        mock_return = {'config': {'test': False}}
         mock_json_file_to_dict.return_value = mock_return
 
-        config = Config(self.target, [directory], app_config=app_config)
+        config = Config(target, [directory], app_config=app_config)
 
         mock_json_file_to_dict.assert_called_once_with(app_config)
-        self.assertEqual(config.app_config_data, mock_return,
-                         "app_config_data should be set to the returned value")
-
-if __name__ == '__main__':
-    unittest.main()
+        assert config.app_config_data == mock_return