diff --git a/UNITTESTS/mbed_unittest.py b/UNITTESTS/mbed_unittest.py index e8e6eadad9c..627c58ed0ec 100755 --- a/UNITTESTS/mbed_unittest.py +++ b/UNITTESTS/mbed_unittest.py @@ -23,12 +23,14 @@ from __future__ import print_function import os import logging +import re from unit_test.options import get_options_parser, \ pretty_print_test_options from unit_test.settings import DEFAULT_CMAKE_GENERATORS from unit_test.test import UnitTestTool from unit_test.new import UnitTestGeneratorTool +from unit_test.coverage import CoverageAPI def _mbed_unittest_test(options, cwd, pwd): if options is None: @@ -80,14 +82,28 @@ def _mbed_unittest_test(options, cwd, pwd): tool.build_tests() if options.run_only: + tool.run_tests(filter_regex=options.test_regex) + # If code coverage generation: if options.coverage: - # Run tests and generate reports - tool.generate_coverage_report(coverage_output_type=options.coverage, - excludes=[pwd, options.build], - build_path=options.build) - else: - tool.run_tests(filter_regex=options.test_regex) # Only run tests + cov_api = CoverageAPI( + mbed_os_root=os.path.normpath(os.path.join(pwd, "..")), + build_dir=options.build) + + # Generate reports + outputs = [options.coverage] + if options.coverage == "both": + outputs = ["html", "xml"] + + excludes = [pwd, options.build] + + if not options.include_headers: + excludes.append(re.compile(".*\\.h")) + + cov_api.generate_reports(outputs=outputs, + excludes=excludes, + filter_regex=options.test_regex, + build_path=options.build) def _mbed_unittest_new(options, pwd): if options is None: diff --git a/UNITTESTS/unit_test/coverage.py b/UNITTESTS/unit_test/coverage.py new file mode 100644 index 00000000000..add8e1d2a9f --- /dev/null +++ b/UNITTESTS/unit_test/coverage.py @@ -0,0 +1,148 @@ +""" +Copyright (c) 2018, Arm Limited +SPDX-License-Identifier: Apache-2.0 + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +GENERATE TEST CODE COVERAGE +""" + +import os +import logging +import posixpath +import re + +from .utils import execute_program +from .get_tools import get_gcov_program, \ + get_gcovr_program +from .settings import COVERAGE_OUTPUT_TYPES + + +class CoverageAPI(object): + """ + Generate code coverage reports for unit tests. + """ + + def __init__(self, mbed_os_root=None, build_dir=None): + self.root = mbed_os_root + + if not self.root: + self.root = os.path.normpath(os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "../..")) + + self.build_dir = build_dir + + if not self.build_dir: + logging.error("No build directory given for CoverageAPI.") + + def _gen_cmd(self, coverage_type, excludes, filter_regex): + # Generate coverage generation command: + args = [get_gcovr_program(), + "--gcov-executable", + get_gcov_program(), + "-r", + os.path.relpath(self.root, self.build_dir), + "."] + + if coverage_type == "html": + args.extend(["--html", + "--html-detail", + "-o", + "./coverage/index.html"]) + elif coverage_type == "xml": + args.extend(["-x", + "-o", + "./coverage.xml"]) + else: + logging.error("Invalid coverage output type: %s" % coverage_type) + + # Add exclude filters: + for path in excludes: + # Use posix separators if path is string + if isinstance(path, ("".__class__, u"".__class__)): + path = path.replace("\\", "/") + args.extend(["-e", path]) + # Use regular expressions as is + elif isinstance(path, type(re.compile(""))): + args.extend(["-e", path.pattern]) + + # Add include filters: + if filter_regex: + filters = filter_regex.split(",") + + for filt in filters: + regex = "(.+/)?%s" % filt.replace("-", "/") + args.extend(["-f", regex]) + + if logging.getLogger().getEffectiveLevel() == logging.DEBUG: + args.append("-v") + + return args + + def generate_reports(self, + outputs, + excludes=None, + filter_regex=None, + build_path=None): + """ + Run tests to generate coverage data, and generate coverage reports. + + Positional arguments: + outputs - list of types to generate + + Keyword arguments: + excludes - list of paths to exclude from the report + filter_regex - comma-separated string to use for test filtering + build_path - build path + """ + + # Check for the tool + if get_gcovr_program() is None: + logging.error( + "No gcovr tool found in path. " + + "Cannot generate coverage reports.") + return + + if build_path is None: + build_path = os.getcwd() + + if outputs is None: + outputs = [] + + if excludes is None: + excludes = [] + + for output in outputs: + # Skip if invalid/unsupported output type + if output not in COVERAGE_OUTPUT_TYPES: + logging.warning( + "Invalid output type. " + + "Skip coverage report for type: %s." % output.upper()) + continue + + if output == "html": + # Create a build directory if not exist + coverage_path = os.path.join(build_path, "coverage") + if not os.path.exists(coverage_path): + os.mkdir(coverage_path) + + # Generate the command + args = self._gen_cmd(output, excludes, filter_regex) + + # Run the coverage tool + execute_program( + args, + "%s code coverage report generation failed." % output.upper(), + "%s code coverage report created." % output.upper()) diff --git a/UNITTESTS/unit_test/options.py b/UNITTESTS/unit_test/options.py index f192006cdfe..211c5d10b9b 100644 --- a/UNITTESTS/unit_test/options.py +++ b/UNITTESTS/unit_test/options.py @@ -21,7 +21,7 @@ import argparse import logging -from .settings import CMAKE_GENERATORS, MAKE_PROGRAMS, COVERAGE_TYPES +from .settings import CMAKE_GENERATORS, MAKE_PROGRAMS, COVERAGE_ARGS from .get_tools import get_make_tool def get_options_parser(): @@ -71,10 +71,15 @@ def get_options_parser(): dest="debug_build") parser.add_argument("--coverage", - choices=COVERAGE_TYPES, + choices=COVERAGE_ARGS, help="Generate code coverage report", dest="coverage") + parser.add_argument("--include-headers", + action="store_true", + help="Include headers to coverage reports, defaults to false.", + dest="include_headers") + parser.add_argument("-m", "--make-program", default=get_make_tool(), @@ -140,3 +145,4 @@ def pretty_print_test_options(options=None): if options.coverage: logging.info(" [%s] \tGenerate coverage reports", "SET") logging.info(" \t\t - Output: %s", options.coverage) + logging.info(" \t\t - Include headers: %s", options.include_headers) diff --git a/UNITTESTS/unit_test/settings.py b/UNITTESTS/unit_test/settings.py index 717cab85238..883b4a82f4a 100644 --- a/UNITTESTS/unit_test/settings.py +++ b/UNITTESTS/unit_test/settings.py @@ -29,10 +29,12 @@ "ninja": "Ninja" } -COVERAGE_TYPES = ["html", +COVERAGE_ARGS = ["html", "xml", "both"] +COVERAGE_OUTPUT_TYPES = ["html", "xml"] + CXX_COMPILERS = ["g++-6", "g++-8", "g++-7", "g++-5", "g++-4.9", "g++"] C_COMPILERS = ["gcc-6", "gcc-8", "gcc-7", "gcc-5", "gcc-4.9", "gcc"] diff --git a/UNITTESTS/unit_test/test.py b/UNITTESTS/unit_test/test.py index 6be991bb794..dbf515e4e0a 100644 --- a/UNITTESTS/unit_test/test.py +++ b/UNITTESTS/unit_test/test.py @@ -27,9 +27,7 @@ from .get_tools import get_make_tool, \ get_cmake_tool, \ get_cxx_tool, \ - get_c_tool, \ - get_gcov_program, \ - get_gcovr_program + get_c_tool from .settings import DEFAULT_CMAKE_GENERATORS class UnitTestTool(object): @@ -115,80 +113,6 @@ def build_tests(self): "Building unit tests failed.", "Unit tests built successfully.") - def _get_coverage_script(self, coverage_type, excludes): - args = [get_gcovr_program(), - "--gcov-executable", - get_gcov_program(), - "-r", - "../..", - "."] - - if coverage_type == "html": - args.extend(["--html", - "--html-detail", - "-o", - "./coverage/index.html"]) - elif coverage_type == "xml": - args.extend(["-x", - "-o", - "./coverage.xml"]) - - for path in excludes: - args.extend(["-e", path.replace("\\", "/")]) - - #Exclude header files from report - args.extend(["-e", ".*\.h"]) - - if logging.getLogger().getEffectiveLevel() == logging.DEBUG: - args.append("-v") - - return args - - def generate_coverage_report(self, - coverage_output_type=None, - excludes=None, - build_path=None): - """ - Run tests to generate coverage data, and generate coverage reports. - """ - - self.run_tests() - - if get_gcovr_program() is None: - logging.error("No gcovr tool found in path. \ - Cannot generate coverage report.") - return - - if build_path is None: - build_path = os.getcwd() - - if coverage_output_type is None: - logging.warning("No coverage output type give. \ - Cannot generate coverage reports.") - return - - if excludes is None: - excludes = [] - - if coverage_output_type == "html" or coverage_output_type == "both": - # Create build directory if not exist. - coverage_path = os.path.join(build_path, "coverage") - if not os.path.exists(coverage_path): - os.mkdir(coverage_path) - - args = self._get_coverage_script("html", excludes) - - execute_program(args, - "HTML code coverage report generation failed.", - "HTML code coverage report created.") - - if coverage_output_type == "xml" or coverage_output_type == "both": - args = self._get_coverage_script("xml", excludes) - - execute_program(args, - "XML code coverage report generation failed.", - "XML code coverage report created.") - def run_tests(self, filter_regex=None): """ Run unit tests.