diff --git a/scripts/ci/test_plan.py b/scripts/ci/test_plan.py index 7004ccd8625d..d03211909662 100755 --- a/scripts/ci/test_plan.py +++ b/scripts/ci/test_plan.py @@ -102,7 +102,7 @@ def __repr__(self): class Filters: def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root, - pull_request=False, platforms=[], detailed_test_id=True, quarantine_list=None, tc_roots_th=20): + pull_request=False, platforms=[], detailed_test_id=True, quarantine_list=None, tc_roots_th=20, testsuite_excludes_file=None): self.modified_files = modified_files self.testsuite_root = testsuite_root self.resolved_files = [] @@ -110,6 +110,7 @@ def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root, self.full_twister = False self.all_tests = [] self.tag_options = [] + self.testsuite_excludes_options = [] self.pull_request = pull_request self.platforms = platforms self.detailed_test_id = detailed_test_id @@ -117,10 +118,12 @@ def __init__(self, modified_files, ignore_path, alt_tags, testsuite_root, self.tag_cfg_file = alt_tags self.quarantine_list = quarantine_list self.tc_roots_th = tc_roots_th + self.testsuite_excludes_file = testsuite_excludes_file def process(self): self.find_modules() self.find_tags() + self.find_testsuite_excludes() self.find_tests() if not self.platforms: # disable for now, this is generating lots of churn when changing @@ -326,9 +329,8 @@ def find_tests(self): _options.extend(["-p", platform]) self.get_plan(_options, use_testsuite_root=False) - def find_tags(self): - - with open(self.tag_cfg_file, 'r') as ymlfile: + def _get_tags(self, yml_path): + with open(yml_path, 'r') as ymlfile: tags_config = yaml.safe_load(ymlfile) tags = {} @@ -358,12 +360,29 @@ def find_tags(self): if t.exclude: exclude_tags.add(t.name) + return exclude_tags + + def find_tags(self): + exclude_tags = self._get_tags(self.tag_cfg_file) + for tag in exclude_tags: self.tag_options.extend(["-e", tag ]) if exclude_tags: logging.info(f'Potential tag based filters: {exclude_tags}') + def find_testsuite_excludes(self): + if self.testsuite_excludes_file is None: + return + + exclude_testsuites = self._get_tags(self.testsuite_excludes_file) + + for tag in exclude_testsuites: + self.testsuite_excludes_options.extend(["--testsuite-exclude-path", tag ]) + + if exclude_testsuites: + logging.info(f'Testsuite exclude filters: {exclude_testsuites}') + def find_excludes(self, skip=[]): with open(self.ignore_path, "r") as twister_ignore: ignores = twister_ignore.read().splitlines() @@ -391,9 +410,11 @@ def find_excludes(self, skip=[]): _options.extend(["-p", platform]) _options.extend(self.tag_options) + _options.extend(self.testsuite_excludes_options) self.get_plan(_options) else: _options.extend(self.tag_options) + _options.extend(self.testsuite_excludes_options) self.get_plan(_options, True) else: logging.info(f'No twister needed or partial twister run only...') @@ -443,6 +464,9 @@ def parse_args(): "the file need to correspond to the test scenarios names as in " "corresponding tests .yaml files. These scenarios " "will be skipped with quarantine as the reason.") + parser.add_argument('--testsuite-excludes-file', + default=None, + help="Path to a file describing relations between directories/paths (modified files) and testsuites filters.") # Include paths in names by default. parser.set_defaults(detailed_test_id=True) @@ -472,7 +496,7 @@ def parse_args(): f = Filters(files, args.ignore_path, args.alt_tags, args.testsuite_root, args.pull_request, args.platform, args.detailed_test_id, args.quarantine_list, - args.testcase_roots_threshold) + args.testcase_roots_threshold, args.testsuite_excludes_file) f.process() # remove dupes and filtered cases diff --git a/scripts/ci/testsuite_excludes.yaml b/scripts/ci/testsuite_excludes.yaml new file mode 100644 index 000000000000..56da5d694fed --- /dev/null +++ b/scripts/ci/testsuite_excludes.yaml @@ -0,0 +1,17 @@ +# This file contains information on what files are associated with which +# twister testsuite excludes patterns. +# +# File format is the same as for tags.yaml - please refer to its description. + +# zephyr-keep-sorted-start +"*tests/kernel*": + files: + - kernel/ + - arch/ + - tests/kernel/ + +"*tests/posix*": + files: + - lib/posix/ + - tests/posix/ +# zephyr-keep-sorted-stop diff --git a/scripts/pylib/twister/twisterlib/environment.py b/scripts/pylib/twister/twisterlib/environment.py index e64fb6934460..a2c35dd96755 100644 --- a/scripts/pylib/twister/twisterlib/environment.py +++ b/scripts/pylib/twister/twisterlib/environment.py @@ -139,6 +139,15 @@ def add_parse_arguments(parser = None) -> argparse.ArgumentParser: "called multiple times. Defaults to the 'samples/' and " "'tests/' directories at the base of the Zephyr tree.") + case_select.add_argument( + "--testsuite-exclude-path", action="append", default=[], type = norm_path, + help="Directory to exclude from searching for test cases. " + "This is a filter pattern for paths provided with the testsuite-root option. " + "Supports Unix shell-style wildcards, e.g. *samples/sub* (fnmatch). " + "The exclude pattern is matched against an absolute path for the test suite. " + "This option can be used multiple times, multiple invocations " + "are treated as a logical 'or' relationship.") + case_select.add_argument( "-f", "--only-failed", @@ -1024,6 +1033,7 @@ def __init__(self, options : argparse.Namespace, default_options=None) -> None: logger.info(f"Using {self.generator}..") self.test_roots = options.testsuite_root + self.test_exclude_paths = options.testsuite_exclude_path if not isinstance(options.board_root, list): self.board_roots = [options.board_root] diff --git a/scripts/pylib/twister/twisterlib/testplan.py b/scripts/pylib/twister/twisterlib/testplan.py index 5d17f5cc5466..83784e4fc82d 100755 --- a/scripts/pylib/twister/twisterlib/testplan.py +++ b/scripts/pylib/twister/twisterlib/testplan.py @@ -7,6 +7,7 @@ # SPDX-License-Identifier: Apache-2.0 import collections import copy +import fnmatch import glob import itertools import json @@ -588,6 +589,15 @@ def add_testsuites(self, testsuite_filter=None): logger.debug("Found possible testsuite in " + dirpath) + skip_path = False + for test_exclude_path in self.env.test_exclude_paths: + if fnmatch.fnmatch(dirpath, test_exclude_path): + skip_path = True + break + if skip_path: + logger.debug(f"Skipping {dirpath} due to excludes") + continue + suite_yaml_path = os.path.join(dirpath, filename) suite_path = os.path.dirname(suite_yaml_path) diff --git a/scripts/tests/twister/test_testplan.py b/scripts/tests/twister/test_testplan.py index 2a006043870e..49514f44512c 100644 --- a/scripts/tests/twister/test_testplan.py +++ b/scripts/tests/twister/test_testplan.py @@ -1451,7 +1451,8 @@ def test_testplan_add_testsuites(tmp_path, testsuite_filter, use_alt_root, detai env = mock.Mock( test_roots=[tmp_test_root_dir], options=mock.Mock(detailed_test_id=detailed_id), - alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else [] + alt_config_root=[tmp_alt_test_root_dir] if use_alt_root else [], + test_exclude_paths=[] ) testplan = TestPlan(env=env)