From e311a703e40e9f43630bc2718027e8cb3d0554a6 Mon Sep 17 00:00:00 2001 From: Pieter De Gendt Date: Fri, 25 Oct 2024 14:33:50 +0200 Subject: [PATCH 1/2] scripts: ci: check_compliancy: Add zephyr-keep-sorted regex support To support checking for sorted blocks of multi-line text add an optional regex pattern for the KeepSorted compliance check. Signed-off-by: Pieter De Gendt --- scripts/ci/check_compliance.py | 59 +++++++++++++++++++++++----------- 1 file changed, 40 insertions(+), 19 deletions(-) diff --git a/scripts/ci/check_compliance.py b/scripts/ci/check_compliance.py index f586b53d1471..a605fbb35bc8 100755 --- a/scripts/ci/check_compliance.py +++ b/scripts/ci/check_compliance.py @@ -7,6 +7,7 @@ import argparse import collections from email.utils import parseaddr +from itertools import takewhile import json import logging import os @@ -1541,20 +1542,39 @@ class KeepSorted(ComplianceTest): MARKER = "zephyr-keep-sorted" - def block_is_sorted(self, block_data): - lines = [] + def block_check_sorted(self, block_data, regex): + def _test_indent(txt: str): + return txt.startswith((" ", "\t")) - for line in textwrap.dedent(block_data).splitlines(): - if len(lines) > 0 and line.startswith((" ", "\t")): - # Fold back indented lines - lines[-1] += line.strip() + if regex is None: + block_data = textwrap.dedent(block_data) + + lines = block_data.splitlines() + last = '' + + for idx, line in enumerate(lines): + if not line.strip(): + # Ignore blank lines + continue + + if regex: + # check for regex + if not re.match(regex, line): + continue else: - lines.append(line.strip()) + if _test_indent(line): + continue + + # Fold back indented lines after the current one + for cont in takewhile(_test_indent, lines[idx + 1:]): + line += cont.strip() + + if line < last: + return idx - if lines != sorted(lines): - return False + last = line - return True + return -1 def check_file(self, file, fp): mime_type = magic.from_file(file, mime=True) @@ -1567,8 +1587,9 @@ def check_file(self, file, fp): start_marker = f"{self.MARKER}-start" stop_marker = f"{self.MARKER}-stop" - start_line = None - stop_line = None + regex_marker = r"re\((.+)\)" + start_line = 0 + regex = None for line_num, line in enumerate(fp.readlines(), start=1): if start_marker in line: @@ -1579,22 +1600,22 @@ def check_file(self, file, fp): in_block = True block_data = "" start_line = line_num + 1 + + # Test for a regex block + match = re.search(regex_marker, line) + regex = match.group(1) if match else None elif stop_marker in line: if not in_block: desc = f"{stop_marker} without {start_marker}" self.fmtd_failure("error", "KeepSorted", file, line_num, desc=desc) in_block = False - stop_line = line_num - 1 - if not self.block_is_sorted(block_data): - desc = (f"sorted block is not sorted, sort by running: " + - f"\"ex -s -c '{start_line},{stop_line} sort i|x' {file}\"") + idx = self.block_check_sorted(block_data, regex) + if idx >= 0: + desc = f"sorted block has out-of-order line at {start_line + idx}" self.fmtd_failure("error", "KeepSorted", file, line_num, desc=desc) - elif not line.strip() or line.startswith("#"): - # Ignore comments and blank lines - continue elif in_block: block_data += line From 7143c2b957ba992112cc662bec001b849ebb693a Mon Sep 17 00:00:00 2001 From: Pieter De Gendt Date: Fri, 25 Oct 2024 15:02:54 +0200 Subject: [PATCH 2/2] scripts: ci: check_compliance: Keep UNDEF list sorted Keep the UNDEF_KCONFIG_ALLOWLIST list sorted, as mentioned in the comment. Signed-off-by: Pieter De Gendt --- scripts/ci/check_compliance.py | 52 ++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/scripts/ci/check_compliance.py b/scripts/ci/check_compliance.py index a605fbb35bc8..d3e6dcab2a87 100755 --- a/scripts/ci/check_compliance.py +++ b/scripts/ci/check_compliance.py @@ -910,6 +910,7 @@ def check_no_undef_outside_kconfig(self, kconf): # Many of these are symbols used as examples. Note that the list is sorted # alphabetically, and skips the CONFIG_ prefix. UNDEF_KCONFIG_ALLOWLIST = { + # zephyr-keep-sorted-start re(^\s+") "ALSO_MISSING", "APP_LINK_WITH_", "APP_LOG_LEVEL", # Application log level is not detected correctly as @@ -920,48 +921,49 @@ def check_no_undef_outside_kconfig(self, kconf): # toolchain Kconfig which is sourced based on # Zephyr toolchain variant and therefore not # visible to compliance. + "BINDESC_", # Used in documentation as a prefix "BOARD_", # Used as regex in scripts/utils/board_v1_to_v2.py "BOOT_DIRECT_XIP", # Used in sysbuild for MCUboot configuration "BOOT_DIRECT_XIP_REVERT", # Used in sysbuild for MCUboot configuration - "BOOT_FIRMWARE_LOADER", # Used in sysbuild for MCUboot configuration - "BOOT_RAM_LOAD", # Used in sysbuild for MCUboot configuration - "BOOT_SWAP_USING_MOVE", # Used in sysbuild for MCUboot configuration - "BOOT_SWAP_USING_SCRATCH", # Used in sysbuild for MCUboot configuration "BOOT_ENCRYPTION_KEY_FILE", # Used in sysbuild "BOOT_ENCRYPT_IMAGE", # Used in sysbuild + "BOOT_FIRMWARE_LOADER", # Used in sysbuild for MCUboot configuration "BOOT_MAX_IMG_SECTORS_AUTO", # Used in sysbuild - "BINDESC_", # Used in documentation as a prefix - "BOOT_UPGRADE_ONLY", # Used in example adjusting MCUboot config, but - # symbol is defined in MCUboot itself. + "BOOT_RAM_LOAD", # Used in sysbuild for MCUboot configuration "BOOT_SERIAL_BOOT_MODE", # Used in (sysbuild-based) test/ # documentation "BOOT_SERIAL_CDC_ACM", # Used in (sysbuild-based) test "BOOT_SERIAL_ENTRANCE_GPIO", # Used in (sysbuild-based) test "BOOT_SERIAL_IMG_GRP_HASH", # Used in documentation + "BOOT_SHARE_BACKEND_RETENTION", # Used in Kconfig text "BOOT_SHARE_DATA", # Used in Kconfig text "BOOT_SHARE_DATA_BOOTINFO", # Used in (sysbuild-based) test - "BOOT_SHARE_BACKEND_RETENTION", # Used in Kconfig text "BOOT_SIGNATURE_KEY_FILE", # MCUboot setting used by sysbuild "BOOT_SIGNATURE_TYPE_ECDSA_P256", # MCUboot setting used by sysbuild "BOOT_SIGNATURE_TYPE_ED25519", # MCUboot setting used by sysbuild "BOOT_SIGNATURE_TYPE_NONE", # MCUboot setting used by sysbuild "BOOT_SIGNATURE_TYPE_RSA", # MCUboot setting used by sysbuild + "BOOT_SWAP_USING_MOVE", # Used in sysbuild for MCUboot configuration + "BOOT_SWAP_USING_SCRATCH", # Used in sysbuild for MCUboot configuration + "BOOT_UPGRADE_ONLY", # Used in example adjusting MCUboot config, but + # symbol is defined in MCUboot itself. "BOOT_VALIDATE_SLOT0", # Used in (sysbuild-based) test "BOOT_WATCHDOG_FEED", # Used in (sysbuild-based) test + "BT_6LOWPAN", # Defined in Linux, mentioned in docs "CDC_ACM_PORT_NAME_", "CHRE", # Optional module "CHRE_LOG_LEVEL_DBG", # Optional module "CLOCK_STM32_SYSCLK_SRC_", + "CMD_CACHE", # Defined in U-Boot, mentioned in docs "CMU", "COMPILER_RT_RTLIB", - "BT_6LOWPAN", # Defined in Linux, mentioned in docs - "CMD_CACHE", # Defined in U-Boot, mentioned in docs "CRC", # Used in TI CC13x2 / CC26x2 SDK comment "DEEP_SLEEP", # #defined by RV32M1 in ext/ "DESCRIPTION", "ERR", "ESP_DIF_LIBRARY", # Referenced in CMake comment "EXPERIMENTAL", + "EXTRA_FIRMWARE_DIR", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "FFT", # Used as an example in cmake/extensions.cmake "FLAG", # Used as an example "FOO", @@ -969,24 +971,28 @@ def check_no_undef_outside_kconfig(self, kconf): "FOO_SETTING_1", "FOO_SETTING_2", "HEAP_MEM_POOL_ADD_SIZE_", # Used as an option matching prefix - "LSM6DSO_INT_PIN", + "HUGETLBFS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "LIBGCC_RTLIB", "LLVM_USE_LD", # Both LLVM_USE_* are in cmake/toolchain/llvm/Kconfig "LLVM_USE_LLD", # which are only included if LLVM is selected but # not other toolchains. Compliance check would complain, # for example, if you are using GCC. - "MCUBOOT_LOG_LEVEL_WRN", # Used in example adjusting MCUboot - # config, - "MCUBOOT_LOG_LEVEL_INF", - "MCUBOOT_DOWNGRADE_PREVENTION", # but symbols are defined in MCUboot - # itself. + "LOG_BACKEND_MOCK_OUTPUT_DEFAULT", #Referenced in tests/subsys/logging/log_syst + "LOG_BACKEND_MOCK_OUTPUT_SYST", #Referenced in testcase.yaml of log_syst test + "LSM6DSO_INT_PIN", "MCUBOOT_ACTION_HOOKS", # Used in (sysbuild-based) test "MCUBOOT_CLEANUP_ARM_CORE", # Used in (sysbuild-based) test + "MCUBOOT_DOWNGRADE_PREVENTION", # but symbols are defined in MCUboot + # itself. + "MCUBOOT_LOG_LEVEL_INF", + "MCUBOOT_LOG_LEVEL_WRN", # Used in example adjusting MCUboot + # config, "MCUBOOT_SERIAL", # Used in (sysbuild-based) test/ # documentation "MCUMGR_GRP_EXAMPLE_OTHER_HOOK", # Used in documentation "MISSING", "MODULES", + "MODVERSIONS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "MYFEATURE", "MY_DRIVER_0", "NORMAL_SLEEP", # #defined by RV32M1 in ext/ @@ -998,8 +1004,7 @@ def check_no_undef_outside_kconfig(self, kconf): "REG1", "REG2", "RIMAGE_SIGNING_SCHEMA", # Optional module - "LOG_BACKEND_MOCK_OUTPUT_DEFAULT", #Referenced in tests/subsys/logging/log_syst - "LOG_BACKEND_MOCK_OUTPUT_SYST", #Referenced in testcase.yaml of log_syst test + "SECURITY_LOADPIN", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "SEL", "SHIFT", "SINGLE_APPLICATION_SLOT", # Used in sysbuild for MCUboot configuration @@ -1012,6 +1017,9 @@ def check_no_undef_outside_kconfig(self, kconf): "SRAM2", # Referenced in a comment in samples/application_development "STACK_SIZE", # Used as an example in the Kconfig docs "STD_CPP", # Referenced in CMake comment + "SUIT_MPI_APP_AREA_PATH", # Used by nRF runners to program provisioning data, based on build configuration + "SUIT_MPI_GENERATE", # Used by nRF runners to program provisioning data, based on build configuration + "SUIT_MPI_RAD_AREA_PATH", # Used by nRF runners to program provisioning data, based on build configuration "TEST1", "TOOLCHAIN_ARCMWDT_SUPPORTS_THREAD_LOCAL_STORAGE", # The symbol is defined in the toolchain # Kconfig which is sourced based on Zephyr @@ -1021,16 +1029,10 @@ def check_no_undef_outside_kconfig(self, kconf): "USB_CONSOLE", "USE_STDC_", "WHATEVER", - "EXTRA_FIRMWARE_DIR", # Linux, in boards/xtensa/intel_adsp_cavs25/doc - "HUGETLBFS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc - "MODVERSIONS", # Linux, in boards/xtensa/intel_adsp_cavs25/doc - "SECURITY_LOADPIN", # Linux, in boards/xtensa/intel_adsp_cavs25/doc "ZEPHYR_TRY_MASS_ERASE", # MCUBoot setting described in sysbuild # documentation "ZTEST_FAIL_TEST_", # regex in tests/ztest/fail/CMakeLists.txt - "SUIT_MPI_GENERATE", # Used by nRF runners to program provisioning data, based on build configuration - "SUIT_MPI_APP_AREA_PATH", # Used by nRF runners to program provisioning data, based on build configuration - "SUIT_MPI_RAD_AREA_PATH", # Used by nRF runners to program provisioning data, based on build configuration + # zephyr-keep-sorted-stop }