From 755ffff5957390d360baf4eb3eac957be1d92582 Mon Sep 17 00:00:00 2001 From: Fred Tingaud Date: Fri, 13 Dec 2024 18:37:29 +0100 Subject: [PATCH] Apply suggestions --- rspec-tools/rspec_tools/cli.py | 3 +- rspec-tools/rspec_tools/coverage.py | 147 ++++++++++++++-------------- rspec-tools/rspec_tools/utils.py | 17 ++-- 3 files changed, 83 insertions(+), 84 deletions(-) diff --git a/rspec-tools/rspec_tools/cli.py b/rspec-tools/rspec_tools/cli.py index eab59584dd6..0fc1a0a1178 100644 --- a/rspec-tools/rspec_tools/cli.py +++ b/rspec-tools/rspec_tools/cli.py @@ -3,7 +3,6 @@ from typing import Optional import click - import rspec_tools.create_rule import rspec_tools.modify_rule from rspec_tools.checklinks import check_html_links @@ -150,7 +149,7 @@ def update_coverage(rulesdir: str, repository: Optional[str], version: Optional[ update_coverage_for_repo(repository, Path(rulesdir)) else: update_coverage_for_repo_version(repository, version, Path(rulesdir)) - collect_coverage_per_product(Path(rulesdir)) + collect_coverage_per_product() @cli.command() @click.option('--message', required=True) diff --git a/rspec-tools/rspec_tools/coverage.py b/rspec-tools/rspec_tools/coverage.py index 1e768e3c310..04e9aced31b 100644 --- a/rspec-tools/rspec_tools/coverage.py +++ b/rspec-tools/rspec_tools/coverage.py @@ -1,12 +1,12 @@ import collections -import json import os import re import sys +from collections import defaultdict from pathlib import Path from git import Git, Repo -from rspec_tools.utils import load_json, pushd +from rspec_tools.utils import load_json, pushd, save_json REPOS = [ 'sonar-abap', @@ -49,11 +49,11 @@ RULES_FILENAME = 'covered_rules.json' -DEPENDENCY_RE = re.compile(r'\s*dependency\s+\'(com|org)\.sonarsource\.([\w-]+):([\w-]+):(\d+(\.\d+)+)\'') +DEPENDENCY_RE = re.compile(r'''\bdependency\s+['"](?:com|org)\.sonarsource\.[\w.-]+:(?P[\w-]+):(?P\d+(\.\d+)+)['"]''') -BUNDLED_SIMPLE = r'[\'"](com|org)\.sonarsource\.([\w.-]+):([\w-]+)[\'"]' -BUNDLED_MULTI = r'\(\s*group:\s*[\'"]([\w.-]+)[\'"],\s*name:\s*[\'"]([\w-]+)[\'"],\s*classifier:\s*[\'"][\w-]+\'\s*\)' -BUNDLED_RE = re.compile(rf'\s*bundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})') +BUNDLED_SIMPLE = r'''['"](?:com|org)\.sonarsource\.(?:[\w.-]+):(?P[\w-]+)['"]''' +BUNDLED_MULTI = r'''\(\s*group:\s*['"]([\w.-]+)['"],\s*name:\s*['"](?P[\w-]+)['"],\s*classifier:\s*['"][\w-]+['"]\s*\)''' +BUNDLED_RE = re.compile(rf'\bbundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})') def get_rule_id(filename): @@ -126,8 +126,7 @@ def __init__(self, filename, rules_dir): self.rules = load_json(filename) def save_to_file(self, filename): - with open(filename, 'w') as outfile: - json.dump(self.rules, outfile, indent=2, sort_keys=True) + save_json(self.rules, filename) def _rule_implemented_for_intermediate_version(self, rule_id, language, repo_and_version): if rule_id not in self.rules[language]: @@ -260,93 +259,89 @@ def update_coverage_for_repo_version(repo, version, rules_dir): def get_plugin_versions(git_repo, version): g = Git(git_repo) repo_dir = git_repo.working_tree_dir - try: - with pushd(repo_dir): - content = g.show(f'{version}:build.gradle') - versions = {} - for m in re.findall(DEPENDENCY_RE, content): - versions[m[2]] = m[3] - return versions - except Exception as e: - print(f"Sonar Enterprise {version} checkout failed: {e}") - raise + with pushd(repo_dir): + content = g.show(f'{version}:build.gradle') + versions = {} + for m in re.finditer(DEPENDENCY_RE, content): + versions[m.groupdict()['plugin_name']] = m.groupdict()['version'] + return versions + + +BUNDLES= {'Community Build': 'sonar-application/bundled_plugins.gradle', + 'Datacenter': 'private/edition-datacenter/bundled_plugins.gradle', + 'Developer': 'private/edition-developer/bundled_plugins.gradle', + 'Enterprise': 'private/edition-enterprise/bundled_plugins.gradle'} + def get_packaged_plugins(git_repo): g = Git(git_repo) repo_dir = git_repo.working_tree_dir with pushd(repo_dir): - BUNDLES= {'Community Build': 'sonar-application/bundled_plugins.gradle', - 'Datacenter': 'private/edition-datacenter/bundled_plugins.gradle', - 'Developer': 'private/edition-developer/bundled_plugins.gradle', - 'Enterprise': 'private/edition-enterprise/bundled_plugins.gradle'} bundle_map = {} for key, bundle in BUNDLES.items(): bundle_map[key] = [] content = g.show(f'master:{bundle}') - for m in re.findall(BUNDLED_RE, content): - if m[3] != '': - bundle_map[key].append(m[3]) + for m in re.finditer(BUNDLED_RE, content): + if m.groupdict()['plugin_name'] != None: + bundle_map[key].append(m.groupdict()['plugin_name']) else: - bundle_map[key].append(m[5]) + bundle_map[key].append(m.groupdict()['plugin_name2']) return bundle_map -def lowest_cb(plugin_versions, plugin, version): - tags = list(filter(lambda k: not k.startswith('sqs-'), plugin_versions.keys())) + +def lowest_version(plugin_versions, plugin, version, skip_suffix): + tags = list(filter(lambda k: not k.startswith(skip_suffix), plugin_versions.keys())) tags.sort(key = comparable_version) for t in tags: if plugin in plugin_versions[t]: pvv = plugin_versions[t][plugin] if comparable_version(pvv) >= comparable_version(version): return t - return None + sys.exit(f'failed finding the oldest version of {plugin} containing {version} from {plugin_versions}') -def lowest_server(plugin_versions, plugin, version): - tags = list(filter(lambda k: not k.startswith('sqcb-'), plugin_versions.keys())) - tags.sort(key = comparable_version) - for t in tags: - if plugin in plugin_versions[t]: - pvv = plugin_versions[t][plugin] - if comparable_version(pvv) >= comparable_version(version): - return t - return None +def lowest_community_build_version(plugin_versions, plugin, version): + return lowest_version(plugin_versions, plugin, version, 'sqs-') -def build_rule_per_product(rules_dir, bundle_map, plugin_versions): - coverage = Coverage(RULES_FILENAME, rules_dir) - rule_per_product = {} - repo_plugin_mapping = load_json(os.path.join(Path(__file__).parent, 'repo_plugin_mapping.json')) - for lang, rules in coverage.rules.items(): - for rule, version in rules.items(): - if isinstance(version, str): - if rule not in rule_per_product: - rule_per_product[rule] = {} - if lang not in rule_per_product[rule]: - rule_per_product[rule][lang] = {} - target_repo, v = version.split(' ') - if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]: - print(f"Couldn't find the corresponding plugin name for {lang} - {target_repo}") - continue - plugin = repo_plugin_mapping[lang][target_repo] - if plugin in bundle_map['Community Build']: - rule_per_product[rule][lang]['SonarQube Community Build'] = lowest_cb(plugin_versions, plugin, v) - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Developer': lowest_server(plugin_versions, plugin, v) - } - elif plugin in bundle_map['Developer']: - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Developer': lowest_server(plugin_versions, plugin, v)} - elif plugin in bundle_map['Enterprise']: - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Enterprise': lowest_server(plugin_versions, plugin, v)} - elif plugin in bundle_map['Datacenter']: - rule_per_product[rule][lang]['SonarQube Server'] = { - 'Datacenter': lowest_server(plugin_versions, plugin, v) - } - else: - print(f'Couldnt find plugin {plugin}') - with open('rule_product_mapping.json', 'w', encoding='utf-8') as outfile: - json.dump(rule_per_product, outfile, indent=2, sort_keys=True) +def lowest_server_version(plugin_versions, plugin, version): + return lowest_version(plugin_versions, plugin, version, 'sqcb-') + + +EDITIONS =['Developer', 'Enterprise', 'Datacenter'] + + +def fill_product_mapping(plugin: str, bundle_map: dict, version: str, plugin_versions: dict, product_per_rule_per_lang: dict): + if plugin in bundle_map['Community Build']: + product_per_rule_per_lang['SonarQube Community Build'] = lowest_community_build_version(plugin_versions, plugin, version) + product_per_rule_per_lang['SonarQube Server'] = { + 'minimal-edition': EDITIONS[0], + 'since-version': lowest_server_version(plugin_versions, plugin, version) + } + return + for edition in EDITIONS: + if plugin in bundle_map[edition]: + product_per_rule_per_lang['SonarQube Server'] = { + 'minimal-edition': edition, + 'since-version': lowest_server_version(plugin_versions, plugin, version) + } + return + sys.exit(f'Couldnt find plugin {plugin}') + + +def build_rule_per_product(bundle_map, plugin_versions): + rules_coverage = load_json(RULES_FILENAME) + rule_per_product = defaultdict(lambda: defaultdict(lambda: {})) + repo_plugin_mapping = load_json(Path(__file__).parent / 'repo_plugin_mapping.json') + for lang, rules in rules_coverage.items(): + for rule, since in rules.items(): + if not isinstance(since, str): + continue + target_repo, version = since.split(' ') + if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]: + sys.exit(f"Couldn't find the corresponding plugin name for {lang} - {target_repo}") + fill_product_mapping(repo_plugin_mapping[lang][target_repo], bundle_map, version, plugin_versions, rule_per_product[rule][lang]) + save_json(rule_per_product, 'rule_product_mapping.json') def is_interesting_version(version): @@ -366,13 +361,13 @@ def is_interesting_version(version): return False return major >= 8 -def collect_coverage_per_product(rules_dir): +def collect_coverage_per_product(): git_repo = checkout_repo('sonar-enterprise') bundle_map = get_packaged_plugins(git_repo) tags = git_repo.tags - tags.sort(key = lambda t: t.commit.committed_date) versions = [tag.name for tag in tags if is_interesting_version(tag.name)] + versions.sort(key = comparable_version) plugin_versions = {} for version in versions: plugin_versions[version] = get_plugin_versions(git_repo, version) - build_rule_per_product(rules_dir, bundle_map, plugin_versions) + build_rule_per_product(bundle_map, plugin_versions) diff --git a/rspec-tools/rspec_tools/utils.py b/rspec-tools/rspec_tools/utils.py index a80caac0db9..f24f037ecf3 100644 --- a/rspec-tools/rspec_tools/utils.py +++ b/rspec-tools/rspec_tools/utils.py @@ -1,11 +1,12 @@ -from pathlib import Path -from typing import List -from contextlib import contextmanager -import shutil -import re -import tempfile import json import os +import re +import shutil +import tempfile +from contextlib import contextmanager +from pathlib import Path +from typing import List + from rspec_tools.errors import InvalidArgumentError SUPPORTED_LANGUAGES_FILENAME = '../supported_languages.adoc' @@ -162,6 +163,10 @@ def load_json(file): with open(file, encoding='utf8') as json_file: return json.load(json_file) +def save_json(content, filename): + with open(filename, 'w', encoding='utf8') as outfile: + json.dump(content, outfile, indent=2, sort_keys=True) + @contextmanager def pushd(new_dir): previous_dir = os.getcwd()