RULEAPI-816 Create a rule / product mapping file

rspec-tools/rspec_tools/cli.py

@@ -7,9 +7,12 @@
 import rspec_tools.create_rule
 import rspec_tools.modify_rule
 from rspec_tools.checklinks import check_html_links
-from rspec_tools.coverage import (update_coverage_for_all_repos,
-                                  update_coverage_for_repo,
-                                  update_coverage_for_repo_version)
+from rspec_tools.coverage import (
+    collect_coverage_per_product,
+    update_coverage_for_all_repos,
+    update_coverage_for_repo,
+    update_coverage_for_repo_version,
+)
 from rspec_tools.errors import RuleValidationError
 from rspec_tools.notify_failure_on_slack import notify_slack
 from rspec_tools.rules import LanguageSpecificRule, RulesRepository
@@ -147,6 +150,7 @@ def update_coverage(rulesdir: str, repository: Optional[str], version: Optional[
       update_coverage_for_repo(repository, Path(rulesdir))
   else:
       update_coverage_for_repo_version(repository, version, Path(rulesdir))
+  collect_coverage_per_product(Path(rulesdir))
 
 @cli.command()
 @click.option('--message', required=True)

rspec-tools/rspec_tools/coverage.py

@@ -49,6 +49,12 @@
 
 RULES_FILENAME = 'covered_rules.json'
 
+DEPENDENCY_RE = re.compile(r'\s*dependency\s+\'(com|org)\.sonarsource\.([\w-]+):([\w-]+):(\d+(\.\d+)+)\'')
+
+BUNDLED_SIMPLE = r'[\'"](com|org)\.sonarsource\.([\w.-]+):([\w-]+)[\'"]'
+BUNDLED_MULTI = r'\(\s*group:\s*[\'"]([\w.-]+)[\'"],\s*name:\s*[\'"]([\w-]+)[\'"],\s*classifier:\s*[\'"][\w-]+\'\s*\)'
+BUNDLED_RE = re.compile(rf'\s*bundledPlugin\s+({BUNDLED_SIMPLE}|{BUNDLED_MULTI})')
+
 
 def get_rule_id(filename):
   rule_id = filename[:-5]
@@ -197,9 +203,13 @@ def is_version_tag(name):
 
 
 def comparable_version(key):
-  if not is_version_tag(key):
-    return [0]
-  return list(map(int, key.split('.')))
+  v = key.removeprefix('sqcb-').removeprefix('sqs-')
+  if not is_version_tag(v):
+    if v == 'master':
+      return [0]
+    else:
+      sys.exit(f'Unexpected version {key}')
+  return list(map(int, v.split('.')))
 
 
 def collect_coverage_for_all_versions(repo, coverage):
@@ -246,3 +256,123 @@ def update_coverage_for_repo_version(repo, version, rules_dir):
   collect_coverage_for_version(repo, git_repo, version, coverage)
   coverage.save_to_file(RULES_FILENAME)
 
+
+def get_plugin_versions(git_repo, version):
+  g = Git(git_repo)
+  repo_dir = git_repo.working_tree_dir
+  try:
+    with pushd(repo_dir):
+      content = g.show(f'{version}:build.gradle')
+      versions = {}
+      for m in re.findall(DEPENDENCY_RE, content):
+        versions[m[2]] = m[3]
+      return versions
+  except Exception as e:
+    print(f"Sonar Enterprise {version} checkout failed: {e}")
+    raise
+
+def get_packaged_plugins(git_repo):
+  g = Git(git_repo)
+  repo_dir = git_repo.working_tree_dir
+  with pushd(repo_dir):
+    BUNDLES= {'Community Build': 'sonar-application/bundled_plugins.gradle',
+              'Datacenter': 'private/edition-datacenter/bundled_plugins.gradle',
+              'Developer': 'private/edition-developer/bundled_plugins.gradle',
+              'Enterprise': 'private/edition-enterprise/bundled_plugins.gradle'}
+    bundle_map = {}
+    for key, bundle in BUNDLES.items():
+      bundle_map[key] = []
+      content = g.show(f'master:{bundle}')
+      for m in re.findall(BUNDLED_RE, content):
+        if m[3] != '':
+          bundle_map[key].append(m[3])
+        else:
+          bundle_map[key].append(m[5])
+    return bundle_map
+
+def lowest_cb(plugin_versions, plugin, version):
+  tags = list(filter(lambda k: not k.startswith('sqs-'), plugin_versions.keys()))
+  tags.sort(key = comparable_version)
+  for t in tags:
+    if plugin in plugin_versions[t]:
+      pvv = plugin_versions[t][plugin]
+      if comparable_version(pvv) >= comparable_version(version):
+        return t
+  return None
+
+
+def lowest_server(plugin_versions, plugin, version):
+  tags = list(filter(lambda k: not k.startswith('sqcb-'), plugin_versions.keys()))
+  tags.sort(key = comparable_version)
+  for t in tags:
+    if plugin in plugin_versions[t]:
+      pvv = plugin_versions[t][plugin]
+      if comparable_version(pvv) >= comparable_version(version):
+        return t
+  return None
+
+
+def build_rule_per_product(rules_dir, bundle_map, plugin_versions):
+  coverage = Coverage(RULES_FILENAME, rules_dir)
+  rule_per_product = {}
+  repo_plugin_mapping = load_json(os.path.join(Path(__file__).parent, 'repo_plugin_mapping.json'))
+  for lang, rules in coverage.rules.items():
+    for rule, version in rules.items():
+      if isinstance(version, str):
+        if rule not in rule_per_product:
+          rule_per_product[rule] = {}
+        if lang not in rule_per_product[rule]:
+          rule_per_product[rule][lang] = {}
+        target_repo, v = version.split(' ')
+        if lang not in repo_plugin_mapping or target_repo not in repo_plugin_mapping[lang]:
+          print(f"Couldn't find the corresponding plugin name for {lang} - {target_repo}")
+          continue
+        plugin = repo_plugin_mapping[lang][target_repo]
+        if plugin in bundle_map['Community Build']:
+          rule_per_product[rule][lang]['SonarQube Community Build'] = lowest_cb(plugin_versions, plugin, v)
+          rule_per_product[rule][lang]['SonarQube Server'] = {
+            'Developer': lowest_server(plugin_versions, plugin, v)
+            }
+        elif plugin in bundle_map['Developer']:
+          rule_per_product[rule][lang]['SonarQube Server'] = {
+            'Developer': lowest_server(plugin_versions, plugin, v)}
+        elif plugin in bundle_map['Enterprise']:
+          rule_per_product[rule][lang]['SonarQube Server'] = {
+            'Enterprise': lowest_server(plugin_versions, plugin, v)}
+        elif plugin in bundle_map['Datacenter']:
+          rule_per_product[rule][lang]['SonarQube Server'] = {
+            'Datacenter': lowest_server(plugin_versions, plugin, v)
+          }
+        else:
+          print(f'Couldnt find plugin {plugin}')
+  with open('rule_product_mapping.json', 'w', encoding='utf-8') as outfile:
+    json.dump(rule_per_product, outfile, indent=2, sort_keys=True)
+
+
+def is_interesting_version(version):
+  if version.startswith('sqs-'):
+    # Sonarqube Server Release
+    return True
+  if version.startswith('sqcb-'):
+    # Sonarqube Community Build Release
+    return True
+  if not is_version_tag(version):
+    # Non official version
+    return False
+  try:
+    # Official release before Dec 2024
+    major = int(version[:version.find('.')])
+  except ValueError:
+    return False
+  return major >= 8
+
+def collect_coverage_per_product(rules_dir):
+  git_repo = checkout_repo('sonar-enterprise')
+  bundle_map = get_packaged_plugins(git_repo)
+  tags = git_repo.tags
+  tags.sort(key = lambda t: t.commit.committed_date)
+  versions = [tag.name for tag in tags if is_interesting_version(tag.name)]
+  plugin_versions = {}
+  for version in versions:
+    plugin_versions[version] = get_plugin_versions(git_repo, version)
+  build_rule_per_product(rules_dir, bundle_map, plugin_versions)

rspec-tools/rspec_tools/repo_plugin_mapping.json

@@ -0,0 +1,119 @@
+{
+  "ABAP": {
+    "sonar-abap": "sonar-abap-plugin"
+  },
+  "ANSIBLE": {
+    "sonar-iac-enterprise": "sonar-iac-enterprise-plugin"
+  },
+  "APEX": {
+    "sonar-apex": "sonar-apex-plugin"
+  },
+  "AZURE_RESOURCE_MANAGER": {
+    "sonar-iac-enterprise": "sonar-iac-enterprise-plugin"
+  },
+  "C": {
+    "sonar-cpp": "sonar-cfamily-plugin"
+  },
+  "CLOUDFORMATION": {
+    "sonar-iac-enterprise": "sonar-iac-enterprise-plugin"
+  },
+  "COBOL": {
+    "sonar-cobol": "sonar-cobol-plugin"
+  },
+  "CPP": {
+    "sonar-cpp": "sonar-cfamily-plugin"
+  },
+  "CSH": {
+    "sonar-dotnet-enterprise": "sonar-csharp-plugin",
+    "sonar-security": "sonar-security-plugin"
+  },
+  "CSS": {
+    "SonarJS": "sonar-javascript-plugin"
+  },
+  "DART": {
+    "sonar-dart": "sonar-dart-plugin"
+  },
+  "DOCKER": {
+    "sonar-iac-enterprise": "sonar-iac-enterprise-plugin"
+  },
+  "FLEX": {
+    "sonar-flex": "sonar-flex-plugin"
+  },
+  "GO": {
+    "sonar-go": "sonar-go-plugin"
+  },
+  "HTML": {
+    "sonar-html": "sonar-html-plugin"
+  },
+  "JAVA": {
+    "sonar-architecture": "sonar-architecture-plugin",
+    "sonar-dataflow-bug-detection": "sonar-dbd-java-frontend-plugin",
+    "sonar-java": "sonar-java-plugin",
+    "sonar-security": "sonar-security-plugin"
+  },
+  "JAVASCRIPT": {
+    "SonarJS": "sonar-javascript-plugin",
+    "sonar-security": "sonar-security-plugin"
+  },
+  "KOTLIN": {
+    "sonar-kotlin": "sonar-kotlin-plugin"
+  },
+  "KUBERNETES": {
+    "sonar-iac-enterprise": "sonar-iac-enterprise-plugin"
+  },
+  "OBJC": {
+    "sonar-cpp": "sonar-cfamily-plugin"
+  },
+  "PHP": {
+    "sonar-php": "sonar-php-plugin",
+    "sonar-security": "sonar-security-plugin"
+  },
+  "PLI": {
+    "sonar-pli": "sonar-pli-plugin"
+  },
+  "PLSQL": {
+    "sonar-plsql": "sonar-plsql-plugin"
+  },
+  "PY": {
+    "sonar-dataflow-bug-detection": "sonar-dbd-python-frontend-plugin",
+    "sonar-python": "sonar-python-plugin",
+    "sonar-security": "sonar-security-plugin"
+  },
+  "RPG": {
+    "sonar-rpg": "sonar-rpg-plugin"
+  },
+  "RUBY": {
+    "sonar-ruby": "sonar-ruby-plugin"
+  },
+  "SCALA": {
+    "sonar-scala": "sonar-scala-plugin"
+  },
+  "SECRETS": {
+    "sonar-text-enterprise": "sonar-text-enterprise-plugin"
+  },
+  "SWIFT": {
+    "sonar-swift": "sonar-swift-plugin"
+  },
+  "TERRAFORM": {
+    "sonar-iac-enterprise": "sonar-iac-enterprise-plugin"
+  },
+  "TEXT": {
+    "sonar-text-enterprise": "sonar-text-enterprise-plugin"
+  },
+  "TSQL": {
+    "sonar-tsql": "sonar-tsql-plugin"
+  },
+  "TYPESCRIPT": {
+    "SonarJS": "sonar-javascript-plugin",
+    "sonar-security": "sonar-security-plugin"
+  },
+  "VB": {
+    "sonar-vb": "sonar-vb-plugin"
+  },
+  "VBNET": {
+    "sonar-dotnet-enterprise": "sonar-vbnet-enterprise-plugin"
+  },
+  "XML": {
+    "sonar-xml": "sonar-xml-plugin"
+  }
+}