[#16891] ccmd: a command-line tool utilizing the compilation database…

… file

Summary:
`ccmd` (`bin/ccmd`) is a new tool that allows to perform various actions based on the compilation database file (compile_commands.json).

Currently the most useful feature of this new tool is the ability to run clang-tidy in parallel.

Changes to how the compilation database is generated:
- Add two new CMake variables: YB_RESOLVED_C_COMPILER and YB_RESOLVED_CXX_COMPILER, for the actual C and C++ compiler executables, and use them instead of compiler wrapper scripts in the compilation database.

Changes to the build system:
- Remove unused support for a clang executable that lives inside the third-party directory (previously used up to Clang version 7).
- Create the `toolchain` symlink in the build root directory pointing to the LLVM toolchain directory even in case YB_LLVM_TOOLCHAIN_DIR is predefined and there is no need to download and extract the LLVM tarball. Skip it during build and test jobs, because it is needed mostly for local development.

Test Plan: Jenkins

Reviewers: bogdan, steve.varnau

Reviewed By: steve.varnau

Subscribers: anijhawan, ybase

Differential Revision: https://phabricator.dev.yugabyte.com/D24333

.arclint

@@ -20,7 +20,8 @@
      "(^managed/src/main/resources/metric/recording_rules[.]yml$)",
      "(^managed/devops/replicated[.]yml$)",
      "(^python/yb/py[.]typed$)",
-     "(^managed/RUNTIME-FLAGS[.]md$)"
+     "(^managed/RUNTIME-FLAGS[.]md$)",
+     "(^[.]clang-tidy)"
   ],
   "linters": {
     "go-files": {

.clang-tidy

@@ -0,0 +1,29 @@
+---
+# Copyright (c) Yugabyte, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except
+# in compliance with the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software distributed under the License
+# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+# or implied.  See the License for the specific language governing permissions and limitations
+# under the License.
+
+# We don't specify the list of checks here, we just turn off all of them by default.
+# When we invoke clang-tidy, we specify the checks on the command line.
+
+# For the list of clang-tidy checks, see
+# https://clang.llvm.org/extra/clang-tidy/checks/list.html
+
+Checks: '-*'
+CheckOptions:
+  - key: bugprone-argument-comment.StrictMode
+    value: true
+  - key: bugprone-argument-comment.IgnoreSingleArgument
+    value: true
+  - key: bugprone-argument-comment.CommentBoolLiterals
+    value: true
+  - key: bugprone-argument-comment.CommentNullPtrs
+    value: true

.gitignore

@@ -117,3 +117,6 @@ submodules/
 managed/yba-installer/bin/yba-ctl
 managed/yba-installer/yba-ctl
 managed/devops/pex/pexEnv/
+
+*.bad_clang_tidy_output.*
+*.compiler_errors.txt

bin/ccmd

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+. "${BASH_SOURCE%/*}/../build-support/common-build-env.sh"
+
+activate_virtualenv
+set_pythonpath
+
+python3 "${YB_SRC_ROOT}/python/yb/ccmd_tool.py" "$@"

build-support/common-build-env.sh

@@ -947,7 +947,6 @@ find_compiler_by_type() {
   fi
 
   validate_compiler_type "$YB_COMPILER_TYPE"
-
   unset cc_executable
   unset cxx_executable
   case "$YB_COMPILER_TYPE" in
@@ -992,7 +991,8 @@ find_compiler_by_type() {
         cxx_executable=$(which "g++-$gcc_major_version")
       fi
     ;;
-    # This is the old Linuxbrew-based Clang 7 build type.
+    # Default Clang compiler on macOS, or a custom Clang installation with explicitly specified
+    # prefix.
     clang)
       if [[ -n ${YB_CLANG_PREFIX:-} ]]; then
         if [[ ! -d $YB_CLANG_PREFIX/bin ]]; then
@@ -1002,32 +1002,15 @@ find_compiler_by_type() {
       elif [[ $OSTYPE =~ ^darwin ]]; then
         cc_executable=/usr/bin/clang
       else
-        local clang_path
-        local clang_found=false
-        local clang_paths_to_try=(
-          "$YB_THIRDPARTY_DIR/clang-toolchain/bin/clang"
-          # clang is present in this location in pre-built third-party archives built before
-          # the transition to Linuxbrew (https://phabricator.dev.yugabyte.com/D982). This can be
-          # removed when the transition is complete.
-          "$YB_THIRDPARTY_DIR/installed/common/bin/clang"
-        )
-        for clang_path in "${clang_paths_to_try[@]}"; do
-          if [[ -f $clang_path ]]; then
-            cc_executable=$clang_path
-            clang_found=true
-            break
-          fi
-        done
-        if ! "$clang_found"; then
-          fatal "Failed to find clang at the following locations: ${clang_paths_to_try[*]}"
-        fi
+        fatal "Cannot determine Clang executable for YB_COMPILER_TYPE=${YB_COMPILER_TYPE}"
       fi
       if [[ -z ${cxx_executable:-} ]]; then
         cxx_executable=$cc_executable++  # clang -> clang++
       fi
       cc_executable+=${YB_CLANG_SUFFIX:-}
       cxx_executable+=${YB_CLANG_SUFFIX:-}
     ;;
+    # Clang of a specific version. We will download our pre-built LLVM package if necessary.
     clang*)
       if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} ]]; then
         cc_executable=$YB_LLVM_TOOLCHAIN_DIR/bin/clang
@@ -1203,6 +1186,19 @@ download_thirdparty() {
   download_toolchain
 }
 
+create_llvm_toolchain_symlink() {
+  local symlink_path=${BUILD_ROOT}/toolchain
+  if [[ ${YB_SKIP_LLVM_TOOLCHAIN_SYMLINK_CREATION:-0} != "1" &&
+        -n ${YB_LLVM_TOOLCHAIN_DIR:-} &&
+        ! -L ${symlink_path} ]]; then
+    if ! ln -s "${YB_LLVM_TOOLCHAIN_DIR}" "${symlink_path}" &&
+       # If someone else created this symlink in the meantime, that's OK.
+       [[ ! -L ${symlink_path} ]]; then
+      fatal "Could not create symlink from ${symlink_path} to ${YB_LLVM_TOOLCHAIN_DIR}"
+    fi
+  fi
+}
+
 download_toolchain() {
   expect_vars_to_be_set YB_COMPILER_TYPE
   local toolchain_urls=()
@@ -1238,6 +1234,7 @@ download_toolchain() {
   if [[ -n ${YB_LLVM_TOOLCHAIN_URL:-} ]]; then
     toolchain_urls+=( "${YB_LLVM_TOOLCHAIN_URL}" )
   fi
+  create_llvm_toolchain_symlink
 
   if [[ ${#toolchain_urls[@]} -eq 0 ]]; then
     return
@@ -1294,10 +1291,6 @@ download_toolchain() {
       fi
     fi
   done
-
-  if [[ -n ${YB_LLVM_TOOLCHAIN_DIR:-} && ! -e ${BUILD_ROOT}/toolchain ]]; then
-    ln -s "${YB_LLVM_TOOLCHAIN_DIR}" "${BUILD_ROOT}/toolchain"
-  fi
 }
 
 # -------------------------------------------------------------------------------------------------
@@ -2213,6 +2206,7 @@ run_shellcheck() {
 
 activate_virtualenv() {
   detect_architecture
+
   local virtualenv_parent_dir=$YB_BUILD_PARENT_DIR
   local virtualenv_dir=$virtualenv_parent_dir/$YB_VIRTUALENV_BASENAME
 

build-support/jenkins/build.sh

@@ -228,6 +228,8 @@ export BUILD_ROOT
 # End of build root setup and build directory cleanup
 # -------------------------------------------------------------------------------------------------
 
+export YB_SKIP_LLVM_TOOLCHAIN_SYMLINK_CREATION=1
+
 # We need to set this prior to the first invocation of yb_build.sh.
 export YB_SKIP_FINAL_LTO_LINK=1
 

build-support/jenkins/test.sh

@@ -125,6 +125,8 @@ export BUILD_ROOT
 # End of build root setup and build directory cleanup
 # -------------------------------------------------------------------------------------------------
 
+export YB_SKIP_LLVM_TOOLCHAIN_SYMLINK_CREATION=1
+
 # We need to set this prior to the first invocation of yb_build.sh.
 export YB_SKIP_FINAL_LTO_LINK=1
 
@@ -400,4 +402,4 @@ if [[ -n ${FAILURES} ]]; then
   echo >&2 "${FAILURES}"
 fi
 
-exit ${EXIT_STATUS}
+exit ${EXIT_STATUS}

codecheck.ini

@@ -53,18 +53,26 @@ included_regex_list =
     ^build-support/validate_build_root[.]py$
     ^build-support/yb_release[.]py$
     ^python/yb/__init__[.]py$
+    ^python/yb/aggregate_test_reports[.]py$
     ^python/yb/build_paths[.]py$
     ^python/yb/build_postgres[.]py$
+    ^python/yb/ccmd_tool[.]py$
+    ^python/yb/clang_tidy_runner[.]py$
     ^python/yb/cmake_cache[.]py$
     ^python/yb/command_util[.]py$
     ^python/yb/common_util[.]py$
+    ^python/yb/compile_commands_processor[.]py
     ^python/yb/compile_commands[.]py$
+    ^python/yb/compiler_args[.]py
+    ^python/yb/compiler_invocation[.]py$
+    ^python/yb/compiler_parallel_runner[.]py$
     ^python/yb/dep_graph_common[.]py$
     ^python/yb/dependency_graph[.]py$
     ^python/yb/fossa_analysis[.]py$
     ^python/yb/gen_auto_flags_json[.]py$
     ^python/yb/gen_flags_metadata[.]py$
     ^python/yb/gen_initial_sys_catalog_snapshot[.]py$
+    ^python/yb/include_path_rewriter[.]py$
     ^python/yb/install_wrapper[.]py$
     ^python/yb/json_util[.]py$
     ^python/yb/library_packager[.]py$
@@ -76,8 +84,10 @@ included_regex_list =
     ^python/yb/optional_components[.]py$
     ^python/yb/os_detection[.]py$
     ^python/yb/os_versions[.]py$
+    ^python/yb/parallel_task_runner[.]py$
     ^python/yb/postgres_build_util[.]py$
     ^python/yb/postprocess_test_result[.]py$
+    ^python/yb/preprocessor[.]py$
     ^python/yb/release_util[.]py$
     ^python/yb/remote[.]py$
     ^python/yb/rewrite_test_log[.]py$
@@ -89,5 +99,6 @@ included_regex_list =
     ^python/yb/test_rewrite_test_log[.]py$
     ^python/yb/thirdparty_tool[.]py$
     ^python/yb/tool_base[.]py$
+    ^python/yb/type_util[.]py$
     ^python/yb/yb_dist_tests[.]py$
     ^yb_build.sh$

python/yb/aggregate_test_reports.py

@@ -150,7 +150,7 @@ def compute_percentage(n: int, total: int) -> float:
     return n * 100.0 / total
 
 
-def postprocess_stats(stats: Dict[str, Any]):
+def postprocess_stats(stats: Dict[str, Any]) -> None:
     num_errors = stats.get('errors', 0)
     num_failures = stats.get('failures', 0)
     num_run = stats.get('run', 0)

python/yb/build_paths.py

@@ -12,7 +12,7 @@
 
 import os
 
-from yb.common_util import read_file
+from yb.file_util import read_file
 
 from typing import Optional, Set
 

python/yb/build_postgres.py

@@ -35,7 +35,6 @@
 from yugabyte_pycommon import (  # type: ignore
     run_program,
     WorkDirContext,
-    mkdir_p,
     quote_for_bash,
     is_verbose_mode
 )
@@ -54,9 +53,10 @@
 )
 from yb.json_util import write_json_file, read_json_file
 from yb import compile_commands
-from yb.compile_commands import (
-    create_compile_commands_symlink, CompileCommandProcessor, get_compile_commands_file_path)
+from yb.compile_commands import create_compile_commands_symlink, get_compile_commands_file_path
+from yb.compile_commands_processor import CompileCommandProcessor
 from yb.cmake_cache import CMakeCache, load_cmake_cache
+from yb.file_util import mkdir_p
 
 
 ALLOW_REMOTE_COMPILATION = True
@@ -79,6 +79,7 @@
 TRANSIENT_BUILD_RETRIES = 3
 
 COMPILER_AND_LINKER_FLAG_ENV_VAR_NAMES = ['CFLAGS', 'CXXFLAGS', 'LDFLAGS', 'LDFLAGS_EX']
+
 # CPPFLAGS are preprocessor flags.
 ALL_FLAG_ENV_VAR_NAMES = COMPILER_AND_LINKER_FLAG_ENV_VAR_NAMES + ['CPPFLAGS']
 
@@ -871,11 +872,14 @@ def write_compile_commands_files(self, pg_compile_commands_paths: List[str]) ->
 
         # Write similar files with postprocessed compilation commands.
         compile_command_processor = CompileCommandProcessor(
-            self.build_root,
-            extra_args=[
-                f'-DDLSUFFIX="{self.shared_library_suffix}"'
-            ],
-            add_original_dir_to_path_for_files=set(FILES_INCLUDING_GENERATED_FILES_FROM_SAME_DIR))
+            build_root=self.build_root,
+            add_original_dir_to_path_for_files=set(FILES_INCLUDING_GENERATED_FILES_FROM_SAME_DIR),
+            resolved_c_compiler=self.cmake_cache.get('YB_RESOLVED_C_COMPILER'),
+            resolved_cxx_compiler=self.cmake_cache.get('YB_RESOLVED_CXX_COMPILER'))
+
+        # -----------------------------------------------------------------------------------------
+        # Non-Postgres commands
+        # -----------------------------------------------------------------------------------------
 
         yb_postprocessed_compile_commands = [
             compile_command_processor.postprocess_compile_command(item)
@@ -885,6 +889,26 @@ def write_compile_commands_files(self, pg_compile_commands_paths: List[str]) ->
             yb_postprocessed_compile_commands,
             compile_commands.YB_POSTPROCESSED_DIR_NAME)
 
+        # -----------------------------------------------------------------------------------------
+        # Postgres postprocessed commands
+        # -----------------------------------------------------------------------------------------
+
+        # Infer the value of the -DDLSUFFIX compiler flag from the raw Postgres compilation
+        # commands. We will apply the same value of the flag to all Postgres compilation commands,
+        # because some of them depend on it. This comes from the error clangd-indexer reported on
+        # the following files:
+        #   src/postgres/src/backend/jit/jit.c
+        #   src/postgres/src/backend/utils/fmgr/dfmgr.c
+        compile_command_processor.infer_preprocessor_definition('DLSUFFIX', pg_raw_compile_commands)
+
+        # Also make sure YB_SO_MAJOR_VERSION is set consistently for all Postgres compilation
+        # commands. We get multiple compilation commands for files such as
+        # src/postgres/src/timezone/localtime.c and src/postgres/src/timezone/pgtz.c,
+        # where the only difference between command lines is presence of absence of
+        # YB_SO_MAJOR_VERSION.
+        compile_command_processor.infer_preprocessor_definition(
+            'YB_SO_MAJOR_VERSION', pg_raw_compile_commands)
+
         pg_postprocessed_compile_commands = [
             compile_command_processor.postprocess_compile_command(item)
             for item in pg_raw_compile_commands
@@ -893,10 +917,15 @@ def write_compile_commands_files(self, pg_compile_commands_paths: List[str]) ->
             pg_postprocessed_compile_commands,
             compile_commands.PG_POSTPROCESSED_DIR_NAME)
 
+        # -----------------------------------------------------------------------------------------
+        # Combined postprocessed commands
+        # -----------------------------------------------------------------------------------------
+
         combined_postprocessed_compile_commands = (
             yb_postprocessed_compile_commands + pg_postprocessed_compile_commands)
+
         combined_postprocessed_compile_commands_path = self.write_compile_commands_file(
-            combined_postprocessed_compile_commands,
+            compile_command_processor.deduplicate_commands(combined_postprocessed_compile_commands),
             compile_commands.COMBINED_POSTPROCESSED_DIR_NAME)
 
         create_compile_commands_symlink(combined_postprocessed_compile_commands_path)