Only run clang-tidy on files in the current diff (#2245)

Update filter-compile-commands.py script to filter the compile db based
on the current diff. Some quick testing seems very promising cutting a
few file changes from 30 min (i.e. linting the entire codebase) to a few
minutes.

Pointed out by superset dashboard, we currently spend the most time
running this job in terms of total compute cycles.

.github/workflows/build-and-test.yml

@@ -60,8 +60,7 @@ jobs:
       shell: bash
       run: |
         echo "work-dir=$(pwd)" >> "$GITHUB_OUTPUT"
-        echo "build-output-dir=$(pwd)/build" >> "$GITHUB_OUTPUT"
-        echo "install-output-dir=$(pwd)/install" >> "$GITHUB_OUTPUT"
+        echo "build-output-dir=$(pwd)/build_tidy" >> "$GITHUB_OUTPUT"
 
     - name: Git safe dir
       run: git config --global --add safe.directory ${{ steps.strings.outputs.work-dir }}
@@ -75,7 +74,6 @@ jobs:
         -DCMAKE_CXX_COMPILER=clang++-17 \
         -DCMAKE_C_COMPILER=clang-17 \
         -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_INSTALL_PREFIX=${{ steps.strings.outputs.install-output-dir }} \
         -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
         -DTTMLIR_ENABLE_RUNTIME=ON \
         -DTTMLIR_ENABLE_RUNTIME_TESTS=ON \
@@ -88,6 +86,8 @@ jobs:
       shell: bash
       run: |
         source env/activate
+        cmake --build ${{ steps.strings.outputs.build-output-dir }} -- COMMON_FBS TTMETAL_FBS TTNN_FBS mlir-headers mlir-generic-headers
+        python3 tools/scripts/filter-compile-commands.py --diff ${{ steps.strings.outputs.build-output-dir }}/compile_commands.json
         cmake --build ${{ steps.strings.outputs.build-output-dir }} -- clang-tidy
 
     - name: Unique-ify clang-tidy fixes

cmake/modules/LintTools.cmake

@@ -1,8 +1,6 @@
 # clang-tidy setup
-add_custom_target(clang-tidy-filter-out-external-srcs COMMAND python3 ${TTMLIR_SOURCE_DIR}/tools/scripts/filter-compile-commands.py ${TTMLIR_BINARY_DIR}/compile_commands.json "${TTMLIR_SOURCE_DIR}")
 add_custom_target(clang-tidy COMMAND run-clang-tidy.py -p ${PROJECT_BINARY_DIR} -export-fixes clang-tidy-fixes.yaml -warnings-as-errors '*' -extra-arg-before=-DDISABLE_STATIC_ASSERT_TESTS -extra-arg-before=-D__cpp_structured_bindings=202400
   DEPENDS
-    clang-tidy-filter-out-external-srcs
     mlir-headers
     mlir-generic-headers
     tt-metal-download

tools/scripts/filter-compile-commands.py

@@ -2,22 +2,103 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-import sys
+import argparse
 import json
+import os
 import re
+import subprocess
+import sys
+
+
+def get_diff_files(compile_commands):
+    fork_point = (
+        subprocess.check_output(["git", "merge-base", "--fork-point", "origin/main"])
+        .decode("utf-8")
+        .strip()
+    )
+    diff = subprocess.check_output(["git", "diff", "--name-only", fork_point]).decode(
+        "utf-8"
+    )
+    cwd = os.getcwd()
+    processed = map(lambda x: os.path.join(cwd, x.strip()), diff.split("\n"))
+    return set(
+        filter(
+            lambda x: x.endswith(".c")
+            or x.endswith(".cc")
+            or x.endswith(".cpp")
+            or x.endswith(".hpp")
+            or x.endswith(".h"),
+            processed,
+        )
+    )
+
 
-assert len(sys.argv) == 3, "Usage: {} <compile_commands.json> <prefix>".format(
-    sys.argv[0]
-)
+def get_deps(command):
+    try:
+        cmd = command["command"].split(" ")
+        try:
+            idx = cmd.index("-o")
+            del cmd[idx : idx + 2]
+        except:
+            pass
+        cmd.insert(1, "-MM")
+        deps = subprocess.check_output(cmd, cwd=command["directory"]).decode("utf-8")
+        return set(map(lambda x: x.strip(" \\"), deps.split("\n")[1:]))
+    except:
+        print("Failed to get deps for", command["file"])
+        print("Command:", command["command"])
+        return set()
 
-with open(sys.argv[1], "r") as f:
-    compile_commands = json.load(f)
 
-filtered_commands = []
-m = re.compile(r"^{}/((?!third_party).)*$".format(sys.argv[2]))
-for command in compile_commands:
-    if m.match(command["file"]):
+def main(args):
+    with open(args.compile_commands, "r") as f:
+        compile_commands = json.load(f)
+
+    filtered_commands = []
+    m = re.compile(r"^{}/((?!third_party).)*$".format(args.prefix))
+    diff_files = get_diff_files(compile_commands) if args.diff else set()
+    compile_commands = sorted(
+        compile_commands, key=lambda x: x["file"] not in diff_files
+    )
+    for command in compile_commands:
+        if args.diff and not diff_files:
+            break
+        if not m.match(command["file"]):
+            continue
+        if args.diff and command["file"] not in diff_files:
+            deps = get_deps(command) & diff_files
+            if not deps:
+                continue
+            diff_files = diff_files - deps
         filtered_commands.append(command)
+        diff_files = diff_files - {command["file"]}
+
+    if args.dry:
+        json.dump(filtered_commands, sys.stdout, indent=2)
+        sys.stdout.write("\n")
+    else:
+        with open(args.compile_commands, "w") as f:
+            json.dump(filtered_commands, f, indent=2)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Filter compile commands to exclude unnecessary files for linting."
+    )
+    parser.add_argument("compile_commands", help="Path to compile_commands.json")
+    parser.add_argument(
+        "--prefix",
+        default=os.getcwd(),
+        help="Prefix to filter out from the compile commands.",
+    )
+    parser.add_argument(
+        "--diff", action="store_true", help="Filter out files that are not in the diff."
+    )
+    parser.add_argument(
+        "--dry",
+        action="store_true",
+        help="Dry run, do not write to compile_commands.json.",
+    )
 
-with open(sys.argv[1], "w") as f:
-    json.dump(filtered_commands, f, indent=2)
+    args = parser.parse_args()
+    main(args)