diff --git a/.github/workflows/generate-query-help-docs.yml b/.github/workflows/generate-query-help-docs.yml deleted file mode 100644 index e3af19c6dea7..000000000000 --- a/.github/workflows/generate-query-help-docs.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: Generate CodeQL query help documentation using Sphinx - -on: - workflow_dispatch: - inputs: - description: - description: A description of the purpose of this job. For human consumption. - required: false - push: - branches: - - 'lgtm.com' - pull_request: - paths: - - '.github/workflows/generate-query-help-docs.yml' - - 'docs/codeql/query-help/**' - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Clone github/codeql - uses: actions/checkout@v2 - with: - path: codeql - - name: Clone github/codeql-go - uses: actions/checkout@v2 - with: - repository: 'github/codeql-go' - path: codeql-go - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Download CodeQL CLI - uses: dsaltares/fetch-gh-release-asset@aa37ae5c44d3c9820bc12fe675e8670ecd93bd1c - with: - repo: "github/codeql-cli-binaries" - version: "latest" - file: "codeql-linux64.zip" - token: ${{ secrets.GITHUB_TOKEN }} - - name: Unzip CodeQL CLI - run: unzip -d codeql-cli codeql-linux64.zip - - name: Set up query help docs folder - run: | - cp -r codeql/docs/codeql/** . - - name: Query help to markdown - run: | - PATH="$PATH:codeql-cli/codeql" python codeql/docs/codeql/query-help-markdown.py - - name: Run Sphinx for query help - uses: ammaraskar/sphinx-action@8b4f60114d7fd1faeba1a712269168508d4750d2 # v0.4 - with: - docs-folder: "query-help/" - pre-build-command: "python -m pip install --upgrade recommonmark && python -m pip install --upgrade sphinx-markdown-tables" - build-command: "sphinx-build -b dirhtml . _build" - - name: Upload HTML artifacts - uses: actions/upload-artifact@v2 - with: - name: query-help-html - path: query-help/_build - diff --git a/docs/codeql/query-help-markdown.py b/docs/codeql/query-help-markdown.py deleted file mode 100644 index 350c9aefce2e..000000000000 --- a/docs/codeql/query-help-markdown.py +++ /dev/null @@ -1,256 +0,0 @@ -import re -import subprocess -import json -import csv -import sys -import os - -""" -This script collects CodeQL queries that are part of code scanning query packs, -renders the accompanying query help as markdown, inserts some useful metadata -into the help, and adds a link to the query in the CodeQL repo. - -This script requires that 'git' and 'codeql' commands -are on the PATH. It'll try to automatically set the CodeQL search path correctly, -as long as you run the script from one of the following locations: - - anywhere from within a clone of the CodeQL Git repo - - from the parent directory of a clone of the CodeQL Git repo (assuming 'codeql' - and 'codeql-go' directories both exist) -""" - -# Define which languages and query packs to consider -languages = [ "cpp", "csharp", "go", "java", "javascript", "python"] - -# Query suites to generate help for - lgtm suite should cover the queries that users are interested in -packs = ["lgtm"] - -def prefix_repo_nwo(filename): - """ - Replaces an absolute path prefix with a GitHub repository name with owner (NWO). - This function relies on `git` being available. - For example: - /home/alice/git/ql/java/ql/src/MyQuery.ql - becomes: - github/codeql/java/ql/src/MyQuery.ql - - If we can't detect a known NWO (e.g. github/codeql, github/codeql-go), the - path will be truncated to the root of the git repo: - ql/java/ql/src/MyQuery.ql - - If the filename is not part of a Git repo, the return value is the - same as the input value: the whole path. - """ - dirname = os.path.dirname(filename) - - try: - git_toplevel_dir_subp = subprocess_run( - ["git", "-C", dirname, "rev-parse", "--show-toplevel"]) - except: - # Not a Git repo - return filename - - git_toplevel_dir = git_toplevel_dir_subp.stdout.strip() - - # Detect 'github/codeql' and 'github/codeql-go' repositories by checking the remote (it's a bit - # of a hack but will work in most cases, as long as the remotes have 'codeql' and 'codeql-go' - # in the URL - git_remotes = subprocess_run( - ["git", "-C", dirname, "remote", "-v"]).stdout.strip() - - if "codeql-go" in git_remotes: - prefix = "github/codeql-go" - elif "codeql" in git_remotes: - prefix = "github/codeql" - else: - prefix = os.path.basename(git_toplevel_dir) - - return os.path.join(prefix, filename[len(git_toplevel_dir)+1:]) - - -def single_spaces(input): - """ - Workaround for https://github.com/github/codeql-coreql-team/issues/470 which causes - some metadata strings to contain newlines and spaces without a good reason. - """ - return " ".join(input.split()) - - -def get_query_metadata(key, metadata, queryfile): - """Returns query metadata or prints a warning to stderr if a particular piece of metadata is not available.""" - if key in metadata: - return single_spaces(metadata[key]) - query_id = metadata['id'] if 'id' in metadata else 'unknown' - print("Warning: no '%s' metadata for query with ID '%s' (%s)" % - (key, query_id, queryfile), file=sys.stderr) - return "" - - -def subprocess_run(cmd): - """Runs a command through subprocess.run, with a few tweaks. Raises an Exception if exit code != 0.""" - return subprocess.run(cmd, capture_output=True, text=True, env=os.environ.copy(), check=True) - - -try: # Check for `git` on path - subprocess_run(["git", "--version"]) -except Exception as e: - print("Error: couldn't invoke 'git'. Is it on the path? Aborting.", file=sys.stderr) - raise e - -try: # Check for `codeql` on path - subprocess_run(["codeql", "--version"]) -except Exception as e: - print("Error: couldn't invoke CodeQL CLI 'codeql'. Is it on the path? Aborting.", file=sys.stderr) - raise e - -# Define CodeQL search path so it'll find the CodeQL repositories: -# - anywhere in the current Git clone (including current working directory) -# - the 'codeql' subdirectory of the cwd -# -# (and assumes the codeql-go repo is in a similar location) - -codeql_search_path = "./codeql:./codeql-go" # will be extended further down -# Extend CodeQL search path by detecting root of the current Git repo (if any). This means that you -# can run this script from any location within the CodeQL git repository. -try: - git_toplevel_dir = subprocess_run(["git", "rev-parse", "--show-toplevel"]) - - # Current working directory is in a Git repo. Add it to the search path, just in case it's the CodeQL repo - #git_toplevel_dir = git_toplevel_dir.stdout.strip() - codeql_search_path += ":" + git_toplevel_dir + ":" + git_toplevel_dir + "/../codeql-go" - codeql_search_path = git_toplevel_dir = git_toplevel_dir.stdout.strip() -except: - # git rev-parse --show-toplevel exited with non-zero exit code. We're not in a Git repo - pass - -# Iterate over all languages and packs, and resolve which queries are part of those packs -for lang in languages: - - code_scanning_queries = subprocess_run( - ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-code-scanning.qls" % (lang)]).stdout.strip() - security_extended_queries = subprocess_run( - ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-security-extended.qls" % (lang)]).stdout.strip() - security_and_quality_queries = subprocess_run( - ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-security-and-quality.qls" % (lang)]).stdout.strip() - # Define empty dictionary to store @name:filename pairs to generate alphabetically sorted Sphinx toctree - index_file_dictionary = {} - for pack in packs: - # Get absolute paths to queries in this pack by using 'codeql resolve queries' - try: - - queries_subp = subprocess_run( - ["codeql", "resolve", "queries", "--search-path", codeql_search_path, "%s-%s.qls" % (lang, pack)]) - except Exception as e: - # Resolving queries might go wrong if the github/codeql and github/codeql-go repositories are not - # on the search path. - print( - "Warning: couldn't find query pack '%s' for language '%s'. Do you have the right repositories in the right places (search path: '%s')?" % ( - pack, lang, codeql_search_path), - file=sys.stderr - ) - continue - - # Define empty dictionary to store @name:filename pairs to generate alphabetically sorted Sphinx toctree later - index_file_dictionary = {} - - # Investigate metadata for every query by using 'codeql resolve metadata' - for queryfile in queries_subp.stdout.strip().split("\n"): - query_metadata_json = subprocess_run( - ["codeql", "resolve", "metadata", queryfile]).stdout.strip() - meta = json.loads(query_metadata_json) - - # Turn an absolute path to a query file into an nwo-prefixed path (e.g. github/codeql/java/ql/src/....) - queryfile_nwo = prefix_repo_nwo(queryfile) - - # Generate the query help for each query - try: - query_help = subprocess_run( - ["codeql", "generate", "query-help", "--format=markdown", "--warnings=error", queryfile]).stdout.strip() - except: - # Print a message if generate query help fails - print("Failed to generate query help for '%s'" % (queryfile_nwo)) - continue - - # Pull out relevant query metadata properties that we want to display in the query help - query_name_meta = get_query_metadata('name', meta, queryfile) - query_description = get_query_metadata( - 'description', meta, queryfile) - query_id = "ID: " + \ - get_query_metadata('id', meta, queryfile) + "\n" - query_kind = "Kind: " + \ - get_query_metadata('kind', meta, queryfile) + "\n" - query_severity = "Severity: " + \ - get_query_metadata('problem.severity', meta, queryfile) + "\n" - query_precision = "Precision: " + \ - get_query_metadata('precision', meta, queryfile) + "\n" - query_tags = "Tags:\n - " + \ - get_query_metadata('tags', meta, queryfile).replace(" ", "\n - ") + "\n" - - # Build a link to the query source file for display in the query help - if "go" in prefix_repo_nwo(queryfile): - transform_link = prefix_repo_nwo(queryfile).replace( - "codeql-go", "codeql-go/blob/main").replace(" ", "%20").replace("\\", "/") - else: - transform_link = prefix_repo_nwo(queryfile).replace( - "codeql", "codeql/blob/main").replace(" ", "%20").replace("\\", "/") - query_link = "[Click to see the query in the CodeQL repository](https://github.com/" + \ - transform_link + ")\n" - - if queryfile in code_scanning_queries: - cs_suites = ' - ' + lang +'-code-scanning.qls\n' - else: - cs_suites = "" - if queryfile in security_extended_queries: - se_suites = ' - ' + lang + '-security-extended.qls\n' - else: - se_suites = "" - if queryfile in security_and_quality_queries: - sq_suites = ' - ' +lang + '-security-and-quality.qls\n' - else: - sq_Suites = "" - - if queryfile in code_scanning_queries or queryfile in security_extended_queries or queryfile in security_and_quality_queries: - suites_list = "Query suites:\n" + cs_suites + se_suites + sq_suites - else: - suites_list = "" - - # Join metadata into a literal block and add query link below - meta_string = "\n"*2 + "```\n" + query_id + query_kind + query_severity + \ - query_precision + query_tags + suites_list + "```\n\n" + query_link + "\n" - - # Insert metadata block into query help directly under title - full_help = query_help.replace("\n", meta_string, 1) - - # Use id property to make name for markdown file, replacing any "/" characters with "-" - query_name = query_id[4:-1].replace("/", "-") - - # Populate index_file_dictionary with @name extracted from metadata and corresponding query filename - index_file_dictionary[query_name_meta] = lang + "/" + query_name - - # Make paths for output of the form: query-help-markdown//.md - docs_dir = 'query-help' - md_dir_path = os.path.join(docs_dir, lang) - md_file_path = os.path.join(md_dir_path, query_name + ".md") - - # Make directories for output paths they don't already exist - if not os.path.isdir(md_dir_path): - os.makedirs(md_dir_path) - - # Generate query help at chosen path if output file doesn't already exist - if not os.path.exists(md_file_path): - file = open(md_file_path, "x") - file.write(full_help) - file.close() - - # Sort index_file_dictionary alphabetically by @name key, and create column of filename values - sorted_index = dict(sorted(index_file_dictionary.items())) - sorted_index = ("\n" + " ").join(sorted_index.values()) - - # Add directives to make sorted_index a valid toctree for sphinx source files - toc_directive = ".. toctree::\n :titlesonly:\n\n " - toc_include = toc_directive + sorted_index - - # Write toctree to rst - toc_file = os.path.join(docs_dir, "toc-" + lang + ".rst") - file = open(toc_file, "x") - file.write(toc_include) - file.close()