From 327c4ce04ffe12a6345f6eac06c5c8f94b564526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Mon, 9 May 2022 16:30:00 +0200 Subject: [PATCH 1/2] fixed bug with rubocop linter installation in older systems + pylint fixes --- .../analysis/linter/apt-pkgs-runtime.txt | 1 - .../analysis/linter/dnf-pkgs-runtime.txt | 1 - src/plugins/analysis/linter/install.py | 6 +- .../analysis/linter/internal/linters.py | 90 +++++++++++-------- 4 files changed, 55 insertions(+), 43 deletions(-) diff --git a/src/plugins/analysis/linter/apt-pkgs-runtime.txt b/src/plugins/analysis/linter/apt-pkgs-runtime.txt index 08bfa6c9b..1777f60a5 100644 --- a/src/plugins/analysis/linter/apt-pkgs-runtime.txt +++ b/src/plugins/analysis/linter/apt-pkgs-runtime.txt @@ -2,4 +2,3 @@ liblua5.3-dev lua5.3 luarocks shellcheck -ruby diff --git a/src/plugins/analysis/linter/dnf-pkgs-runtime.txt b/src/plugins/analysis/linter/dnf-pkgs-runtime.txt index ccfa7b8e2..88046d660 100644 --- a/src/plugins/analysis/linter/dnf-pkgs-runtime.txt +++ b/src/plugins/analysis/linter/dnf-pkgs-runtime.txt @@ -3,4 +3,3 @@ lua-devel luarocks nodejs ShellCheck -ruby diff --git a/src/plugins/analysis/linter/install.py b/src/plugins/analysis/linter/install.py index fbf4a338f..9a345919c 100755 --- a/src/plugins/analysis/linter/install.py +++ b/src/plugins/analysis/linter/install.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +# pylint: disable=ungrouped-imports import logging from pathlib import Path @@ -22,12 +23,12 @@ def install_other_packages(self): run_cmd_with_logging('sudo luarocks install argparse') run_cmd_with_logging('sudo luarocks install luacheck') run_cmd_with_logging('sudo luarocks install luafilesystem') - run_cmd_with_logging('sudo gem install rubocop') def install_docker_images(self): run_cmd_with_logging('docker pull crazymax/linguist') run_cmd_with_logging('docker pull cytopia/eslint') run_cmd_with_logging('docker pull ghcr.io/phpstan/phpstan') + run_cmd_with_logging('docker pull pipelinecomponents/rubocop') # Alias for generic use @@ -35,6 +36,5 @@ def install_docker_images(self): if __name__ == '__main__': logging.basicConfig(level=logging.INFO) - distribution = check_distribution() - installer = Installer(distribution) + installer = Installer(distribution=check_distribution()) installer.install() diff --git a/src/plugins/analysis/linter/internal/linters.py b/src/plugins/analysis/linter/internal/linters.py index fb79dd182..dc6394fc3 100644 --- a/src/plugins/analysis/linter/internal/linters.py +++ b/src/plugins/analysis/linter/internal/linters.py @@ -1,9 +1,9 @@ import json import logging -import shlex import subprocess from pathlib import Path -from subprocess import DEVNULL, PIPE, STDOUT +from subprocess import PIPE, STDOUT +from typing import List, Tuple from docker.types import Mount @@ -35,27 +35,28 @@ def run_eslint(file_path): def run_shellcheck(file_path): shellcheck_process = subprocess.run( - 'shellcheck --format=json {}'.format(file_path), + f'shellcheck --format=json {file_path}', shell=True, stdout=PIPE, stderr=STDOUT, + check=False, universal_newlines=True, ) if shellcheck_process.returncode == 2: - logging.debug('Failed to execute shellcheck:\n{}'.format(shellcheck_process.stdout)) - return list() + logging.debug(f'Failed to execute shellcheck:\n{shellcheck_process.stdout}') + return [] try: shellcheck_json = json.loads(shellcheck_process.stdout) except json.JSONDecodeError: - return list() + return [] - return _shellcheck_extract_relevant_warnings(shellcheck_json) + return _extract_shellcheck_warnings(shellcheck_json) -def _shellcheck_extract_relevant_warnings(shellcheck_json): - issues = list() +def _extract_shellcheck_warnings(shellcheck_json): + issues = [] for issue in shellcheck_json: if issue['level'] in ['warning', 'error']: issues.append({ @@ -72,10 +73,11 @@ def run_luacheck(file_path): luacheckrc_path = Path(Path(__file__).parent, 'config', 'luacheckrc') luacheck_process = subprocess.run( - 'luacheck -q --ranges --config {} {}'.format(luacheckrc_path, file_path), + f'luacheck -q --ranges --config {luacheckrc_path} {file_path}', shell=True, stdout=PIPE, stderr=STDOUT, + check=False, universal_newlines=True, ) return _luacheck_parse_linter_output(luacheck_process.stdout) @@ -86,11 +88,11 @@ def _luacheck_parse_linter_output(output): https://luacheck.readthedocs.io/en/stable/warnings.html ignore_cases = ['(W611)', '(W612)', '(W613)', '(W614)', '(W621)', '(W631)'] ''' - issues = list() + issues = [] for line in output.splitlines(): try: line_number, columns, code_and_message = _luacheck_split_issue_line(line) - code, message = _luacheck_separate_message_and_code(code_and_message) + code, message = _separate_message_and_code(code_and_message) if not code.startswith('(W6'): issues.append({ 'line': int(line_number), @@ -100,8 +102,8 @@ def _luacheck_parse_linter_output(output): }) else: pass - except (IndexError, ValueError) as e: - logging.warning('Lualinter failed to parse line: {}\n{}'.format(line, e)) + except (IndexError, ValueError) as error: + logging.warning(f'Lualinter failed to parse line: {line}\n{error}') return issues @@ -111,7 +113,7 @@ def _luacheck_split_issue_line(line): return split_by_colon[1], split_by_colon[2], ':'.join(split_by_colon[3:]).strip() -def _luacheck_separate_message_and_code(message_string): +def _separate_message_and_code(message_string: str) -> Tuple[str, str]: return message_string[1:5], message_string[6:].strip() @@ -120,19 +122,26 @@ def _luacheck_get_first_column(columns): def run_pylint(file_path): - pylint_process = subprocess.run('pylint --output-format=json {}'.format(file_path), shell=True, stdout=PIPE, stderr=STDOUT, universal_newlines=True) + pylint_process = subprocess.run( + f'pylint --output-format=json {file_path}', + shell=True, + stdout=PIPE, + stderr=STDOUT, + check=False, + universal_newlines=True + ) try: pylint_json = json.loads(pylint_process.stdout) except json.JSONDecodeError: - logging.warning('Failed to execute pylint:\n{}'.format(pylint_process.stdout)) - return list() + logging.warning(f'Failed to execute pylint:\n{pylint_process.stdout}') + return [] return _pylint_extract_relevant_warnings(pylint_json) def _pylint_extract_relevant_warnings(pylint_json): - issues = list() + issues = [] for issue in pylint_json: if issue['type'] in ['error', 'warning']: for unnecessary_information in ['module', 'obj', 'path', 'message-id']: @@ -141,27 +150,32 @@ def _pylint_extract_relevant_warnings(pylint_json): return issues -def run_rubocop(file_path): - rubocop_p = subprocess.run( - shlex.split(f'rubocop --format json {file_path}'), - stdout=PIPE, - stderr=DEVNULL, - check=False, +def run_rubocop(file_path: str) -> List[dict]: + container_path = '/input' + process = run_docker_container( + 'pipelinecomponents/rubocop:latest', + combine_stderr_stdout=False, + mounts=[ + Mount(container_path, file_path, type='bind', read_only=True), + ], + command=f'rubocop --format json -- {container_path}', ) - linter_output = json.loads(rubocop_p.stdout) - - issues = [] - for offense in linter_output['files'][0]['offenses']: - issues.append( - { - 'symbol': offense['cop_name'], - 'line': offense['location']['start_line'], - 'column': offense['location']['column'], - 'message': offense['message'], - } - ) - return issues + try: + linter_output = json.loads(process.stdout) + except json.JSONDecodeError: + logging.warning(f'Failed to execute rubocop linter:\n{process.stderr}') + return [] + + return [ + { + 'symbol': offense['cop_name'], + 'line': offense['location']['start_line'], + 'column': offense['location']['column'], + 'message': offense['message'], + } + for offense in linter_output['files'][0]['offenses'] + ] def run_phpstan(file_path): From 1f5276b411de8593d277d118491caa481ed8f5a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Stucke?= Date: Mon, 9 May 2022 17:03:03 +0200 Subject: [PATCH 2/2] test bugfix --- src/plugins/analysis/linter/test/test_ruby_linter.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/plugins/analysis/linter/test/test_ruby_linter.py b/src/plugins/analysis/linter/test/test_ruby_linter.py index 7873610ff..e6b8e639e 100644 --- a/src/plugins/analysis/linter/test/test_ruby_linter.py +++ b/src/plugins/analysis/linter/test/test_ruby_linter.py @@ -44,7 +44,10 @@ def test_do_analysis(monkeypatch): - monkeypatch.setattr('plugins.analysis.linter.internal.linters.subprocess.run', lambda *_, **__: CompletedProcess('args', 0, stdout=MOCK_RESPONSE)) + monkeypatch.setattr( + 'plugins.analysis.linter.internal.linters.run_docker_container', + lambda *_, **__: CompletedProcess('args', 0, stdout=MOCK_RESPONSE) + ) result = run_rubocop('any/path') assert len(result) == 1