From 5549998c4a4e161fbaf02196104c3b4e1b8bdeb6 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 3 Jan 2025 21:20:07 +0000 Subject: [PATCH 01/11] wip --- .github/workflows/main.yml | 43 ++++++++++++++++++ .gitignore | 77 ++++++++++++++++++++++++++++++++ .yamlfix.toml | 17 +++++++ README.md | 9 ++++ pyproject.toml | 37 +++++++++++++++ src/pytest_nm_releng/__init__.py | 13 ++++++ src/pytest_nm_releng/plugin.py | 53 ++++++++++++++++++++++ tests/conftest.py | 1 + tests/test_helpers.py | 6 +++ tests/test_nm_releng.py | 13 ++++++ tox.ini | 37 +++++++++++++++ 11 files changed, 306 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 .gitignore create mode 100644 .yamlfix.toml create mode 100644 pyproject.toml create mode 100644 src/pytest_nm_releng/__init__.py create mode 100644 src/pytest_nm_releng/plugin.py create mode 100644 tests/conftest.py create mode 100644 tests/test_helpers.py create mode 100644 tests/test_nm_releng.py create mode 100644 tox.ini diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..fab7483 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,43 @@ +--- +name: main + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + style: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: 3.9 + + - name: Install tox + run: pip install tox + + - name: Run style check + run: tox -e style + + test: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - python-version: ['3.9', '3.10', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e94d899 --- /dev/null +++ b/.gitignore @@ -0,0 +1,77 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +.pytest_cache + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask instance folder +instance/ + +# Sphinx documentation +docs/_build/ + +# MkDocs documentation +/site/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + + +# ruff +.ruff_cache diff --git a/.yamlfix.toml b/.yamlfix.toml new file mode 100644 index 0000000..a6e2298 --- /dev/null +++ b/.yamlfix.toml @@ -0,0 +1,17 @@ +allow_duplicate_keys = false +comments_min_spaces_from_content = 2 +comments_require_starting_space = true +whitelines = 1 +comments_whitelines = 1 +section_whitelines = 1 +explicit_start = true +sequence_style = "flow_style" +indent_mapping = 2 +ident_offset = 2 +indent_sequence = 4 +line_length = 88 +none_representation = "null" +quote_basic_values = false +quote_keys_and_basic_values = false +quote_representation = "'" +preserve_quotes = false diff --git a/README.md b/README.md index c5eac6a..3354cea 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,11 @@ # pytest-nm-releng + pytest plugin used by Release Engineering + +## Acknowledgements + +This pytest plugin was generated with [Cookiecutter] along with [@hackebrot]'s [cookiecutter-pytest-plugin] template. + +[@hackebrot]: https://github.com/hackebrot +[cookiecutter]: https://github.com/audreyr/cookiecutter +[cookiecutter-pytest-plugin]: https://github.com/pytest-dev/cookiecutter-pytest-plugin diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..6df6db3 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pytest-nm-releng" +description = "A pytest plugin providing custom functionality for the Neural Magic release engineering team." +version = "0.1.0" +readme = "README.md" +requires-python = ">=3.9" +authors = [{ name = "Domenic Barbuzzi", email = "domenic@neuralmagic.com" }] +maintainers = [{ name = "Domenic Barbuzzi", email = "domenic@neuralmagic.com" }] +license = { file = "LICENSE" } +classifiers = [ + "Framework :: Pytest", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Topic :: Software Development :: Testing", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "License :: OSI Approved :: Apache Software License", +] +dependencies = [ + "pytest>=8", +] + +[project.urls] +Repository = "https://github.com/neuralmagic/pytest-nm-releng" + +[project.entry-points.pytest11] +nm-releng = "pytest_nm_releng.plugin" diff --git a/src/pytest_nm_releng/__init__.py b/src/pytest_nm_releng/__init__.py new file mode 100644 index 0000000..c40afb2 --- /dev/null +++ b/src/pytest_nm_releng/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/pytest_nm_releng/plugin.py b/src/pytest_nm_releng/plugin.py new file mode 100644 index 0000000..656434d --- /dev/null +++ b/src/pytest_nm_releng/plugin.py @@ -0,0 +1,53 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from datetime import datetime, timezone +from pathlib import Path + + +def get_utc_timestamp() -> float: + return datetime.now(timezone.utc).timestamp() + + +def generate_junit_flags() -> list[str]: + if not (junitxml_base_dir := os.getenv("PDRN_JUNIT_BASE")): + return [] + + junitxml_file = Path(junitxml_base_dir) / f"{get_utc_timestamp()}.xml" + + if prefix := os.getenv("PDRN_JUNIT_PREFIX"): + junitxml_file = junitxml_file.with_name(f"{prefix}{junitxml_file.name}") + + return [f"--junit-xml={junitxml_file.as_posix()}"] + + +def generate_coverage_flags() -> list[str]: + if not (cc_package_name := os.getenv("PDRN_COV_NAME")): + return [] + + return [ + f"--cov={cc_package_name}", + "--cov-append", + "--cov-report=html:coverage-html", + "--cov-report=json:coverage.json", + ] + + +def pytest_load_initial_conftests(early_config, args: list[str], parser): + new_args: list[str] = [] + new_args.extend(generate_junit_flags()) + new_args.extend(generate_coverage_flags()) + args[:] = [*args, *new_args] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..694d7d5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = "pytester" diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..b6a9f90 --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,6 @@ +def test_generate_coverage_flags(): + pass + + +def test_generate_junit_flags(): + pass diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py new file mode 100644 index 0000000..9314a79 --- /dev/null +++ b/tests/test_nm_releng.py @@ -0,0 +1,13 @@ +import pytest + + +def test_plugin_loaded(pytester: pytest.Pytester): + """Verify the plugin is loaded by pytest""" + + pytester.makepyfile(""" + def test_pass(): + pass + """) + + result = pytester.runpytest() + result.stdout.fnmatch_lines(["plugins:*nm-releng-*"]) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..159bd35 --- /dev/null +++ b/tox.ini @@ -0,0 +1,37 @@ +# For more information about tox, see https://tox.readthedocs.io/en/latest/ +[tox] +envlist = py39,py310,py311,py312 + +[testenv] +deps = pytest>=8 +commands = pytest {posargs:tests} + +[testenv:style] +skip_install = true +deps = + ruff ~= 0.8.5 + mdformat ~= 0.7.21 + mdformat-footnote ~= 0.1.1 + mdformat-frontmatter ~= 2.0.8 + mdformat-gfm ~= 0.4.1 + yamlfix ~= 1.16 +commands = + ruff check src tests + ruff format --check --diff src tests + mdformat --check README.md + yamlfix --check --config-file .yamlfix.toml .github + +[testenv:format] +skip_install = true +deps = + ruff ~= 0.8.5 + mdformat ~= 0.7.21 + mdformat-footnote ~= 0.1.1 + mdformat-frontmatter ~= 2.0.8 + mdformat-gfm ~= 0.4.1 + yamlfix ~= 1.16 +commands = + ruff check --fix src tests + ruff format src tests + mdformat README.md + yamlfix --config-file .yamlfix.toml .github From cb8b2f4d24300fdabe303fe7f55fea649f1a06de Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 3 Jan 2025 21:26:30 +0000 Subject: [PATCH 02/11] wip --- tests/test_nm_releng.py | 6 +++++- tox.ini | 6 ++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py index 9314a79..fa3cac4 100644 --- a/tests/test_nm_releng.py +++ b/tests/test_nm_releng.py @@ -1,13 +1,17 @@ +from importlib.metadata import version + import pytest def test_plugin_loaded(pytester: pytest.Pytester): """Verify the plugin is loaded by pytest""" + plugin_version = version("pytest-nm-releng") + pytester.makepyfile(""" def test_pass(): pass """) result = pytester.runpytest() - result.stdout.fnmatch_lines(["plugins:*nm-releng-*"]) + result.stdout.fnmatch_lines([f"plugins:*nm-releng-{plugin_version}*"]) diff --git a/tox.ini b/tox.ini index 159bd35..09f75ba 100644 --- a/tox.ini +++ b/tox.ini @@ -3,8 +3,10 @@ envlist = py39,py310,py311,py312 [testenv] -deps = pytest>=8 -commands = pytest {posargs:tests} +deps = + pytest >= 8 +commands = + pytest {posargs:tests} [testenv:style] skip_install = true From 1d39e319bc8ef7dba9c16a0d3b5569131179292b Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 3 Jan 2025 21:30:37 +0000 Subject: [PATCH 03/11] wip --- tests/conftest.py | 15 +++++++++++++++ tests/test_helpers.py | 15 +++++++++++++++ tests/test_nm_releng.py | 15 +++++++++++++++ 3 files changed, 45 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 694d7d5..f97cb63 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,16 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + pytest_plugins = "pytester" diff --git a/tests/test_helpers.py b/tests/test_helpers.py index b6a9f90..33d346d 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,3 +1,18 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + def test_generate_coverage_flags(): pass diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py index fa3cac4..d2c27a2 100644 --- a/tests/test_nm_releng.py +++ b/tests/test_nm_releng.py @@ -1,3 +1,18 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from importlib.metadata import version import pytest From 8262dd809925dca5cf852a7404aa4122bfbc0b4d Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 3 Jan 2025 21:41:42 +0000 Subject: [PATCH 04/11] wip --- .github/workflows/main.yml | 2 +- README.md | 28 +++++++++++++++++++++++++++- src/pytest_nm_releng/plugin.py | 6 +++--- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fab7483..859cd5c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: '3.9' - name: Install tox run: pip install tox diff --git a/README.md b/README.md index 3354cea..5300d73 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,32 @@ # pytest-nm-releng -pytest plugin used by Release Engineering +A pytest plugin offering various functionality for Neural Magic’s Release Engineering team. + +## Contributing + +To contribute, follow these general steps: + +1. Fork the repository +2. Create a new branch +3. Make your changes +4. Install `tox` + ```shell + # example: using pipx + pipx install tox + # example: using uv + uv tool install tox --with tox-uv + ``` +5. Run quality checks and tests + ```shell + # apply available automatic style/formatting fixes + tox -e format + # check style/formatting + tox -e style + # run tests + tox -e py + ``` +6. Submit a pull request with your changes + ## Acknowledgements diff --git a/src/pytest_nm_releng/plugin.py b/src/pytest_nm_releng/plugin.py index 656434d..f26e231 100644 --- a/src/pytest_nm_releng/plugin.py +++ b/src/pytest_nm_releng/plugin.py @@ -23,19 +23,19 @@ def get_utc_timestamp() -> float: def generate_junit_flags() -> list[str]: - if not (junitxml_base_dir := os.getenv("PDRN_JUNIT_BASE")): + if not (junitxml_base_dir := os.getenv("NMRE_JUNIT_BASE")): return [] junitxml_file = Path(junitxml_base_dir) / f"{get_utc_timestamp()}.xml" - if prefix := os.getenv("PDRN_JUNIT_PREFIX"): + if prefix := os.getenv("NMRE_JUNIT_PREFIX"): junitxml_file = junitxml_file.with_name(f"{prefix}{junitxml_file.name}") return [f"--junit-xml={junitxml_file.as_posix()}"] def generate_coverage_flags() -> list[str]: - if not (cc_package_name := os.getenv("PDRN_COV_NAME")): + if not (cc_package_name := os.getenv("NMRE_COV_NAME")): return [] return [ From 4e571e91a12c6e8dbfd9ef04711ad8029a092d06 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Fri, 3 Jan 2025 21:59:37 +0000 Subject: [PATCH 05/11] wip --- README.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/README.md b/README.md index 5300d73..dad8704 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,58 @@ A pytest plugin offering various functionality for Neural Magic’s Release Engineering team. +## Features + +### Dynamically-named JUnit report files + +`pytest-nm-releng` can automatically generate unique, dynamically-named JUnit report files with an optional prefix. The report file is generated when the test run begins using Python’s `datetime.timestamp()` method (UTC). + +> [!NOTE] +> This works by appending the `--junit-xml` flag after the command is run, meaning it will override any previously-specified instances of this flag. + +To enable this behavior, define the environment variable `NMRE_JUNIT_BASE` with a value to the path where the test files should be stored. This can be absolute or relative. + +The following examples will both write JUnit report files in a folder named "test-results" in the current working directory. + +```shell +# example: prefixing a command +NMRE_JUNIT_BASE=test-results pytest [...] + +# example: export the environment variable (useful if pytest is not being +# invoked directly) +export NMRE_JUNIT_BASE=test-results +pytest [...] + +# after either example, a file named something like +# `test-results/1735941024.348248.xml` will be written +``` + +Optionally, you can define `NMRE_JUNIT_PREFIX` with a value to be prefixed onto the file name. Note that no separator is used so you may want to include one. + + +```shell +export NMRE_JUNIT_BASE=test-results +export NMRE_JUNIT_PREFIX="report-" +pytest [...] + +# after either example, a file named something like +# `test-results/report-1735941218.338192.xml` will be written +``` + +### Code coverage + +`pytest-nm-releng` can automatically add some code coverage flags as well (requires [pytest-cov]). + +To enable this behavior, define the `NMRE_COV_NAME` environment variable with a value of the project’s *_module_* name (e.g., the name that is used to import it within Python code). + +```shell +# example: used with `nm-vllm-ent`, which is imported as `vllm` +NMRE_COV_NAME=vllm pytest [...] + +# this will result in the following flags being appended: +# --cov=vllm --cov-append --cov-report=html:coverage-html --cov-report=json:coverage.json +``` + ## Contributing To contribute, follow these general steps: @@ -32,6 +84,7 @@ To contribute, follow these general steps: This pytest plugin was generated with [Cookiecutter] along with [@hackebrot]'s [cookiecutter-pytest-plugin] template. +[pytest-cov]: https://github.com/pytest-dev/pytest-cov [@hackebrot]: https://github.com/hackebrot [cookiecutter]: https://github.com/audreyr/cookiecutter [cookiecutter-pytest-plugin]: https://github.com/pytest-dev/cookiecutter-pytest-plugin From 51491ba947a960026841c0530fee3e1dadcb2b48 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Tue, 7 Jan 2025 16:54:27 +0000 Subject: [PATCH 06/11] add unit tests --- tests/test_helpers.py | 97 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 93 insertions(+), 4 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 33d346d..47b6947 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -12,10 +12,99 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import re +from typing import Union -def test_generate_coverage_flags(): - pass +import pytest +from pytest_nm_releng.plugin import generate_coverage_flags, generate_junit_flags -def test_generate_junit_flags(): - pass +EnvVarValue = Union[str, None] + + +def _setenv( + monkeypatch: pytest.MonkeyPatch, name: str, value: Union[str, None] +) -> None: + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) + + +@pytest.mark.parametrize( + "env_cov_name", + [ + pytest.param("vllm", id="value:vllm"), + pytest.param("llmcompressor", id="value:llmcompressor"), + pytest.param("", id="empty"), + pytest.param(None, id="unset"), + ], +) +def test_generate_coverage_flags_set( + monkeypatch: pytest.MonkeyPatch, env_cov_name: EnvVarValue +): + _setenv(monkeypatch, "NMRE_COV_NAME", env_cov_name) + + if env_cov_name in (None, ""): + expected_flags = [] + else: + expected_flags = [ + f"--cov={env_cov_name}", + "--cov-append", + "--cov-report=html:coverage-html", + "--cov-report=json:coverage.json", + ] + + result = generate_coverage_flags() + assert result == expected_flags + + +@pytest.mark.parametrize( + ("env_junit_base", "env_junit_prefix"), + [ + pytest.param("test-results", None, id="base-with-unset-prefix"), + pytest.param("test-results", "", id="base-with-empty-prefix"), + pytest.param("test-results", "run-", id="base-with-prefix"), + pytest.param(None, None, id="unset-base-with-unset-prefix"), + pytest.param(None, "", id="unset-base-with-empty-prefix"), + pytest.param(None, "run-", id="unset-base-with-prefix"), + pytest.param("", None, id="empty-base-with-unset-prefix"), + pytest.param("", "", id="empty-base-with-empty-prefix"), + pytest.param("", "run-", id="empty-base-with-prefix"), + ], +) +def test_generate_junit_flags( + monkeypatch: pytest.MonkeyPatch, + env_junit_base: EnvVarValue, + env_junit_prefix: EnvVarValue, +): + _setenv(monkeypatch, "NMRE_JUNIT_BASE", env_junit_base) + _setenv(monkeypatch, "NMRE_JUNIT_PREFIX", env_junit_prefix) + + result = generate_junit_flags() + + if env_junit_base in (None, ""): + assert result == [] + return + + # basic assertions + assert len(result) == 1 + flag = result[0] + assert flag.startswith("--junit-xml") + assert flag.endswith(".xml") + + # assertions about flag value structure + value = flag[12:] + assert os.sep in value + + # assertions about flag value contents + fpath, fname = value.rsplit(os.sep, 1) + assert fpath == env_junit_base + + pattern = r"\d{10,}\.\d+\.xml" + if env_junit_prefix not in (None, ""): + pattern = f"{env_junit_prefix}{pattern}" + + pattern = re.compile(pattern) + assert pattern.fullmatch(fname) From d823013d8eeec00bb21ea3c236aa52d41a635a5a Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Tue, 7 Jan 2025 16:57:06 +0000 Subject: [PATCH 07/11] wip --- tests/test_helpers.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 47b6947..03276c1 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -36,7 +36,6 @@ def _setenv( "env_cov_name", [ pytest.param("vllm", id="value:vllm"), - pytest.param("llmcompressor", id="value:llmcompressor"), pytest.param("", id="empty"), pytest.param(None, id="unset"), ], @@ -63,15 +62,15 @@ def test_generate_coverage_flags_set( @pytest.mark.parametrize( ("env_junit_base", "env_junit_prefix"), [ - pytest.param("test-results", None, id="base-with-unset-prefix"), - pytest.param("test-results", "", id="base-with-empty-prefix"), pytest.param("test-results", "run-", id="base-with-prefix"), - pytest.param(None, None, id="unset-base-with-unset-prefix"), - pytest.param(None, "", id="unset-base-with-empty-prefix"), - pytest.param(None, "run-", id="unset-base-with-prefix"), - pytest.param("", None, id="empty-base-with-unset-prefix"), - pytest.param("", "", id="empty-base-with-empty-prefix"), + pytest.param("test-results", "", id="base-with-empty-prefix"), + pytest.param("test-results", None, id="base-with-unset-prefix"), pytest.param("", "run-", id="empty-base-with-prefix"), + pytest.param("", "", id="empty-base-with-empty-prefix"), + pytest.param("", None, id="empty-base-with-unset-prefix"), + pytest.param(None, "run-", id="unset-base-with-prefix"), + pytest.param(None, "", id="unset-base-with-empty-prefix"), + pytest.param(None, None, id="unset-base-with-unset-prefix"), ], ) def test_generate_junit_flags( From d9d2b706ecc6808b61fb264b1952977b89b1ba7b Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Tue, 7 Jan 2025 17:27:36 +0000 Subject: [PATCH 08/11] wip --- README.md | 42 ++++++++++++++++----------------- pyproject.toml | 3 +++ tests/__init__.py | 0 tests/test_helpers.py | 16 ++++--------- tests/test_nm_releng.py | 52 ++++++++++++++++++++++++++++++++++++++++- tests/utils.py | 10 ++++++++ 6 files changed, 88 insertions(+), 35 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/utils.py diff --git a/README.md b/README.md index dad8704..cb22764 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,6 @@ pytest [...] Optionally, you can define `NMRE_JUNIT_PREFIX` with a value to be prefixed onto the file name. Note that no separator is used so you may want to include one. - ```shell export NMRE_JUNIT_BASE=test-results export NMRE_JUNIT_PREFIX="report-" @@ -59,32 +58,31 @@ NMRE_COV_NAME=vllm pytest [...] To contribute, follow these general steps: 1. Fork the repository -2. Create a new branch -3. Make your changes -4. Install `tox` - ```shell - # example: using pipx - pipx install tox - # example: using uv - uv tool install tox --with tox-uv - ``` -5. Run quality checks and tests - ```shell - # apply available automatic style/formatting fixes - tox -e format - # check style/formatting - tox -e style - # run tests - tox -e py - ``` -6. Submit a pull request with your changes - +1. Create a new branch +1. Make your changes +1. Install `tox` + ```shell + # example: using pipx + pipx install tox + # example: using uv + uv tool install tox --with tox-uv + ``` +1. Run quality checks and tests + ```shell + # apply available automatic style/formatting fixes + tox -e format + # check style/formatting + tox -e style + # run tests + tox -e py + ``` +1. Submit a pull request with your changes ## Acknowledgements This pytest plugin was generated with [Cookiecutter] along with [@hackebrot]'s [cookiecutter-pytest-plugin] template. -[pytest-cov]: https://github.com/pytest-dev/pytest-cov [@hackebrot]: https://github.com/hackebrot [cookiecutter]: https://github.com/audreyr/cookiecutter [cookiecutter-pytest-plugin]: https://github.com/pytest-dev/cookiecutter-pytest-plugin +[pytest-cov]: https://github.com/pytest-dev/pytest-cov diff --git a/pyproject.toml b/pyproject.toml index 6df6db3..365b115 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,3 +35,6 @@ Repository = "https://github.com/neuralmagic/pytest-nm-releng" [project.entry-points.pytest11] nm-releng = "pytest_nm_releng.plugin" + +[tool.ruff.lint] +extend-select = ["I"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 03276c1..fda16dd 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -19,19 +19,11 @@ import pytest from pytest_nm_releng.plugin import generate_coverage_flags, generate_junit_flags +from tests.utils import setenv EnvVarValue = Union[str, None] -def _setenv( - monkeypatch: pytest.MonkeyPatch, name: str, value: Union[str, None] -) -> None: - if value is None: - monkeypatch.delenv(name, raising=False) - else: - monkeypatch.setenv(name, value) - - @pytest.mark.parametrize( "env_cov_name", [ @@ -43,7 +35,7 @@ def _setenv( def test_generate_coverage_flags_set( monkeypatch: pytest.MonkeyPatch, env_cov_name: EnvVarValue ): - _setenv(monkeypatch, "NMRE_COV_NAME", env_cov_name) + setenv(monkeypatch, "NMRE_COV_NAME", env_cov_name) if env_cov_name in (None, ""): expected_flags = [] @@ -78,8 +70,8 @@ def test_generate_junit_flags( env_junit_base: EnvVarValue, env_junit_prefix: EnvVarValue, ): - _setenv(monkeypatch, "NMRE_JUNIT_BASE", env_junit_base) - _setenv(monkeypatch, "NMRE_JUNIT_PREFIX", env_junit_prefix) + setenv(monkeypatch, "NMRE_JUNIT_BASE", env_junit_base) + setenv(monkeypatch, "NMRE_JUNIT_PREFIX", env_junit_prefix) result = generate_junit_flags() diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py index d2c27a2..33b478b 100644 --- a/tests/test_nm_releng.py +++ b/tests/test_nm_releng.py @@ -13,11 +13,18 @@ # limitations under the License. -from importlib.metadata import version +from importlib.metadata import PackageNotFoundError, version import pytest +try: + version("pytest-cov") + _pytest_cov_installed = True +except PackageNotFoundError: + _pytest_cov_installed = False + + def test_plugin_loaded(pytester: pytest.Pytester): """Verify the plugin is loaded by pytest""" @@ -30,3 +37,46 @@ def test_pass(): result = pytester.runpytest() result.stdout.fnmatch_lines([f"plugins:*nm-releng-{plugin_version}*"]) + + +def test_plugin_adds_junit_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected junit args""" + + monkeypatch.setenv("NMRE_JUNIT_BASE", "results") + + cf = pytester.parseconfigure() + actual = cf.getoption("--junit-xml", None) + assert actual is not None + assert actual.startswith("results") + + +@pytest.mark.skipif(not _pytest_cov_installed, reason="pytest-cov is required") +def test_plugin_adds_coverage_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected coverage args""" + + monkeypatch.setenv("NMRE_COV_NAME", "vllm") + + cf = pytester.parseconfigure() + + assert "vllm" in cf.getoption("--cov", None) + assert cf.getoption("--cov-append") is True + + +@pytest.mark.skipif(not _pytest_cov_installed, reason="pytest-cov is required") +def test_plugin_adds_all_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected coverage args""" + + monkeypatch.setenv("NMRE_JUNIT_BASE", "results") + monkeypatch.setenv("NMRE_COV_NAME", "vllm") + + cf = pytester.parseconfigure() + + assert cf.getoption("--junit-xml", None).startswith("results") + assert "vllm" in cf.getoption("--cov", None) + assert cf.getoption("--cov-append") is True diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..56c8c69 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,10 @@ +from typing import Union + +import pytest + + +def setenv(monkeypatch: pytest.MonkeyPatch, name: str, value: Union[str, None]) -> None: + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) From 8ffe9c071e4bd9856fbbc7fb65fcb2c631934c56 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Tue, 7 Jan 2025 17:40:01 +0000 Subject: [PATCH 09/11] wip --- tests/test_nm_releng.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py index 33b478b..1010c3f 100644 --- a/tests/test_nm_releng.py +++ b/tests/test_nm_releng.py @@ -17,7 +17,6 @@ import pytest - try: version("pytest-cov") _pytest_cov_installed = True From 9e0d5f67b1fc8b3c8e00683ebd380882ad618315 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Wed, 8 Jan 2025 20:14:43 +0000 Subject: [PATCH 10/11] wip --- README.md | 16 ++++++++++++++-- tests/__init__.py | 13 +++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cb22764..c87770b 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,18 @@ A pytest plugin offering various functionality for Neural Magic’s Release Engineering team. +## Installation + +The Python package can be installed directly from this git repository from either a branch or tag: + +```shell +# recommended: use a version tag (e.g., v0.1.0) +pip install https://github.com/neuralmagic/pytest-nm-releng/archive/v0.1.0.tar.gz + +# alternative: install based on a branch (e.g., main) +pip install https://github.com/neuralmagic/pytest-nm-releng/archive/main.tar.gz +``` + ## Features ### Dynamically-named JUnit report files @@ -25,7 +37,7 @@ export NMRE_JUNIT_BASE=test-results pytest [...] # after either example, a file named something like -# `test-results/1735941024.348248.xml` will be written +# `test-results/1735941024.348248.xml` will be created ``` Optionally, you can define `NMRE_JUNIT_PREFIX` with a value to be prefixed onto the file name. Note that no separator is used so you may want to include one. @@ -36,7 +48,7 @@ export NMRE_JUNIT_PREFIX="report-" pytest [...] # after either example, a file named something like -# `test-results/report-1735941218.338192.xml` will be written +# `test-results/report-1735941218.338192.xml` will be created ``` ### Code coverage diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..c40afb2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 8f6094e5b97d9684c84c69356db6f886f77fca08 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Wed, 8 Jan 2025 20:18:30 +0000 Subject: [PATCH 11/11] fix workflow --- .github/workflows/main.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 859cd5c..7d53427 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,8 +27,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - include: - - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.9', '3.10', '3.11', '3.12'] steps: - uses: actions/checkout@v4