diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..7d53427 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,42 @@ +--- +name: main + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + style: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.9' + + - name: Install tox + run: pip install tox + + - name: Run style check + run: tox -e style + + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install tox + run: pip install tox + + - name: Run tests + run: tox -e py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e94d899 --- /dev/null +++ b/.gitignore @@ -0,0 +1,77 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ +.pytest_cache + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask instance folder +instance/ + +# Sphinx documentation +docs/_build/ + +# MkDocs documentation +/site/ + +# PyBuilder +target/ + +# IPython Notebook +.ipynb_checkpoints + +# pyenv +.python-version + + +# ruff +.ruff_cache diff --git a/.yamlfix.toml b/.yamlfix.toml new file mode 100644 index 0000000..a6e2298 --- /dev/null +++ b/.yamlfix.toml @@ -0,0 +1,17 @@ +allow_duplicate_keys = false +comments_min_spaces_from_content = 2 +comments_require_starting_space = true +whitelines = 1 +comments_whitelines = 1 +section_whitelines = 1 +explicit_start = true +sequence_style = "flow_style" +indent_mapping = 2 +ident_offset = 2 +indent_sequence = 4 +line_length = 88 +none_representation = "null" +quote_basic_values = false +quote_keys_and_basic_values = false +quote_representation = "'" +preserve_quotes = false diff --git a/README.md b/README.md index c5eac6a..c87770b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,100 @@ # pytest-nm-releng -pytest plugin used by Release Engineering + +A pytest plugin offering various functionality for Neural Magic’s Release Engineering team. + +## Installation + +The Python package can be installed directly from this git repository from either a branch or tag: + +```shell +# recommended: use a version tag (e.g., v0.1.0) +pip install https://github.com/neuralmagic/pytest-nm-releng/archive/v0.1.0.tar.gz + +# alternative: install based on a branch (e.g., main) +pip install https://github.com/neuralmagic/pytest-nm-releng/archive/main.tar.gz +``` + +## Features + +### Dynamically-named JUnit report files + +`pytest-nm-releng` can automatically generate unique, dynamically-named JUnit report files with an optional prefix. The report file is generated when the test run begins using Python’s `datetime.timestamp()` method (UTC). + +> [!NOTE] +> This works by appending the `--junit-xml` flag after the command is run, meaning it will override any previously-specified instances of this flag. + +To enable this behavior, define the environment variable `NMRE_JUNIT_BASE` with a value to the path where the test files should be stored. This can be absolute or relative. + +The following examples will both write JUnit report files in a folder named "test-results" in the current working directory. + +```shell +# example: prefixing a command +NMRE_JUNIT_BASE=test-results pytest [...] + +# example: export the environment variable (useful if pytest is not being +# invoked directly) +export NMRE_JUNIT_BASE=test-results +pytest [...] + +# after either example, a file named something like +# `test-results/1735941024.348248.xml` will be created +``` + +Optionally, you can define `NMRE_JUNIT_PREFIX` with a value to be prefixed onto the file name. Note that no separator is used so you may want to include one. + +```shell +export NMRE_JUNIT_BASE=test-results +export NMRE_JUNIT_PREFIX="report-" +pytest [...] + +# after either example, a file named something like +# `test-results/report-1735941218.338192.xml` will be created +``` + +### Code coverage + +`pytest-nm-releng` can automatically add some code coverage flags as well (requires [pytest-cov]). + +To enable this behavior, define the `NMRE_COV_NAME` environment variable with a value of the project’s *_module_* name (e.g., the name that is used to import it within Python code). + +```shell +# example: used with `nm-vllm-ent`, which is imported as `vllm` +NMRE_COV_NAME=vllm pytest [...] + +# this will result in the following flags being appended: +# --cov=vllm --cov-append --cov-report=html:coverage-html --cov-report=json:coverage.json +``` + +## Contributing + +To contribute, follow these general steps: + +1. Fork the repository +1. Create a new branch +1. Make your changes +1. Install `tox` + ```shell + # example: using pipx + pipx install tox + # example: using uv + uv tool install tox --with tox-uv + ``` +1. Run quality checks and tests + ```shell + # apply available automatic style/formatting fixes + tox -e format + # check style/formatting + tox -e style + # run tests + tox -e py + ``` +1. Submit a pull request with your changes + +## Acknowledgements + +This pytest plugin was generated with [Cookiecutter] along with [@hackebrot]'s [cookiecutter-pytest-plugin] template. + +[@hackebrot]: https://github.com/hackebrot +[cookiecutter]: https://github.com/audreyr/cookiecutter +[cookiecutter-pytest-plugin]: https://github.com/pytest-dev/cookiecutter-pytest-plugin +[pytest-cov]: https://github.com/pytest-dev/pytest-cov diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..365b115 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "pytest-nm-releng" +description = "A pytest plugin providing custom functionality for the Neural Magic release engineering team." +version = "0.1.0" +readme = "README.md" +requires-python = ">=3.9" +authors = [{ name = "Domenic Barbuzzi", email = "domenic@neuralmagic.com" }] +maintainers = [{ name = "Domenic Barbuzzi", email = "domenic@neuralmagic.com" }] +license = { file = "LICENSE" } +classifiers = [ + "Framework :: Pytest", + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Topic :: Software Development :: Testing", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: Implementation :: CPython", + "License :: OSI Approved :: Apache Software License", +] +dependencies = [ + "pytest>=8", +] + +[project.urls] +Repository = "https://github.com/neuralmagic/pytest-nm-releng" + +[project.entry-points.pytest11] +nm-releng = "pytest_nm_releng.plugin" + +[tool.ruff.lint] +extend-select = ["I"] diff --git a/src/pytest_nm_releng/__init__.py b/src/pytest_nm_releng/__init__.py new file mode 100644 index 0000000..c40afb2 --- /dev/null +++ b/src/pytest_nm_releng/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/src/pytest_nm_releng/plugin.py b/src/pytest_nm_releng/plugin.py new file mode 100644 index 0000000..f26e231 --- /dev/null +++ b/src/pytest_nm_releng/plugin.py @@ -0,0 +1,53 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +from datetime import datetime, timezone +from pathlib import Path + + +def get_utc_timestamp() -> float: + return datetime.now(timezone.utc).timestamp() + + +def generate_junit_flags() -> list[str]: + if not (junitxml_base_dir := os.getenv("NMRE_JUNIT_BASE")): + return [] + + junitxml_file = Path(junitxml_base_dir) / f"{get_utc_timestamp()}.xml" + + if prefix := os.getenv("NMRE_JUNIT_PREFIX"): + junitxml_file = junitxml_file.with_name(f"{prefix}{junitxml_file.name}") + + return [f"--junit-xml={junitxml_file.as_posix()}"] + + +def generate_coverage_flags() -> list[str]: + if not (cc_package_name := os.getenv("NMRE_COV_NAME")): + return [] + + return [ + f"--cov={cc_package_name}", + "--cov-append", + "--cov-report=html:coverage-html", + "--cov-report=json:coverage.json", + ] + + +def pytest_load_initial_conftests(early_config, args: list[str], parser): + new_args: list[str] = [] + new_args.extend(generate_junit_flags()) + new_args.extend(generate_coverage_flags()) + args[:] = [*args, *new_args] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..c40afb2 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..f97cb63 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,16 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +pytest_plugins = "pytester" diff --git a/tests/test_helpers.py b/tests/test_helpers.py new file mode 100644 index 0000000..fda16dd --- /dev/null +++ b/tests/test_helpers.py @@ -0,0 +1,101 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +from typing import Union + +import pytest + +from pytest_nm_releng.plugin import generate_coverage_flags, generate_junit_flags +from tests.utils import setenv + +EnvVarValue = Union[str, None] + + +@pytest.mark.parametrize( + "env_cov_name", + [ + pytest.param("vllm", id="value:vllm"), + pytest.param("", id="empty"), + pytest.param(None, id="unset"), + ], +) +def test_generate_coverage_flags_set( + monkeypatch: pytest.MonkeyPatch, env_cov_name: EnvVarValue +): + setenv(monkeypatch, "NMRE_COV_NAME", env_cov_name) + + if env_cov_name in (None, ""): + expected_flags = [] + else: + expected_flags = [ + f"--cov={env_cov_name}", + "--cov-append", + "--cov-report=html:coverage-html", + "--cov-report=json:coverage.json", + ] + + result = generate_coverage_flags() + assert result == expected_flags + + +@pytest.mark.parametrize( + ("env_junit_base", "env_junit_prefix"), + [ + pytest.param("test-results", "run-", id="base-with-prefix"), + pytest.param("test-results", "", id="base-with-empty-prefix"), + pytest.param("test-results", None, id="base-with-unset-prefix"), + pytest.param("", "run-", id="empty-base-with-prefix"), + pytest.param("", "", id="empty-base-with-empty-prefix"), + pytest.param("", None, id="empty-base-with-unset-prefix"), + pytest.param(None, "run-", id="unset-base-with-prefix"), + pytest.param(None, "", id="unset-base-with-empty-prefix"), + pytest.param(None, None, id="unset-base-with-unset-prefix"), + ], +) +def test_generate_junit_flags( + monkeypatch: pytest.MonkeyPatch, + env_junit_base: EnvVarValue, + env_junit_prefix: EnvVarValue, +): + setenv(monkeypatch, "NMRE_JUNIT_BASE", env_junit_base) + setenv(monkeypatch, "NMRE_JUNIT_PREFIX", env_junit_prefix) + + result = generate_junit_flags() + + if env_junit_base in (None, ""): + assert result == [] + return + + # basic assertions + assert len(result) == 1 + flag = result[0] + assert flag.startswith("--junit-xml") + assert flag.endswith(".xml") + + # assertions about flag value structure + value = flag[12:] + assert os.sep in value + + # assertions about flag value contents + fpath, fname = value.rsplit(os.sep, 1) + assert fpath == env_junit_base + + pattern = r"\d{10,}\.\d+\.xml" + if env_junit_prefix not in (None, ""): + pattern = f"{env_junit_prefix}{pattern}" + + pattern = re.compile(pattern) + assert pattern.fullmatch(fname) diff --git a/tests/test_nm_releng.py b/tests/test_nm_releng.py new file mode 100644 index 0000000..1010c3f --- /dev/null +++ b/tests/test_nm_releng.py @@ -0,0 +1,81 @@ +# Copyright (c) 2025 - present / Neuralmagic, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from importlib.metadata import PackageNotFoundError, version + +import pytest + +try: + version("pytest-cov") + _pytest_cov_installed = True +except PackageNotFoundError: + _pytest_cov_installed = False + + +def test_plugin_loaded(pytester: pytest.Pytester): + """Verify the plugin is loaded by pytest""" + + plugin_version = version("pytest-nm-releng") + + pytester.makepyfile(""" + def test_pass(): + pass + """) + + result = pytester.runpytest() + result.stdout.fnmatch_lines([f"plugins:*nm-releng-{plugin_version}*"]) + + +def test_plugin_adds_junit_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected junit args""" + + monkeypatch.setenv("NMRE_JUNIT_BASE", "results") + + cf = pytester.parseconfigure() + actual = cf.getoption("--junit-xml", None) + assert actual is not None + assert actual.startswith("results") + + +@pytest.mark.skipif(not _pytest_cov_installed, reason="pytest-cov is required") +def test_plugin_adds_coverage_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected coverage args""" + + monkeypatch.setenv("NMRE_COV_NAME", "vllm") + + cf = pytester.parseconfigure() + + assert "vllm" in cf.getoption("--cov", None) + assert cf.getoption("--cov-append") is True + + +@pytest.mark.skipif(not _pytest_cov_installed, reason="pytest-cov is required") +def test_plugin_adds_all_args( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +): + """Verify the plugin adds the expected coverage args""" + + monkeypatch.setenv("NMRE_JUNIT_BASE", "results") + monkeypatch.setenv("NMRE_COV_NAME", "vllm") + + cf = pytester.parseconfigure() + + assert cf.getoption("--junit-xml", None).startswith("results") + assert "vllm" in cf.getoption("--cov", None) + assert cf.getoption("--cov-append") is True diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..56c8c69 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,10 @@ +from typing import Union + +import pytest + + +def setenv(monkeypatch: pytest.MonkeyPatch, name: str, value: Union[str, None]) -> None: + if value is None: + monkeypatch.delenv(name, raising=False) + else: + monkeypatch.setenv(name, value) diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..09f75ba --- /dev/null +++ b/tox.ini @@ -0,0 +1,39 @@ +# For more information about tox, see https://tox.readthedocs.io/en/latest/ +[tox] +envlist = py39,py310,py311,py312 + +[testenv] +deps = + pytest >= 8 +commands = + pytest {posargs:tests} + +[testenv:style] +skip_install = true +deps = + ruff ~= 0.8.5 + mdformat ~= 0.7.21 + mdformat-footnote ~= 0.1.1 + mdformat-frontmatter ~= 2.0.8 + mdformat-gfm ~= 0.4.1 + yamlfix ~= 1.16 +commands = + ruff check src tests + ruff format --check --diff src tests + mdformat --check README.md + yamlfix --check --config-file .yamlfix.toml .github + +[testenv:format] +skip_install = true +deps = + ruff ~= 0.8.5 + mdformat ~= 0.7.21 + mdformat-footnote ~= 0.1.1 + mdformat-frontmatter ~= 2.0.8 + mdformat-gfm ~= 0.4.1 + yamlfix ~= 1.16 +commands = + ruff check --fix src tests + ruff format src tests + mdformat README.md + yamlfix --config-file .yamlfix.toml .github