From 6625140362d24f81a14dbfea3d4e36afaa21a8e7 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Sat, 6 Apr 2024 12:37:27 +0000 Subject: [PATCH 01/23] bump dev 0.0.9_dev --- .pre-commit-config.yaml | 2 +- src/nbmetaclean/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c686b53..ac9fcfc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/ayasyrev/nbmetaclean - rev: 0.0.7 + rev: 0.0.8 hooks: - id: nbclean name: nbclean diff --git a/src/nbmetaclean/version.py b/src/nbmetaclean/version.py index 0e12f90..eb14da7 100644 --- a/src/nbmetaclean/version.py +++ b/src/nbmetaclean/version.py @@ -1 +1 @@ -__version__ = "0.0.8" # pragma: no cover +__version__ = "0.0.9_dev" # pragma: no cover From e8a7fd929529067169cd973ae721102cd6db387b Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Wed, 10 Jul 2024 08:25:53 +0000 Subject: [PATCH 02/23] reguirements rearrange --- requirements_dev.txt | 2 ++ requirements_test_extra.txt | 2 -- setup.py | 30 ++++++++++++++++++------------ 3 files changed, 20 insertions(+), 14 deletions(-) delete mode 100644 requirements_test_extra.txt diff --git a/requirements_dev.txt b/requirements_dev.txt index 8778d84..d400fe5 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,7 +1,9 @@ black black[jupyter] +coverage[toml] flake8 isort mypy +nox pre-commit ruff diff --git a/requirements_test_extra.txt b/requirements_test_extra.txt deleted file mode 100644 index 16ac163..0000000 --- a/requirements_test_extra.txt +++ /dev/null @@ -1,2 +0,0 @@ -coverage[toml] -nox diff --git a/setup.py b/setup.py index f7abcb7..8c11aa0 100644 --- a/setup.py +++ b/setup.py @@ -1,24 +1,30 @@ from setuptools import setup + REQUIREMENTS_FILENAME = "requirements.txt" REQUIREMENTS_TEST_FILENAME = "requirements_test.txt" +REQUIREMENTS_DEV_FILENAME = "requirements_dev.txt" + + +def load_requirements(filename: str) -> list[str]: + """Load requirements from file""" + try: + with open(filename, encoding="utf-8") as fh: + return fh.read().splitlines() + except FileNotFoundError: + return [] -# Requirements -try: - with open(REQUIREMENTS_FILENAME, encoding="utf-8") as fh: - REQUIRED = fh.read().split("\n") -except FileNotFoundError: - REQUIRED = [] +REQUIRED = load_requirements(REQUIREMENTS_FILENAME) +TEST_REQUIRED = load_requirements(REQUIREMENTS_TEST_FILENAME) +DEV_REQUIRED = load_requirements(REQUIREMENTS_DEV_FILENAME) -try: - with open(REQUIREMENTS_TEST_FILENAME, encoding="utf-8") as fh: - TEST_REQUIRED = fh.read().split("\n") -except FileNotFoundError: - TEST_REQUIRED = [] # What packages are optional? -EXTRAS = {"test": TEST_REQUIRED} +EXTRAS = { + "test": TEST_REQUIRED, + "dev": DEV_REQUIRED + TEST_REQUIRED, +} setup( From bda952788db09019c3d8cdd36a5df99a60e33bd6 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Wed, 10 Jul 2024 09:30:14 +0000 Subject: [PATCH 03/23] pre-commit update --- .pre-commit-config.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ac9fcfc..a17a98f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,7 +7,7 @@ repos: entry: nbclean - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-added-large-files - id: check-ast @@ -34,7 +34,7 @@ repos: - id: requirements-txt-fixer - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.3.5 + rev: v0.5.1 hooks: # Run the linter. - id: ruff @@ -47,5 +47,4 @@ repos: - id: python-check-mock-methods - id: python-use-type-annotations - id: python-check-blanket-noqa - - id: python-use-type-annotations - id: text-unicode-replacement-char From f5ffb64a8d16dd7b1c44aa6d7295d5ebb607a536 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Tue, 16 Jul 2024 08:51:53 +0000 Subject: [PATCH 04/23] nox uv + ruff --- noxfile.py | 5 ++--- noxfile_conda.py | 4 ++-- noxfile_conda_lint.py | 6 +++--- noxfile_lint.py | 8 ++++---- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/noxfile.py b/noxfile.py index 770d95b..68b78e7 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,9 +1,8 @@ import nox -@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"], venv_backend="uv") def tests(session: nox.Session) -> None: args = session.posargs or ["--cov"] - session.install("-r", "requirements_test.txt") - session.install(".") + session.install(".[test]") session.run("pytest", *args) diff --git a/noxfile_conda.py b/noxfile_conda.py index 7ba7b37..ef4a184 100644 --- a/noxfile_conda.py +++ b/noxfile_conda.py @@ -4,6 +4,6 @@ @nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"], venv_backend="mamba") def conda_tests(session: nox.Session) -> None: args = session.posargs or ["--cov"] - session.conda_install("--file", "requirements_test.txt") - session.install(".") + session.conda_install("uv") + session.install(".[test]") session.run("pytest", *args) diff --git a/noxfile_conda_lint.py b/noxfile_conda_lint.py index f4052aa..88ab393 100644 --- a/noxfile_conda_lint.py +++ b/noxfile_conda_lint.py @@ -1,10 +1,10 @@ import nox -locations = "src/nbmetaclean", "tests", "noxfile.py" +locations = "." @nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"], venv_backend="mamba") def conda_lint(session: nox.Session) -> None: args = session.posargs or locations - session.conda_install("flake8") - session.run("flake8", *args) + session.conda_install("ruff") + session.run("ruff", "check", *args) diff --git a/noxfile_lint.py b/noxfile_lint.py index 84af18a..d2429a7 100644 --- a/noxfile_lint.py +++ b/noxfile_lint.py @@ -1,10 +1,10 @@ import nox -locations = "src/nbmetaclean", "tests", "noxfile.py" +locations = "." -@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"]) +@nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"], venv_backend="uv") def lint(session: nox.Session) -> None: args = session.posargs or locations - session.install("flake8") - session.run("flake8", *args) + session.install("ruff") + session.run("ruff", "check", *args) From 67f0207bf3aec4d529df1d9b901fd051cbbadef2 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Tue, 16 Jul 2024 09:13:36 +0000 Subject: [PATCH 05/23] fix settings, nox files --- .flake8 | 3 --- .pre-commit-config.yaml | 2 +- noxfile.py | 2 +- noxfile_conda.py | 2 +- pyproject.toml | 2 +- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/.flake8 b/.flake8 index fe13e52..619d0e3 100644 --- a/.flake8 +++ b/.flake8 @@ -6,6 +6,3 @@ extend-ignore = W503 disable-noqa = True application-import-names = nbmetaclean, tests import-order-style = google -per-file-ignores = - # imported but unused - __init__.py: F401 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a17a98f..76f3f7a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - id: requirements-txt-fixer - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.5.1 + rev: v0.5.2 hooks: # Run the linter. - id: ruff diff --git a/noxfile.py b/noxfile.py index 68b78e7..5819a1e 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,5 +4,5 @@ @nox.session(python=["3.8", "3.9", "3.10", "3.11", "3.12"], venv_backend="uv") def tests(session: nox.Session) -> None: args = session.posargs or ["--cov"] - session.install(".[test]") + session.install("-e .[test]") session.run("pytest", *args) diff --git a/noxfile_conda.py b/noxfile_conda.py index ef4a184..305308c 100644 --- a/noxfile_conda.py +++ b/noxfile_conda.py @@ -5,5 +5,5 @@ def conda_tests(session: nox.Session) -> None: args = session.posargs or ["--cov"] session.conda_install("uv") - session.install(".[test]") + session.install("uv", "pip", "install", "-e", ".[test]") session.run("pytest", *args) diff --git a/pyproject.toml b/pyproject.toml index a40cc44..aef05e2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [tool.coverage.paths] -source = ["src", "*/site-packages"] +source = ["src"] [tool.coverage.run] branch = true From 89dad8b225d095da045e41fd4e860b7a9f9fc70a Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Tue, 16 Jul 2024 09:53:19 +0000 Subject: [PATCH 06/23] gh ci --- .github/workflows/lint.yml | 17 +++++++++++++++++ .github/workflows/tests.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..477534a --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,17 @@ +name: Lint +on: + push: + branches: + - dev + - main +jobs: + tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@main + - uses: actions/setup-python@main + with: + python-version: "3.11" + architecture: x64 + - run: pip install ruff + - run: ruff check . diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..31b3345 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,27 @@ +name: Tests +on: + push: + branches: + - dev + - main +jobs: + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.8", "3.9", "3.10", "3.11", "3.12"] + steps: + - name: Checkout + uses: actions/checkout@main + - name: Setup Python ${{ matrix.python }} + uses: actions/setup-python@main + with: + python-version: ${{ matrix.python }} + architecture: x64 + + - name: Install + run: | + pip install uv + uv pip install --system .[test] + - name: Tests + run: pytest From 8ea5f3ae3790f7141dbb289f5e79dc0d8cb78042 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Fri, 26 Jul 2024 08:20:28 +0000 Subject: [PATCH 07/23] add codecov --- .github/workflows/tests.yml | 8 ++++++++ README.md | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 31b3345..4816688 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,5 +23,13 @@ jobs: run: | pip install uv uv pip install --system .[test] + - name: Tests run: pytest + + - name: CodeCov + if: ${{ matrix.python == '3.11' }} + uses: codecov/codecov-action@main + with: + token: ${{ secrets.CODECOV_TOKEN }} + slug: ayasyrev/nbmetaclean diff --git a/README.md b/README.md index 0b29f63..5dd4025 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ Pre-commit hook to clean Jupyter Notebooks metadata, execution_count and optionally output. +[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/benchmark-utils)](https://pypi.org/project/nbmetaclean/) +[![PyPI Status](https://badge.fury.io/py/nbmetaclean.svg)](https://badge.fury.io/py/nbmetaclean) +[![Tests](https://github.com/ayasyrev/nbmetaclean/workflows/Tests/badge.svg)](https://github.com/ayasyrev/nbmetaclean/actions?workflow=Tests) [![Codecov](https://codecov.io/gh/ayasyrev/nbmetaclean/branch/main/graph/badge.svg)](https://codecov.io/gh/ayasyrev/nbmetaclean) + Pure Python, no dependencies. Can be used as a pre-commit hook or as a command line tool. From 1987af01bd61fe87c383a89d0ff9b0964d96812a Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Fri, 26 Jul 2024 08:22:32 +0000 Subject: [PATCH 08/23] fix coverage gh ci --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4816688..e4220a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -22,10 +22,10 @@ jobs: - name: Install run: | pip install uv - uv pip install --system .[test] + uv pip install --system .[test] "coverage[toml]" - name: Tests - run: pytest + run: pytest --cov - name: CodeCov if: ${{ matrix.python == '3.11' }} From e4c78db9f13ccd4181b11c6c48dd7d0a0d17a3b3 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 29 Jul 2024 13:42:35 +0000 Subject: [PATCH 09/23] add dry_run to clean_nb_file --- src/nbmetaclean/clean.py | 13 ++++++------- tests/test_clean.py | 19 ++++++++++++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index cf69e55..559d35f 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -33,8 +33,9 @@ class CleanConfig: Preserve mask for notebook metadata. Defaults to None. cell_metadata_preserve_mask (Optional[tuple[str, ...]], optional): Preserve mask for cell metadata. Defaults to None. - mask_merge (bool, optional): Merge masks. Add new mask to default. + mask_merge (bool): Merge masks. Add new mask to default. If False - use new mask. Defaults to True. + dry_run (bool): perform a trial run, don't write results. Defaults to False. """ clear_nb_metadata: bool = True @@ -46,6 +47,7 @@ class CleanConfig: nb_metadata_preserve_mask: Optional[tuple[TupleStr, ...]] = None cell_metadata_preserve_mask: Optional[tuple[TupleStr, ...]] = None mask_merge: bool = True + dry_run: bool = False def filter_meta_mask( @@ -175,12 +177,7 @@ def clean_nb_file( Args: path (Union[str, PosixPath]): Notebook filename or list of names. - clear_nb_metadata (bool): Clear notebook metadata. Defaults to True. - clear_cell_metadata (bool): Clear cell metadata. Defaults to False. - clear_outputs (bool): Clear outputs. Defaults to False. - preserve_timestamp (bool): Preserve timestamp. Defaults to True. - clear_execution_count (bool, optional): Clean execution count. Defaults to True. - silent (bool): Silent mode. Defaults to False. + cfg (CleanConfig, optional): Config for job, if None, used default settings. Default is None. Returns: tuple[List[Path], List[TuplePath]]: List of cleaned notebooks, list of notebooks with errors. @@ -204,6 +201,8 @@ def clean_nb_file( ) if result: cleaned.append(filename) + if cfg.dry_run: + continue if cfg.preserve_timestamp: stat = filename.stat() write_nb(nb, filename) diff --git a/tests/test_clean.py b/tests/test_clean.py index 813f3e1..ab30826 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -255,7 +255,24 @@ def test_clean_nb_file(tmp_path: Path, capsys: CaptureFixture[str]): nb_clean = read_nb(path / "test_nb_2_clean.ipynb") # prepare temp test notebook - test_nb_path = write_nb(read_nb(path / nb_name), tmp_path / nb_name) + nb_source = read_nb(path / nb_name) + test_nb_path = write_nb(nb_source, tmp_path / nb_name) + + # clean meta, leave execution_count + # first lets dry run + cleaned, errors = clean_nb_file( + test_nb_path, + cfg=CleanConfig( + clear_execution_count=False, + dry_run=True, + ), + ) + assert len(cleaned) == 1 + assert len(errors) == 0 + nb = read_nb(cleaned[0]) + assert nb["metadata"] == nb_source["metadata"] + assert nb["cells"][1]["execution_count"] == 1 + assert nb["cells"][1]["outputs"][0]["execution_count"] == 1 # clean meta, leave execution_count cleaned, errors = clean_nb_file( From a01b03ec7e60224ea97e92085bae045949b371c2 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 29 Jul 2024 14:10:21 +0000 Subject: [PATCH 10/23] add dry run to app --- src/nbmetaclean/app.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index 68e969e..5131a7d 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -61,6 +61,11 @@ action="store_true", help="Clean hidden notebooks.", ) +parser.add_argument( + "--dry_run", + action="store_true", + help="perform a trial run, don't write results", +) def process_mask(mask: Union[list[str], None]) -> Union[tuple[TupleStr, ...], None]: From de1c0dc48e7f13bf8717881c223adb64bebc933b Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 29 Jul 2024 14:17:36 +0000 Subject: [PATCH 11/23] rename typing to types --- src/nbmetaclean/clean.py | 2 +- src/nbmetaclean/helpers.py | 2 +- src/nbmetaclean/{typing.py => types.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/nbmetaclean/{typing.py => types.py} (100%) diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index 559d35f..a0e0272 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -8,7 +8,7 @@ from nbmetaclean.helpers import read_nb, write_nb -from .typing import Cell, CodeCell, Metadata, Nb, Output +from .types import Cell, CodeCell, Metadata, Nb, Output TupleStr = Tuple[str, ...] diff --git a/src/nbmetaclean/helpers.py b/src/nbmetaclean/helpers.py index b3f1c3a..90352f2 100644 --- a/src/nbmetaclean/helpers.py +++ b/src/nbmetaclean/helpers.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import Optional -from .typing import Nb, PathOrStr +from .types import Nb, PathOrStr def read_nb(path: PathOrStr) -> Nb: diff --git a/src/nbmetaclean/typing.py b/src/nbmetaclean/types.py similarity index 100% rename from src/nbmetaclean/typing.py rename to src/nbmetaclean/types.py From c241df7f7884718424d52d85662668c626d31c0d Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Tue, 30 Jul 2024 18:04:58 +0000 Subject: [PATCH 12/23] fix app - dry run --- src/nbmetaclean/app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index 5131a7d..15c291c 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -98,6 +98,7 @@ def app() -> None: nb_metadata_preserve_mask=process_mask(cfg.nb_metadata_preserve_mask), cell_metadata_preserve_mask=process_mask(cfg.cell_metadata_preserve_mask), mask_merge=not cfg.dont_merge_masks, + dry_run=cfg.dry_run, ) cleaned, errors = clean_nb_file( nb_files, From b67d0e82b09bc03b4e2f48fc33ef1c170b540b51 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Thu, 8 Aug 2024 12:00:34 +0300 Subject: [PATCH 13/23] add -verbose argument, shorts flag for dry-run --- src/nbmetaclean/app.py | 10 ++++++++++ src/nbmetaclean/clean.py | 2 ++ 2 files changed, 12 insertions(+) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index 15c291c..2310fee 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -62,10 +62,17 @@ help="Clean hidden notebooks.", ) parser.add_argument( + "-D", "--dry_run", action="store_true", help="perform a trial run, don't write results", ) +parser.add_argument( + "-V", + "--verbose", + action="store_true", + help="Print more info - cleaned notebooks.", +) def process_mask(mask: Union[list[str], None]) -> Union[tuple[TupleStr, ...], None]: @@ -106,6 +113,9 @@ def app() -> None: ) if not cfg.silent: print(f"cleaned nbs: {len(cleaned)}") + if cfg.verbose: + for nb in cleaned: + print("- ", nb) if errors: print(f"with errors: {len(errors)}") for nb, exc in errors: diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index a0e0272..2bfccf2 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -36,6 +36,7 @@ class CleanConfig: mask_merge (bool): Merge masks. Add new mask to default. If False - use new mask. Defaults to True. dry_run (bool): perform a trial run, don't write results. Defaults to False. + verbose (bool): Print more info - cleaned notebooks. Defaults to False. """ clear_nb_metadata: bool = True @@ -48,6 +49,7 @@ class CleanConfig: cell_metadata_preserve_mask: Optional[tuple[TupleStr, ...]] = None mask_merge: bool = True dry_run: bool = False + verbose: bool = False def filter_meta_mask( From 4cdee5779eb90ce730f06f9de7400bffa4de2a61 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Thu, 8 Aug 2024 13:07:41 +0300 Subject: [PATCH 14/23] cfg to one line --- src/nbmetaclean/clean.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index 2bfccf2..7a066e4 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -184,8 +184,7 @@ def clean_nb_file( Returns: tuple[List[Path], List[TuplePath]]: List of cleaned notebooks, list of notebooks with errors. """ - if cfg is None: - cfg = CleanConfig() + cfg = cfg or CleanConfig() if not isinstance(path, list): path = [path] cleaned: list[Path] = [] From 6fc083bbdeb2e476a6eb21831dab6fd491eb8606 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Thu, 8 Aug 2024 16:54:08 +0300 Subject: [PATCH 15/23] remove verbose, remove print from `clean_nb_file` --- src/nbmetaclean/app.py | 11 ++--------- src/nbmetaclean/clean.py | 8 ++------ tests/test_clean.py | 14 -------------- 3 files changed, 4 insertions(+), 29 deletions(-) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index 2310fee..3d1dd8a 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -67,12 +67,6 @@ action="store_true", help="perform a trial run, don't write results", ) -parser.add_argument( - "-V", - "--verbose", - action="store_true", - help="Print more info - cleaned notebooks.", -) def process_mask(mask: Union[list[str], None]) -> Union[tuple[TupleStr, ...], None]: @@ -113,9 +107,8 @@ def app() -> None: ) if not cfg.silent: print(f"cleaned nbs: {len(cleaned)}") - if cfg.verbose: - for nb in cleaned: - print("- ", nb) + for nb in cleaned: + print("- ", nb) if errors: print(f"with errors: {len(errors)}") for nb, exc in errors: diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index 7a066e4..b889021 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -36,7 +36,6 @@ class CleanConfig: mask_merge (bool): Merge masks. Add new mask to default. If False - use new mask. Defaults to True. dry_run (bool): perform a trial run, don't write results. Defaults to False. - verbose (bool): Print more info - cleaned notebooks. Defaults to False. """ clear_nb_metadata: bool = True @@ -49,7 +48,6 @@ class CleanConfig: cell_metadata_preserve_mask: Optional[tuple[TupleStr, ...]] = None mask_merge: bool = True dry_run: bool = False - verbose: bool = False def filter_meta_mask( @@ -189,8 +187,7 @@ def clean_nb_file( path = [path] cleaned: list[Path] = [] errors: list[tuple[Path, Exception]] = [] - to_clean = len(path) - for num, filename in enumerate(path): + for filename in path: try: nb = read_nb(filename) except Exception as ex: @@ -209,6 +206,5 @@ def clean_nb_file( write_nb(nb, filename) if cfg.preserve_timestamp: os.utime(filename, (stat.st_atime, stat.st_mtime)) - if not cfg.silent: - print(f"done {num + 1} of {to_clean}: {filename}") + return cleaned, errors diff --git a/tests/test_clean.py b/tests/test_clean.py index ab30826..465df66 100644 --- a/tests/test_clean.py +++ b/tests/test_clean.py @@ -289,10 +289,6 @@ def test_clean_nb_file(tmp_path: Path, capsys: CaptureFixture[str]): # clean meta, execution_count # path as list cleaned, errors = clean_nb_file([test_nb_path], CleanConfig()) - captured = capsys.readouterr() - out = captured.out - assert out.startswith("done") - assert "test_clean_nb_file0/.test_nb_2_meta.ipynb" in out assert len(cleaned) == 1 nb = read_nb(cleaned[0]) assert nb == nb_clean @@ -300,17 +296,7 @@ def test_clean_nb_file(tmp_path: Path, capsys: CaptureFixture[str]): # try clean cleaned cleaned, errors = clean_nb_file(test_nb_path, CleanConfig()) assert len(cleaned) == 0 - captured = capsys.readouterr() - out = captured.out - assert not out.strip() - - # silent - test_nb_path = write_nb(read_nb(path / nb_name), tmp_path / nb_name) - cleaned, errors = clean_nb_file(test_nb_path, CleanConfig(silent=True)) - assert len(cleaned) == 1 assert len(errors) == 0 - captured = capsys.readouterr() - assert not captured.out.strip() def test_clean_nb_file_errors(capsys: CaptureFixture[str], tmp_path: Path): From 2cbb4acf2505150ccce4a268c3af50bb29f0f0d2 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Fri, 9 Aug 2024 11:11:31 +0300 Subject: [PATCH 16/23] set timestamp at write_nb --- src/nbmetaclean/clean.py | 8 ++++---- src/nbmetaclean/helpers.py | 7 ++++++- tests/test_read_write.py | 7 +++++++ 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index b889021..4675d2f 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -1,7 +1,6 @@ from __future__ import annotations import copy -import os from dataclasses import dataclass from pathlib import Path from typing import Optional, Tuple, Union @@ -203,8 +202,9 @@ def clean_nb_file( continue if cfg.preserve_timestamp: stat = filename.stat() - write_nb(nb, filename) - if cfg.preserve_timestamp: - os.utime(filename, (stat.st_atime, stat.st_mtime)) + timestamp = (stat.st_atime, stat.st_mtime) + else: + timestamp = None + write_nb(nb, filename, timestamp) return cleaned, errors diff --git a/src/nbmetaclean/helpers.py b/src/nbmetaclean/helpers.py index 90352f2..ff0d181 100644 --- a/src/nbmetaclean/helpers.py +++ b/src/nbmetaclean/helpers.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Optional @@ -22,12 +23,14 @@ def read_nb(path: PathOrStr) -> Nb: def write_nb( nb: Nb, path: PathOrStr, + timestamp: Optional[tuple[float, float]] = None, ) -> Path: - """Write notebook to file + """Write notebook to file, optionally set timestamp. Args: nb (Notebook): Notebook to write path (Union[str, PosixPath]): filename to write + timestamp (Optional[tuple[float, float]]): timestamp to set, (st_atime, st_mtime) defaults to None Returns: Path: Filename of written notebook. """ @@ -45,6 +48,8 @@ def write_nb( ) + "\n", ) + if timestamp is not None: + os.utime(filename, timestamp) return filename diff --git a/tests/test_read_write.py b/tests/test_read_write.py index 0a7e301..b2b94b5 100644 --- a/tests/test_read_write.py +++ b/tests/test_read_write.py @@ -41,3 +41,10 @@ def test_write_nb(tmp_path: Path): # write with name w/o suffix result = write_nb(nb, tmp_path / "test_nb_1") assert result == tmp_path / "test_nb_1.ipynb" + + # write with stat + stat = file.stat() + timestamp = (stat.st_atime, stat.st_mtime) + result = write_nb(nb, tmp_path / "test_nb_1", timestamp=timestamp) + res_stat = result.stat() + assert timestamp == (res_stat.st_atime, res_stat.st_mtime) From 08e6351083c8fa8d277acb962c0022bd623cd86a Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 12 Aug 2024 15:37:09 +0300 Subject: [PATCH 17/23] update pre-commit --- .pre-commit-config.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 76f3f7a..927b2fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,8 +3,6 @@ repos: rev: 0.0.8 hooks: - id: nbclean - name: nbclean - entry: nbclean - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 @@ -34,7 +32,7 @@ repos: - id: requirements-txt-fixer - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.5.2 + rev: v0.5.7 hooks: # Run the linter. - id: ruff From f89d03f2e645513020e811ff46dd46ff2b3e571d Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 12 Aug 2024 18:34:53 +0300 Subject: [PATCH 18/23] add ruff settings and make pre-commit nbclean local --- .pre-commit-config.yaml | 10 +++++++--- pyproject.toml | 13 +++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 927b2fb..b176598 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,8 +1,12 @@ repos: -- repo: https://github.com/ayasyrev/nbmetaclean - rev: 0.0.8 +- repo: local hooks: - - id: nbclean + # local version for testing + - id: nbclean + name: nbclean local + entry: nbclean + language: system + files: \.ipynb - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.6.0 diff --git a/pyproject.toml b/pyproject.toml index aef05e2..b4abb8e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,3 +7,16 @@ source = ["nbmetaclean"] [tool.coverage.report] show_missing = true + +[tool.ruff] +extend-include = ["*.ipynb"] +indent-width = 4 + +[tool.ruff.lint] +explicit-preview-rules = true + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" +skip-magic-trailing-comma = false +line-ending = "auto" From d3da28bca3185e1b650334b6b6a9e4f3306e4829 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Mon, 12 Aug 2024 19:09:22 +0300 Subject: [PATCH 19/23] Add verbose mode to CLI, app function, and CleanConfig class. --- src/nbmetaclean/app.py | 29 ++++++++++++++++++----------- src/nbmetaclean/clean.py | 2 ++ 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index 3d1dd8a..f74a096 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -67,6 +67,12 @@ action="store_true", help="perform a trial run, don't write results", ) +parser.add_argument( + "-V", + "--verbose", + action="store_true", + help="Verbose mode. Print extra information.", +) def process_mask(mask: Union[list[str], None]) -> Union[tuple[TupleStr, ...], None]: @@ -78,17 +84,6 @@ def process_mask(mask: Union[list[str], None]) -> Union[tuple[TupleStr, ...], No def app() -> None: """Clean metadata and execution_count from Jupyter notebook.""" cfg = parser.parse_args() - path_list = cfg.path if isinstance(cfg.path, list) else [cfg.path] - nb_files: list[Path] = [] - if not cfg.silent: - print(f"Path: {', '.join(cfg.path)}, preserve timestamp: {not cfg.not_pt}") - for path in path_list: - try: - nb_files.extend(get_nb_names(path, hidden=cfg.clean_hidden_nbs)) - except FileNotFoundError: - print(f"{path} not exists!") - if not cfg.silent: - print(f"notebooks to check: {len(nb_files)} ") clean_config = CleanConfig( clear_nb_metadata=not cfg.dont_clear_nb_metadata, clear_cell_metadata=cfg.clear_cell_metadata, @@ -100,7 +95,19 @@ def app() -> None: cell_metadata_preserve_mask=process_mask(cfg.cell_metadata_preserve_mask), mask_merge=not cfg.dont_merge_masks, dry_run=cfg.dry_run, + verbose=cfg.verbose if not cfg.silent else False, ) + path_list = cfg.path if isinstance(cfg.path, list) else [cfg.path] + nb_files: list[Path] = [] + if clean_config.verbose: + print(f"Path: {', '.join(cfg.path)}, preserve timestamp: {not cfg.not_pt}") + for path in path_list: + try: + nb_files.extend(get_nb_names(path, hidden=cfg.clean_hidden_nbs)) + except FileNotFoundError: + print(f"{path} not exists!") + if clean_config.verbose: + print(f"notebooks to check: {len(nb_files)} ") cleaned, errors = clean_nb_file( nb_files, clean_config, diff --git a/src/nbmetaclean/clean.py b/src/nbmetaclean/clean.py index 4675d2f..f4efa26 100644 --- a/src/nbmetaclean/clean.py +++ b/src/nbmetaclean/clean.py @@ -35,6 +35,7 @@ class CleanConfig: mask_merge (bool): Merge masks. Add new mask to default. If False - use new mask. Defaults to True. dry_run (bool): perform a trial run, don't write results. Defaults to False. + verbose (bool): Verbose mode. Print extra information. Defaults to False. """ clear_nb_metadata: bool = True @@ -47,6 +48,7 @@ class CleanConfig: cell_metadata_preserve_mask: Optional[tuple[TupleStr, ...]] = None mask_merge: bool = True dry_run: bool = False + verbose: bool = False def filter_meta_mask( From ce955c0005bfbf76016e7f77f2cf3ccb8b0226af Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Tue, 13 Aug 2024 13:50:33 +0300 Subject: [PATCH 20/23] clean app output --- src/nbmetaclean/app.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index f74a096..33eaacf 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -113,9 +113,13 @@ def app() -> None: clean_config, ) if not cfg.silent: - print(f"cleaned nbs: {len(cleaned)}") - for nb in cleaned: - print("- ", nb) + if cleaned: + if len(cleaned) == 1: + print(f"cleaned: {cleaned[0]}") + else: + print(f"cleaned: {len(cleaned)} notebooks") + for nb in cleaned: + print("- ", nb) if errors: print(f"with errors: {len(errors)}") for nb, exc in errors: From 024e6bd44eaf84ec2911319a495ff4d49af834e1 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Tue, 13 Aug 2024 14:20:59 +0300 Subject: [PATCH 21/23] Add print_result function and refactor app function to use it for output. --- src/nbmetaclean/app.py | 48 +++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index 33eaacf..d3c26ca 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -81,6 +81,31 @@ def process_mask(mask: Union[list[str], None]) -> Union[tuple[TupleStr, ...], No return tuple(tuple(item.split(".")) for item in mask) +def print_result( + cleaned: list[Path], + errors: list[tuple[Path, Exception]], + clean_config: CleanConfig, + path: list[Path], + num_nbs: int, +) -> None: + if clean_config.verbose: + print( + f"Path: {', '.join(path)}, preserve timestamp: {clean_config.preserve_timestamp}" + ) + print(f"checked: {num_nbs} notebooks") + if cleaned: + if len(cleaned) == 1: + print(f"cleaned: {cleaned[0]}") + else: + print(f"cleaned: {len(cleaned)} notebooks") + for nb in cleaned: + print("- ", nb) + if errors: + print(f"with errors: {len(errors)}") + for nb, exc in errors: + print(f"{nb}: {exc}") + + def app() -> None: """Clean metadata and execution_count from Jupyter notebook.""" cfg = parser.parse_args() @@ -99,31 +124,20 @@ def app() -> None: ) path_list = cfg.path if isinstance(cfg.path, list) else [cfg.path] nb_files: list[Path] = [] - if clean_config.verbose: - print(f"Path: {', '.join(cfg.path)}, preserve timestamp: {not cfg.not_pt}") for path in path_list: - try: + path = Path(path) + if path.exists(): nb_files.extend(get_nb_names(path, hidden=cfg.clean_hidden_nbs)) - except FileNotFoundError: + else: print(f"{path} not exists!") - if clean_config.verbose: - print(f"notebooks to check: {len(nb_files)} ") + cleaned, errors = clean_nb_file( nb_files, clean_config, ) + if not cfg.silent: - if cleaned: - if len(cleaned) == 1: - print(f"cleaned: {cleaned[0]}") - else: - print(f"cleaned: {len(cleaned)} notebooks") - for nb in cleaned: - print("- ", nb) - if errors: - print(f"with errors: {len(errors)}") - for nb, exc in errors: - print(f"{nb}: {exc}") + print_result(cleaned, errors, clean_config, path_list, len(nb_files)) if __name__ == "__main__": From 6905efe8a8132e9ff80a6b28164895df7112ead2 Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Thu, 15 Aug 2024 11:48:05 +0300 Subject: [PATCH 22/23] app argument `not_ec` - preserve execution count --- src/nbmetaclean/app.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/nbmetaclean/app.py b/src/nbmetaclean/app.py index d3c26ca..fba439d 100644 --- a/src/nbmetaclean/app.py +++ b/src/nbmetaclean/app.py @@ -21,6 +21,11 @@ action="store_true", help="Silent mode.", ) +parser.add_argument( + "--not_ec", + action="store_false", + help="Do not clear execution_count.", +) parser.add_argument( "--not-pt", action="store_true", @@ -112,7 +117,7 @@ def app() -> None: clean_config = CleanConfig( clear_nb_metadata=not cfg.dont_clear_nb_metadata, clear_cell_metadata=cfg.clear_cell_metadata, - clear_execution_count=True, + clear_execution_count=cfg.not_ec, clear_outputs=cfg.clear_outputs, preserve_timestamp=not cfg.not_pt, silent=cfg.silent, From ce6c7bcae36638580d86c95c6c21f6f518281fff Mon Sep 17 00:00:00 2001 From: ayasyrev Date: Thu, 15 Aug 2024 13:26:39 +0300 Subject: [PATCH 23/23] bump version 0.0.9 --- src/nbmetaclean/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nbmetaclean/version.py b/src/nbmetaclean/version.py index eb14da7..00ec2dc 100644 --- a/src/nbmetaclean/version.py +++ b/src/nbmetaclean/version.py @@ -1 +1 @@ -__version__ = "0.0.9_dev" # pragma: no cover +__version__ = "0.0.9"