From fb8cfdc130f8c1488a3984e4f5a4f9e589d3cdfe Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 18 Feb 2025 09:47:04 -0600 Subject: [PATCH 1/3] feat: allow keeping stuff when isolating data dir (#2515) --- src/ape/managers/config.py | 28 +++++++++++++++++++--- tests/functional/test_config.py | 42 +++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/ape/managers/config.py b/src/ape/managers/config.py index f96ad38ec6..451954e0c7 100644 --- a/src/ape/managers/config.py +++ b/src/ape/managers/config.py @@ -1,9 +1,10 @@ import os -from collections.abc import Iterator +import shutil +from collections.abc import Iterable, Iterator from contextlib import contextmanager from functools import cached_property from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Optional, Union from ape.api.config import ApeConfig from ape.managers.base import BaseManager @@ -113,11 +114,22 @@ def extract_config(cls, manifest: "PackageManifest", **overrides) -> ApeConfig: return ApeConfig.from_manifest(manifest, **overrides) @contextmanager - def isolate_data_folder(self) -> Iterator[Path]: + def isolate_data_folder( + self, keep: Optional[Union[Iterable[str], str]] = None + ) -> Iterator[Path]: """ Change Ape's DATA_FOLDER to point a temporary path, in a context, for testing purposes. Any data cached to disk will not persist. + + Args: + keep (Optional[Union[Iterable[str], str]]): Optionally, pass in + a key of subdirectory names to include in the new isolated + data folder. For example, pass ing ``"packages"`` to avoid + having to re-download dependencies in an isolated environment. + + Returns: + Iterator[Path]: The temporary data folder. """ original_data_folder = self.DATA_FOLDER if in_tempdir(original_data_folder): @@ -125,8 +137,18 @@ def isolate_data_folder(self) -> Iterator[Path]: yield original_data_folder else: + keep = [keep] if isinstance(keep, str) else keep or [] try: with create_tempdir() as temp_data_folder: + # Copy in items from "keep". + for item in keep: + path_to_keep = original_data_folder / item + if not path_to_keep.is_dir(): + continue + + dest_path = temp_data_folder / item + shutil.copytree(path_to_keep, dest_path) + self.DATA_FOLDER = temp_data_folder yield temp_data_folder diff --git a/tests/functional/test_config.py b/tests/functional/test_config.py index 2c31d22381..fcbb3c3d21 100644 --- a/tests/functional/test_config.py +++ b/tests/functional/test_config.py @@ -1,5 +1,6 @@ import os import re +import shutil from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Optional, Union @@ -689,3 +690,44 @@ def test_project_level_settings(project): assert project.config.my_string == "my_string" assert project.config.my_int == 123 assert project.config.my_bool is True + + +def test_isolate_data_folder(config): + original_data_folder = config.DATA_FOLDER + madeup_path = Path.home() / ".ape" / "__madeup__" + config.DATA_FOLDER = madeup_path + try: + with config.isolate_data_folder(): + assert config.DATA_FOLDER != original_data_folder + finally: + config.DATA_FOLDER = original_data_folder + + +def test_isolate_data_folder_already_isolated(config): + data_folder = config.DATA_FOLDER + with config.isolate_data_folder(): + # Already isolated, so there is no change. + assert config.DATA_FOLDER == data_folder + + +def test_isolate_data_folder_keep(config): + original_data_folder = config.DATA_FOLDER + madeup_path = Path.home() / ".ape" / "__core_aoe_test__" + madeup_path.mkdir(parents=True, exist_ok=True) + sub_dir = madeup_path / "subdir" + file = sub_dir / "file.txt" + sub_dir.mkdir(parents=True, exist_ok=True) + file.write_text("search for 'test_isolate_data_folder_keep' ape's tests.") + + config.DATA_FOLDER = madeup_path + try: + with config.isolate_data_folder(keep="subdir"): + assert config.DATA_FOLDER != original_data_folder + + expected_file = config.DATA_FOLDER / "subdir" / "file.txt" + assert expected_file.is_file() + + finally: + config.DATA_FOLDER = original_data_folder + if madeup_path.is_dir(): + shutil.rmtree(str(madeup_path), ignore_errors=True) From 0d61d251e910ee11ad4fb5b047442058719c66a9 Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 18 Feb 2025 09:59:50 -0600 Subject: [PATCH 2/3] feat: allow `project.chdir()` to work in a context (#2516) --- src/ape/managers/project.py | 44 ++++++++++++++++++++++++------ src/ape/utils/os.py | 46 ++++++++++++++++++++++++++++++++ tests/functional/test_project.py | 13 ++++++--- 3 files changed, 92 insertions(+), 11 deletions(-) diff --git a/src/ape/managers/project.py b/src/ape/managers/project.py index 05fede30ad..518be5a2c5 100644 --- a/src/ape/managers/project.py +++ b/src/ape/managers/project.py @@ -1,5 +1,4 @@ import json -import os import random import shutil from collections.abc import Callable, Iterable, Iterator @@ -32,6 +31,7 @@ ) from ape.utils.misc import SOURCE_EXCLUDE_PATTERNS, log_instead_of_fail from ape.utils.os import ( + ChangeDirectory, clean_path, create_tempdir, get_all_files_in_directory, @@ -2592,22 +2592,50 @@ def chdir(self, path: Optional[Path] = None): path (Path): The path of the new project. If not given, defaults to the project's path. """ - path = path or self.path - os.chdir(path) - if self.path == path: - return # Already setup. + return ChangeDirectory( + self.path, + path or self.path, + on_push=self._handle_path_changed, + on_pop=self._handle_path_restored, + ) + + def _handle_path_changed(self, path: Path) -> dict: + cache: dict = { + "__dict__": {}, + "_session_source_change_check": self._session_source_change_check, + "_config_override": self._config_override, + "_base_path": self._base_path, + "manifest_path": self.manifest_path, + "_manifest": self._manifest, + } # New path: clear cached properties. for attr in list(self.__dict__.keys()): if isinstance(getattr(type(self), attr, None), cached_property): - del self.__dict__[attr] + cache["__dict__"][attr] = self.__dict__.pop(attr) - # Re-initialize self._session_source_change_check = set() self._config_override = {} self._base_path = Path(path).resolve() - self.manifest_path = self._base_path / ".build" / "__local__.json" + + if self.manifest_path.name == "__local__.json": + self.manifest_path = self._base_path / ".build" / "__local__.json" + self._manifest = self.load_manifest() + return cache + + def _handle_path_restored(self, cache: dict) -> None: + self.__dict__ = {**(self.__dict__ or {}), **cache.get("__dict__", {})} + self._session_source_change_check = cache.get("_session_source_change_check", set()) + self._config_override = cache.get("_config_override", {}) + if base_path := self._base_path: + self._base_path = base_path + + if manifest_path := cache.get("manifest_path"): + self.manifest_path = manifest_path + + if manifest := cache.get("_manifest"): + self._manifest = manifest @contextmanager def within_project_path(self): diff --git a/src/ape/utils/os.py b/src/ape/utils/os.py index 58f6287130..12e0fe53b8 100644 --- a/src/ape/utils/os.py +++ b/src/ape/utils/os.py @@ -463,3 +463,49 @@ def within_directory(directory: Path): finally: if Path.cwd() != here: os.chdir(here) + + +class ChangeDirectory: + """ + A context-manager for changing a directory. Initializing it + will still change the directory, but you can optionally exit + out of it to restore back to the original directory. Additionally, + provides hooks to run when each of these events occur. + """ + + def __init__( + self, + original_path: Path, + new_path: Path, + chdir: Optional[Callable[[Path], None]] = None, + on_push: Optional[Callable[[Path], dict]] = None, + on_pop: Optional[Callable[[dict], None]] = None, + ): + self.original_path = original_path + self.new_path = new_path + self._on_push = on_push + self._on_pop = on_pop + self._chdir = chdir or os.chdir + + # Initiate the change now so you can still use this class + # on methods that are not intended to be used in a context. + if self.original_path != self.new_path: + self._chdir(new_path) + self._cache: dict = {} if self._on_push is None else self._on_push(new_path) + self._did_change = True + else: + self._cache = {} + self._did_change = False + + def __enter__(self): + return self.new_path + + def __exit__(self, *args): + if not self._did_change: + # Don't do anything. Nothing changed. + return + + # Handle the return to the original path. + self._chdir(self.original_path) + if self._on_pop: + self._on_pop(self._cache) diff --git a/tests/functional/test_project.py b/tests/functional/test_project.py index d4a1e0e057..7eb603652d 100644 --- a/tests/functional/test_project.py +++ b/tests/functional/test_project.py @@ -1065,9 +1065,16 @@ def test_chdir(project): project.chdir(new_path) assert project.path == new_path - # Undo. - project.chdir(original_path) - assert project.path == original_path + # Undo. + project.chdir(original_path) + assert project.path == original_path + + # Show you can also use it in a context. + with project.chdir(new_path): + assert project.path == new_path + + # It should have automatically undone. + assert project.path == original_path def test_within_project_path(): From d504d186caa25bc62f8d9f13dad0ce5c9a28e011 Mon Sep 17 00:00:00 2001 From: antazoey Date: Tue, 18 Feb 2025 11:41:26 -0600 Subject: [PATCH 3/3] feat: isolate changes context in packages cache (#2518) --- src/ape/managers/project.py | 19 +++++++++++++++++++ tests/functional/test_dependencies.py | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/src/ape/managers/project.py b/src/ape/managers/project.py index 518be5a2c5..fc6370801a 100644 --- a/src/ape/managers/project.py +++ b/src/ape/managers/project.py @@ -1021,6 +1021,25 @@ def remove(self, package_id: str, version: str): manifest_file = self.get_manifest_path(package_id, version) manifest_file.unlink(missing_ok=True) + @contextmanager + def isolate_changes(self): + """ + Allows you to make changes affecting the Ape packages cache in a context. + For example, temporarily install local "dev" packages for testing purposes. + """ + with create_tempdir() as tmpdir: + packages_cache = tmpdir / "packages" + packages_cache.parent.mkdir(parents=True, exist_ok=True) + if self.root.is_dir(): + shutil.copytree(self.root, packages_cache) + try: + yield + finally: + shutil.rmtree(self.root) + if packages_cache.is_dir(): + # Restore. + shutil.copytree(packages_cache, self.root) + def _version_to_options(version: str) -> tuple[str, ...]: if version.startswith("v"): diff --git a/tests/functional/test_dependencies.py b/tests/functional/test_dependencies.py index 58f7987ceb..f17a9b5745 100644 --- a/tests/functional/test_dependencies.py +++ b/tests/functional/test_dependencies.py @@ -412,6 +412,14 @@ def test_get_project_path_unneeded_v_prefix(self, cache, data_folder): actual = cache.get_project_path("project-test-2", "v1.0.0") assert actual == path + def test_isolate_cache_changes(self, cache): + dep = LocalDependency(name="isotestdep", local=Path("isotestdep"), version="v1.0.0") + with cache.isolate_changes(): + path = cache.cache_api(dep) + assert path.is_file() + + assert not path.is_file() + class TestLocalDependency: NAME = "testlocaldep"