Add test PyPI mirror and support in all

README.md

@@ -105,10 +105,12 @@ The `--bg` flag can be passed to run the index in the background.
 When running an index in the background, state will be stored in the `~/.packse` directory. The `PACKSE_STORAGE_PATH`
 environment variable or the `--storage-path` option can be used to change the storage path.
 
-Two package indexes are available:
+The following package indexes are available:
 
 - `packages/local`: which only allows locally published packages to be installed
-- `packages/all`: which includes local packages but allows missing packages to be pulled from PyPI
+- `packages/pypi`: which mirrors the production PyPI index
+- `packages/test-pypi`: which mirrors the test PyPI index
+- `packages/all`: which includes all of the listed indexes, with priority following the list order above
 
 When installing packages, you can choose the index to use by passing the `--index-url` flag to the installer
 e.g. with `pip`:

src/packse/index.py

@@ -87,27 +87,32 @@ def index_up(
 
             all_index = "packages/all"
             local_index = "packages/local"
-            pypi_index = "root/pypi"
+            pypi_index = "packages/pypi"
+            test_pypi_index = "packages/test-pypi"
 
             # First, create the "local" index which does not allow fallback to PyPI
             create_index(local_index, client_storage, exists_ok=background, bases=[])
 
+            # Crate mirror indexes for PyPI
+            create_mirror_index(
+                test_pypi_index, client_storage, "https://test.pypi.org/simple/"
+            )
+            create_mirror_index(pypi_index, client_storage, "https://pypi.org/simple/")
+
             # Then, create the "all" index which pulls from the "local" index or PyPI
             create_index(
                 all_index,
                 client_storage,
                 exists_ok=background,
-                bases=[local_index, pypi_index],
+                bases=[local_index, pypi_index, test_pypi_index],
             )
 
-            logger.info("Ensuring local index has build dependencies...")
+            logger.info("Uploading build dependencies to local index...")
             add_build_requirements(local_index, client_storage)
 
             all_index_url = f"{server_url}/{all_index}"
             local_index_url = f"{server_url}/{local_index}"
-            logger.info(
-                "Indexes available at %s and %s", all_index_url, local_index_url
-            )
+            logger.info("Indexes available at %s/<index-name>", server_url)
 
             logger.debug(
                 "To use `devpi` commands, include `--clientdir %s`", client_storage
@@ -289,6 +294,34 @@ def add_build_requirements(
     )
 
 
+def create_mirror_index(
+    name: str,
+    client_storage: Path,
+    url: str,
+    exists_ok: bool = False,
+):
+    logger.info("Creating mirror package index %r for %s...", name, url)
+    command = [
+        "devpi",
+        "index",
+        "--clientdir",
+        str(client_storage),
+        "-c",
+        name,
+        "type=mirror",
+        f"mirror_url={url}",
+    ]
+    try:
+        subprocess.check_output(
+            command,
+            stderr=subprocess.STDOUT,
+        )
+    except subprocess.CalledProcessError as exc:
+        # TODO(zanieb): Improve these error handling cases
+        if not exists_ok and "409 Conflict" not in exc.stdout.decode():
+            raise
+
+
 def create_index(
     name: str,
     client_storage: Path,

tests/__snapshots__/test_index.ambr

@@ -49,9 +49,11 @@
       Starting server at http://localhost:3141...
       Configuring client...
       Creating package index 'packages/local'...
+      Creating mirror package index 'packages/test-pypi' for https://test.pypi.org/simple/...
+      Creating mirror package index 'packages/pypi' for https://pypi.org/simple/...
       Creating package index 'packages/all'...
-      Ensuring local index has build dependencies...
-      Indexes available at http://localhost:3141/packages/all and http://localhost:3141/packages/local
+      Uploading build dependencies to local index...
+      Indexes available at http://localhost:3141/<index-name>
       Running in background with pid [PID]
       Stop index server with `packse index down`.
 
@@ -70,9 +72,11 @@
       Starting server at http://localhost:3141...
       Configuring client...
       Creating package index 'packages/local'...
+      Creating mirror package index 'packages/test-pypi' for https://test.pypi.org/simple/...
+      Creating mirror package index 'packages/pypi' for https://pypi.org/simple/...
       Creating package index 'packages/all'...
-      Ensuring local index has build dependencies...
-      Indexes available at http://localhost:3141/packages/all and http://localhost:3141/packages/local
+      Uploading build dependencies to local index...
+      Indexes available at http://localhost:3141/<index-name>
       Ready! [Stop with Ctrl-C]
       Interrupted!
 
@@ -88,9 +92,11 @@
       Starting server at http://localhost:3141...
       Configuring client...
       Creating package index 'packages/local'...
+      Creating mirror package index 'packages/test-pypi' for https://test.pypi.org/simple/...
+      Creating mirror package index 'packages/pypi' for https://pypi.org/simple/...
       Creating package index 'packages/all'...
-      Ensuring local index has build dependencies...
-      Indexes available at http://localhost:3141/packages/all and http://localhost:3141/packages/local
+      Uploading build dependencies to local index...
+      Indexes available at http://localhost:3141/<index-name>
       Running in background with pid [PID]
       Stop index server with `packse index down`.