Merged PR 6563359: Python and file content support

This PR adds support for two new types of content: Python packages and files. Each can be managed by Pulp.

For right now, I am referring to file content (e.g. [the clamav signatures](https://packages.microsoft.com/clamav/)) with the name `FilePackage` to be consistent with the other types of content we offer (RpmPackage, PythonPackage, DebPackage). I'm not totally happy with this name but there are two arguments for it:

1. It keeps things consistent/easy/simple
2. We (the pmc team) will probably be the only ones to manage file content so the name doesn't matter too much IMO

That said, I'm open to suggestion for how to name things. Here are some examples of where/how this name manifests:

```
# schemas
RpmPackageResponse
DebPackageResponse
PythonPackageResponse
FilePackageResponse ?

# api endpoints
/rpm/packages/
/python/packages/
/python/packages/
/file/packages/ ?

# cli commands
pmc package rpm list
pmc package deb list
pmc package python list
pmc package file list ?
```

Related work items: #15206527

cli/.gitignore

@@ -1 +1,2 @@
 tests/settings.toml
+dist

cli/README.md

@@ -84,66 +84,42 @@ A default Service Principal is available to simplify your dev environment
   - `./update_role.sh Repo_Admin --create`
   - You can call this script again any time you wish to change roles (`./update_role.sh Account_Admin`)
 
-## Example Workflows
+## Workflows
 
-### apt
+Once you've set up the server and CLI, view the `docs/admin/workflows.md` file for some example
+workflows.
 
-```
-# create a repo. Note: only legacy signing is available in dev environments.
-pmc repo create myrepo-apt apt --signing-service legacy
-
-# create a repo release
-pmc repo releases create myrepo-apt jammy
-
-# create a distro
-pmc distro create mydistro-apt apt "some/path" --repository myrepo-apt
-
-# upload a package
-cp tests/assets/signed-by-us.deb .
-PACKAGE_ID=$(pmc --id-only package upload signed-by-us.deb)
+## Publishing
 
-# add our package to the repo release
-pmc repo packages update myrepo-apt jammy --add-packages $PACKAGE_ID
+### Server Setup
 
-# publish the repo
-pmc repo publish myrepo-apt
+These steps describe how to prepare the PMC server to distribute the CLI package. They assume that
+your pmc client is setup for the server and that you want to distribute the package at `pypi`.
 
-# check out our repo
-http :8081/pulp/content/some/path/
 ```
+# create a repo named pypi-python
+pmc repo create pypi-python python
 
-### yum
-
+# create a distro pypi that serves from a folder 'pypi'
+pmc distro create pypi pypi pypi --repository pypi-python
 ```
-# create a repo. Note: only legacy signing is available in dev environments.
-pmc repo create myrepo-yum yum --signing-service legacy
-
-# create a distro
-pmc distro create mydistro-yum yum "awesome/path" --repository myrepo-yum
 
-# upload a package
-cp tests/assets/signed-by-us.rpm .
-PACKAGE_ID=$(pmc --id-only package upload signed-by-us.deb)
+### Packaging
 
-# add our package to the repo release
-pmc repo packages update myrepo-yum --add-packages $PACKAGE_ID
+1. Open up pyproject.toml file and confirm that the version field is correct.
+2. If it's not correct, update it and open a new PR with your change.
+3. In the cli directory, run `poetry build`.
+4. Now proceed to the next section to upload your CLI package.
 
-# publish the repo
-pmc repo publish myrepo-yum
-
-# check out our repo
-http :8081/pulp/content/awesome/path/
-```
+### Uploading
 
-### syncing
+These steps assume that your pmc client is set up for the server from which you want to distribute
+the pmc cli package.
 
 ```
-# create a remote
-pmc remote create microsoft-ubuntu-focal-prod apt "https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod/" --distributions nightly
+PACKAGE_ID=$(pmc package upload dist/pmc_cli-0.0.1-py3-none-any.whl)
 
-# create a repo
-pmc repo create microsoft-ubuntu-focal-prod apt --remote microsoft-ubuntu-focal-prod
+pmc repo packages update pypi-python --add-packages $PACKAGE_ID
 
-# sync
-pmc repo sync microsoft-ubuntu-focal-prod
+pmc repo publish pypi-python
 ```

cli/pmc/commands/package.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 import typer
 
@@ -9,15 +9,19 @@
 app = UserFriendlyTyper()
 deb = UserFriendlyTyper()
 rpm = UserFriendlyTyper()
+python = UserFriendlyTyper()
+file = UserFriendlyTyper()
+
 app.add_typer(deb, name="deb", help="Manage deb packages")
 app.add_typer(rpm, name="rpm", help="Manage rpm packages")
+app.add_typer(python, name="python", help="Manage python packages")
+app.add_typer(file, name="file", help="Manage files")
 
 
-def _list(type: PackageType, ctx: typer.Context, limit: int, offset: int) -> None:
+def _list(package_type: PackageType, ctx: typer.Context, limit: int, offset: int) -> None:
     params: Dict[str, Any] = dict(limit=limit, offset=offset)
-
     with get_client(ctx.obj) as client:
-        resp = client.get(f"/{type}/packages/", params=params)
+        resp = client.get(f"/{package_type}/packages/", params=params)
         handle_response(ctx.obj, resp)
 
 
@@ -33,6 +37,18 @@ def rpm_list(ctx: typer.Context, limit: int = LIMIT_OPT, offset: int = OFFSET_OP
     _list(PackageType.rpm, ctx, limit, offset)
 
 
+@python.command(name="list")
+def python_list(ctx: typer.Context, limit: int = LIMIT_OPT, offset: int = OFFSET_OPT) -> None:
+    """List python packages."""
+    _list(PackageType.python, ctx, limit, offset)
+
+
+@file.command(name="list")
+def file_list(ctx: typer.Context, limit: int = LIMIT_OPT, offset: int = OFFSET_OPT) -> None:
+    """List files."""
+    _list(PackageType.file, ctx, limit, offset)
+
+
 @app.command()
 def upload(
     ctx: typer.Context,
@@ -43,6 +59,15 @@ def upload(
         show_default=False,
         help="Ignore the signature check. Only allowable for legacy packages.",
     ),
+    file_type: Optional[PackageType] = typer.Option(
+        None,
+        "--type",
+        "-t",
+        help=(
+            "Manually specify the type of file being uploaded. Otherwise the file's extension "
+            "is used to guess the file type."
+        ),
+    ),
 ) -> None:
     """Upload a package."""
 
@@ -52,7 +77,9 @@ def show_func(task: Any) -> Any:
         with get_client(ctx.obj) as client:
             return client.get(f"/packages/{package_id}/")
 
-    data = {"ignore_signature": ignore_signature}
+    data: Dict[str, Any] = {"ignore_signature": ignore_signature}
+    if file_type:
+        data["file_type"] = file_type
     files = {"file": file}
     with get_client(ctx.obj) as client:
         resp = client.post("/packages/", params=data, files=files)

cli/pmc/commands/repository.py

@@ -25,23 +25,6 @@
 )
 
 
-def _signing_service_default(
-    ctx: typer.Context, value: Optional[RepoSigningService]
-) -> RepoSigningService:
-    # value will be the default if not user specified so check ctx.params instead
-    if service_val := ctx.params.get("signing_service"):
-        # use the user input
-        service = service_val
-    elif service_default := (ctx.find_root().default_map or {}).get("signing_service"):
-        # use the config value
-        service = service_default
-    else:
-        # default to esrp
-        service = RepoSigningService.esrp
-
-    return RepoSigningService(service)
-
-
 @app.command()
 def list(
     ctx: typer.Context,
@@ -65,7 +48,8 @@ def create(
     name: str,
     repo_type: RepoType,
     signing_service: Optional[RepoSigningService] = typer.Option(
-        RepoSigningService.esrp, callback=_signing_service_default
+        None,
+        help="Signing service to use for the repo. Defaults to 'esrp' for yum and apt repos.",
     ),
     remote: Optional[str] = id_or_name(
         "remotes", typer.Option(None, help="Remote id or name to use for sync.")
@@ -75,7 +59,19 @@ def create(
     ),
 ) -> None:
     """Create a repository."""
-    data = {"name": name, "type": repo_type, "signing_service": signing_service, "remote": remote}
+    data = {"name": name, "type": repo_type, "remote": remote}
+
+    # set signing service
+    if repo_type in [RepoType.yum, RepoType.apt]:
+        if signing_service:
+            service = signing_service
+        elif service_default := (ctx.find_root().default_map or {}).get("signing_service"):
+            service = service_default
+        else:
+            service = RepoSigningService.esrp
+
+        data["signing_service"] = service
+
     with get_client(ctx.obj) as client:
         repo_resp = client.post("/repositories/", json=data)
         handle_response(ctx.obj, repo_resp)

cli/pmc/schemas.py

@@ -40,6 +40,8 @@ class RepoType(StringEnum):
 
     apt = "apt"
     yum = "yum"  # maps to 'rpm' in Pulp
+    python = "python"
+    file = "file"
 
 
 class RepoSigningService(StringEnum):
@@ -54,6 +56,8 @@ class DistroType(StringEnum):
 
     apt = "apt"
     yum = "yum"  # maps to 'rpm' in Pulp
+    python = "pypi"
+    file = "file"
 
 
 class RemoteType(StringEnum):
@@ -68,6 +72,8 @@ class PackageType(StringEnum):
 
     deb = "deb"
     rpm = "rpm"
+    python = "python"
+    file = "file"
 
 
 class Format(StringEnum):

cli/tests/assets/hello.txt

@@ -0,0 +1 @@
+world

cli/tests/commands/test_config.py

@@ -19,7 +19,7 @@ def test_config_with_invalid_value() -> None:
     config.flush()
     result = invoke_command(["--config", config.name, "repo", "list"])
     assert result.exit_code == 1
-    assert "DecodeError" in result.stdout
+    assert "ValidationError" in result.stdout
 
 
 @pytest.mark.skip(reason="Authentication is required for all commands")

cli/tests/commands/test_package.py

@@ -1,5 +1,6 @@
 import json
-from typing import Any
+from pathlib import Path
+from typing import Any, Optional
 
 import pytest
 
@@ -10,7 +11,22 @@
 # Note that "package upload" is exercised by fixture.
 
 
-def _assert_package_list_not_empty(type: str) -> None:
+def test_upload_file_type(orphan_cleanup: None) -> None:
+    become(Role.Package_Admin)
+
+    # file without file type
+    path = Path.cwd() / "tests" / "assets" / "hello.txt"
+    result = invoke_command(["package", "upload", str(path)])
+    assert result.exit_code != 0
+    assert "Unrecognized file extension" in result.stdout
+
+    # python with file type
+    path = Path.cwd() / "tests" / "assets" / "helloworld-0.0.1.tar.gz"
+    result = invoke_command(["package", "upload", "--type", "python", str(path)])
+    assert result.exit_code == 0
+
+
+def _assert_package_list_not_empty(type: str, file_type: Optional[str] = None) -> None:
     result = invoke_command(["package", type, "list"])
     assert result.exit_code == 0
     response = json.loads(result.stdout)
@@ -25,6 +41,14 @@ def test_rpm_list(rpm_package: Any) -> None:
     _assert_package_list_not_empty("rpm")
 
 
+def test_file_list(file_package: Any) -> None:
+    _assert_package_list_not_empty("file", "file")
+
+
+def test_python_list(python_package: Any) -> None:
+    _assert_package_list_not_empty("python")
+
+
 def test_show(deb_package: Any) -> None:
     result = invoke_command(["package", "show", deb_package["id"]])
     assert result.exit_code == 0

cli/tests/conftest.py

@@ -83,6 +83,18 @@ def yum_repo() -> Generator[Any, None, None]:
         yield r
 
 
+@pytest.fixture()
+def file_repo() -> Generator[Any, None, None]:
+    with _object_manager(repo_create_cmd(gen_repo_attrs(RepoType.file)), Role.Repo_Admin) as r:
+        yield r
+
+
+@pytest.fixture()
+def python_repo() -> Generator[Any, None, None]:
+    with _object_manager(repo_create_cmd(gen_repo_attrs(RepoType.python)), Role.Repo_Admin) as r:
+        yield r
+
+
 @pytest.fixture()
 def distro() -> Generator[Any, None, None]:
     become(Role.Repo_Admin)
@@ -165,21 +177,26 @@ def _my_cmd(action: str) -> List[str]:
         yield o
 
 
-def package_upload_command(package_name: str, unsigned: Optional[bool] = False) -> List[str]:
+def package_upload_command(
+    package_name: str, unsigned: Optional[bool] = False, file_type: Optional[str] = None
+) -> List[str]:
     package = Path.cwd() / "tests" / "assets" / package_name
     cmd = ["package", "upload", str(package)]
 
     if unsigned:
         cmd.append("--ignore-signature")
 
+    if file_type:
+        cmd += ["--type", file_type]
+
     return cmd
 
 
 @contextmanager
 def _package_manager(
-    package_name: str, unsigned: Optional[bool] = False
+    package_name: str, unsigned: Optional[bool] = False, file_type: Optional[str] = None
 ) -> Generator[Any, None, None]:
-    cmd = package_upload_command(package_name, unsigned)
+    cmd = package_upload_command(package_name, unsigned, file_type)
     with _object_manager(cmd, Role.Package_Admin, False) as p:
         yield p
 
@@ -202,6 +219,18 @@ def rpm_package(orphan_cleanup: None) -> Generator[Any, None, None]:
         yield p
 
 
+@pytest.fixture()
+def file_package(orphan_cleanup: None) -> Generator[Any, None, None]:
+    with _package_manager("hello.txt", file_type="file") as p:
+        yield p
+
+
+@pytest.fixture()
+def python_package(orphan_cleanup: None) -> Generator[Any, None, None]:
+    with _package_manager("helloworld-0.0.1-py3-none-any.whl") as p:
+        yield p
+
+
 @pytest.fixture()
 def forced_unsigned_package(orphan_cleanup: None) -> Generator[Any, None, None]:
     with _package_manager("unsigned.rpm", unsigned=True) as p: