Move build support into lkdev package

.github/workflows/test.yml

@@ -31,9 +31,14 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: 🐍 Setup bootstrap Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: 👢 Generate Conda environment file
         run: |
-          pipx run ./utils/conda-tool.py --env -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml
+          pip install -e .
+          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml
       - id: setup
         name: 📦 Set up Conda environment
         uses: mamba-org/setup-micromamba@v1
@@ -236,9 +241,14 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: 🐍 Setup bootstrap Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: 👢 Generate Conda environment file
         run: |
-          pipx run ./utils/conda-tool.py --env -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml
+          pip install -e .
+          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml
       - id: setup
         name: 📦 Set up Conda environment
         uses: mamba-org/setup-micromamba@v1
@@ -333,9 +343,14 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: 🐍 Setup bootstrap Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: 👢 Generate Conda environment file
         run: |
-          pipx run ./utils/conda-tool.py --env -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-implicit/pyproject.toml
+          pip install -e .
+          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-implicit/pyproject.toml
       - id: setup
         name: 📦 Set up Conda environment
         uses: mamba-org/setup-micromamba@v1
@@ -476,9 +491,14 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: 🐍 Setup bootstrap Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: 👢 Generate Conda environment file
         run: |
-          pipx run ./utils/conda-tool.py --env -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml lenskit-implicit/pyproject.toml
+          pip install -e .
+          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml lenskit-implicit/pyproject.toml
       - id: setup
         name: 📦 Set up Conda environment
         uses: mamba-org/setup-micromamba@v1
@@ -525,9 +545,14 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: 🐍 Setup bootstrap Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: 👢 Generate Conda environment file
         run: |
-          pipx run ./utils/conda-tool.py --env -o ci-environment.yml -e all requirements-test.txt requirements-demo.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml lenskit-implicit/pyproject.toml
+          pip install -e .
+          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt requirements-demo.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml lenskit-implicit/pyproject.toml
       - id: setup
         name: 📦 Set up Conda environment
         uses: mamba-org/setup-micromamba@v1

README.md

@@ -55,13 +55,13 @@ contains other information on *developing* LensKit. User-facing documentation is
 at <https://lkpy.lenskit.org>.
 
 [conda-lock]: https://github.com/conda-incubator/conda-lock
-[lkbuild]: https://github.com/lenskit/lkbuild
+[lkdev]: https://github.com/lenskit/lkdev
 
 We recommend using an Anaconda environment for developing LensKit.  We provide a
 tool to automate setting up Conda environments from the LensKit dependencies; to
 create a dev environment, checkout LensKit, then run:
 
-    pipx ./utils/conda-tool.py --env -n lkpy pyproject.toml dev-requirements.txt
+    pipx --spec . lk-conda -n lkpy pyproject.toml dev-requirements.txt
     conda activate lkpy
 
 That will create and activate an environment named `lkpy` with all the LensKit

codecov.yml

@@ -1,4 +1,4 @@
 ignore:
   - build*.py
   - build-tools/*
-  - lkbuild/*
+  - lkdev/*

docs/environment.yml

@@ -11,8 +11,8 @@ dependencies:
   - myst-nb >=0.13
   - pytorch-cpu
   - nomkl
-  - pandas >=1.5,<3
-  - numpy >=1.23
+  - pandas <3,>=1.5
+  - numpy >=1.23,<2
   - scipy >=1.9.0
   - pytorch >=2.1,<3
   - threadpoolctl >=3.0

justfile

@@ -33,12 +33,13 @@ install-dev:
 
 # create a conda environment file for development
 conda-env-file version="3.11":
-    pipx run ./utils/conda-tool.py --env -o environment.yml -e all requirements-dev.txt \
+    pipx run --spec . lk-conda -o environment.yml -e all requirements-dev.txt \
+        pyproject.toml \
         {{ append('/pyproject.toml', PACKAGES) }}
 
 # create a conda environment for development
 conda-env version="3.11" name="lkpy":
-    pipx run ./utils/conda-tool.py --env -n {{name}} -e all requirements-dev.txt \
+    pipx run --spec . lk-conda -n {{name}} -e all requirements-dev.txt \
         {{ append('/pyproject.toml', PACKAGES) }}
 
 # run tests with default configuration
@@ -63,15 +64,16 @@ preview-docs:
 
 # update the environment file used to install documentation
 update-doc-env:
-    pipx run ./utils/conda-tool.py --env -o docs/environment.yml \
+    python -m lkdev.conda -o docs/environment.yml \
         -e all requirements-doc.txt docs/doc-dep-constraints.yml \
         {{ append('/pyproject.toml', PACKAGES) }}
-
+    -pre-commit run --files docs/environment.yml
 
 # update source file headers
 update-headers:
     unbehead
 
 # update GH workflows
 update-workflows:
-    python ./utils/render-test-workflow.py -o .github/workflows/test.yml
+    python -m lkdev.ghactions --render test
+    -pre-commit run --files .github/workflows/*.yml

lenskit/README.md

@@ -55,7 +55,7 @@ contains other information on *developing* LensKit. User-facing documentation is
 at <https://lkpy.lenskit.org>.
 
 [conda-lock]: https://github.com/conda-incubator/conda-lock
-[lkbuild]: https://github.com/lenskit/lkbuild
+[lkdev]: https://github.com/lenskit/lkdev
 
 We recommend using an Anaconda environment for developing LensKit.  We provide a
 tool to automate setting up Conda environments from the LensKit dependencies; to

lkdev/__init__.py

@@ -0,0 +1,4 @@
+"""
+LensKit build bootstrap utilities.  This repository should generally not be
+installed.
+"""

utils/conda-tool.py → lkdev/conda.py

@@ -9,9 +9,9 @@
 Generate and manage Conda environments.
 
 Usage:
-    conda-tool.py [-v] --env [-o FILE | -n NAME]
-        [--micromamba]
-        [-p VER] [--mkl] [--cuda] [-e EXTRA]... REQFILE...
+    lkdev.conda [-v] [-o FILE | -n NAME]
+        [--micromamba] [--mkl] [--cuda]
+        [-p VER] [-e EXTRA]... REQFILE...
 
 Options:
     -v, --verbose   enable verbose logging
@@ -32,10 +32,6 @@
 bootstrap LensKit and therefore cannot depend on LensKit being installed.
 """
 
-# /// script
-# requires-python = ">= 3.10"
-# dependencies = ["tomlkit>=0.12", "pyyaml==6.*", "packaging>=24.0", "docopt>=0.6"]
-# ///
 # pyright: strict, reportPrivateUsage=false
 from __future__ import annotations
 
@@ -103,38 +99,33 @@ def __str__(self):
 
 
 def main():
-    init_logging()
-
-    if options["--env"]:
-        specs = [
-            r
-            for r in load_reqfiles(options["REQFILE"])
-            if not re.match(r"lenskit($|-\w+)$", r.name)
-        ]
-        env = make_env_object(specs, options["--python-version"])
-        if options["--name"]:
-            tmpfd, tmpf = mkstemp(suffix="-env.yml", prefix="lkpy-")
-            try:
-                _log.debug("saving environment to %s", tmpf)
-                with os.fdopen(tmpfd, "wt") as tf:
-                    yaml.safe_dump(env, tf)
-                _log.debug("creating environment")
-                conda = find_conda_executable()
-                sp.check_call([conda, "env", "create", "-n", options["--name"], "-f", tmpf])
-            finally:
-                os.unlink(tmpf)
-        elif options["--output"]:
-            _log.info("writing to file %s", options["--output"])
-            with open(options["--output"], "wt") as f:
-                yaml.safe_dump(env, f)
-        else:
-            yaml.safe_dump(env, sys.stdout)
-
-
-def init_logging():
+    global options
+    options = docopt(__doc__)
     level = logging.DEBUG if options["--verbose"] else logging.INFO
     logging.basicConfig(stream=sys.stderr, level=level)
 
+    specs = [
+        r for r in load_reqfiles(options["REQFILE"]) if not re.match(r"lenskit($|-\w+)$", r.name)
+    ]
+    env = make_env_object(specs, options["--python-version"])
+    if options["--name"]:
+        tmpfd, tmpf = mkstemp(suffix="-env.yml", prefix="lkpy-")
+        try:
+            _log.debug("saving environment to %s", tmpf)
+            with os.fdopen(tmpfd, "wt") as tf:
+                yaml.safe_dump(env, tf)
+            _log.debug("creating environment")
+            conda = find_conda_executable()
+            sp.check_call([conda, "env", "create", "-n", options["--name"], "-f", tmpf])
+        finally:
+            os.unlink(tmpf)
+    elif options["--output"]:
+        _log.info("writing to file %s", options["--output"])
+        with open(options["--output"], "wt") as f:
+            yaml.safe_dump(env, f)
+    else:
+        yaml.safe_dump(env, sys.stdout)
+
 
 def load_reqfiles(files: Iterable[Path | str]) -> Iterator[ParsedReq]:
     for file in files:
@@ -303,5 +294,4 @@ def make_env_object(specs: list[ParsedReq], python: Optional[str] = None) -> dic
 
 
 if __name__ == "__main__":
-    options = docopt(__doc__)
     main()

lkdev/ghactions.py

@@ -0,0 +1,108 @@
+"""
+Manage GitHub Actions workflows and template infrastructure
+
+Usage:
+    render-test-workflow.py [-v] [-o FILE] --render WORKFLOW
+
+Options:
+    -v, --verbose           verbose logging
+    --render WORKFLOW       render the workflow WORKFLOW
+    -o FILE, --output=FILE  render to FILE (- for stdout)
+"""
+
+# pyright: strict
+from __future__ import annotations
+
+import logging
+import sys
+from importlib import import_module
+from textwrap import dedent
+from typing import Any, NotRequired, TypedDict
+
+import yaml
+from docopt import docopt
+
+_log = logging.getLogger("lkdev.ghactions")
+
+
+class script:
+    source: str
+
+    def __init__(self, source: str):
+        self.source = dedent(source).strip() + "\n"
+
+    @staticmethod
+    def presenter(dumper: yaml.Dumper, script: script):
+        return dumper.represent_scalar("tag:yaml.org,2002:str", script.source, style="|")  # type: ignore
+
+    @classmethod
+    def command(cls, args: list[str]):
+        return cls(" ".join(args))
+
+
+yaml.add_representer(script, script.presenter)
+
+GHStep = TypedDict(
+    "GHStep",
+    {
+        "id": NotRequired[str],
+        "name": NotRequired[str],
+        "uses": NotRequired[str],
+        "run": NotRequired[str | script],
+        "shell": NotRequired["str"],
+        "with": NotRequired[dict[str, str | int | bool | script]],
+        "env": NotRequired[dict[str, str | int]],
+    },
+)
+
+GHJob = TypedDict(
+    "GHJob",
+    {
+        "name": str,
+        "runs-on": str,
+        "timeout-minutes": NotRequired[int],
+        "strategy": NotRequired[dict[str, Any]],
+        "defaults": NotRequired[dict[str, Any]],
+        "needs": NotRequired[list[str]],
+        "steps": NotRequired[list[GHStep]],
+    },
+)
+
+
+def main():
+    options = docopt(__doc__)
+    level = logging.DEBUG if options["--verbose"] else logging.INFO
+    logging.basicConfig(stream=sys.stderr, level=level)
+
+    render = options["--render"]
+    if render:
+        render_workflow(render, options)
+
+
+def render_workflow(name: str, options: dict[str, Any]):
+    mod_name = f"lkdev.workflows.{name}"
+    _log.info("loading module %s", mod_name)
+    mod = import_module(mod_name)
+
+    workflow = mod.workflow()
+
+    outfn = options["--output"]
+    if outfn == "-":
+        out = sys.stdout
+    elif outfn:
+        _log.info("saving to %s", outfn)
+        out = open(outfn, "wt")
+    else:
+        outfn = f".github/workflows/{name}.yml"
+        _log.info("saving to %s", outfn)
+        out = open(outfn, "wt")
+
+    try:
+        yaml.dump(workflow, out, allow_unicode=True, sort_keys=False)
+    finally:
+        if out is not sys.stdout:
+            out.close()
+
+
+if __name__ == "__main__":
+    main()

lkdev/workflows/__init__.py

@@ -0,0 +1,3 @@
+"""
+Workflow definitions for LensKit build.
+"""