diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index b1516eb3..9cb153fe 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -35,6 +35,7 @@ jobs: type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} type=ref,event=tag type=raw,value=stable,enable=${{ startsWith(github.ref, 'refs/tags/') }} + type=ref,event=tag type=ref,event=pr - name: Show tags @@ -65,7 +66,7 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} build-args: | - BASE_APE_IMAGE_TAG=latest-slim + BASE_APE_IMAGE_TAG=stable - name: Fetch all tags and store them run: | diff --git a/Dockerfile b/Dockerfile index 8593a635..d718c673 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,13 @@ RUN pip install --upgrade pip && pip install wheel RUN pip wheel . --wheel-dir=/wheels # Install from wheels -FROM ghcr.io/apeworx/ape:${BASE_APE_IMAGE_TAG:-latest-slim} +FROM ghcr.io/apeworx/ape:${BASE_APE_IMAGE_TAG:-latest} USER root COPY --from=builder /wheels /wheels RUN pip install --upgrade pip \ - && pip install silverback --no-cache-dir --find-links=/wheels + && pip install silverback \ + 'taskiq-sqs>=0.0.11' \ + --no-cache-dir --find-links=/wheels USER harambe ENTRYPOINT ["silverback"] diff --git a/README.md b/README.md index 0968f00d..ab0b8c3e 100644 --- a/README.md +++ b/README.md @@ -54,21 +54,69 @@ The example makes use of the [Ape Tokens](https://github.com/ApeWorX/ape-tokens) Be sure to properly configure your environment for the USDC and YFI tokens on Ethereum mainnet. ``` -To run your bot against a live network, this SDK includes a simple runner command you can use via: +To run your bot against a live network, this SDK includes a simple bot command you can use via: ```sh -$ silverback run "example:app" --network :mainnet:alchemy +$ silverback run example --network :mainnet:alchemy ``` ```{note} -This runner uses an in-memory task broker by default. +This bot uses an in-memory task broker by default. If you want to learn more about what that means, please visit the [development userguide](https://docs.apeworx.io/silverback/stable/userguides/development.html). ``` +```{note} +It is suggested that you create a bots/ folder in the root of your project. +Silverback will automatically register files in this folder as separate bots that can be run via the `silverback run` command. +``` + +```{note} +It is also suggested that you treat this as a scripts folder, and do not include an __init__.py +If you have a complicated project, follow the previous example to ensure you run the application correctly. +``` + +```{note} +A final suggestion would be to name your `SilverbackApp` object `bot`. Silverback automatically searches +for this object name when running. If you do not do so, once again, ensure you replace `example` with +`example:` the previous example. +``` + +To auto-generate Dockerfiles for your bots, from the root of your project, you can run: + +```bash +silverback build +``` + +This will place the generated dockerfiles in a special directory in the root of your project. + +As an example, if you have a bots directory that looks like: + +``` +bots/ +├── botA.py +├── botB.py +├── botC.py +``` + +This method will generate 3 Dockerfiles: + +``` +.silverback-images/ +├── Dockerfile.botA +├── Dockerfile.botB +├── Dockerfile.botC +``` + +These Dockerfiles can be deployed with the `docker push` command documented in the next section so you can use it in cloud-based deployments. + +```{note} +As an aside, if your bots/ directory is a python package, you will cause conflicts with the dockerfile generation feature. This method will warn you that you are generating bots for a python package, but will not stop you from doing so. If you choose to generate dockerfiles, the user should be aware that it will only copy each individual file into the Dockerfile, and will not include any supporting python functionality. Each python file is expected to run independently. If you require more complex bots, you will have to build a custom docker image. +``` + ## Docker Usage ```sh -$ docker run --volume $PWD:/home/harambe/project --volume ~/.tokenlists:/home/harambe/.tokenlists apeworx/silverback:latest run "example:app" --network :mainnet +$ docker run --volume $PWD:/home/harambe/project --volume ~/.tokenlists:/home/harambe/.tokenlists apeworx/silverback:latest run example --network :mainnet ``` ```{note} @@ -85,7 +133,7 @@ If you want to use a hosted provider with websocket support like Alchemy to run If you attempt to run the `Docker Usage` command without supplying this key, you will get the following error: ```bash -$ docker run --volume $PWD:/home/harambe/project --volume ~/.tokenlists:/home/harambe/.tokenlists apeworx/silverback:latest run "example:app" --network :mainnet:alchemy +$ docker run --volume $PWD:/home/harambe/project --volume ~/.tokenlists:/home/harambe/.tokenlists apeworx/silverback:latest run example --network :mainnet:alchemy Traceback (most recent call last): ... ape_alchemy.exceptions.MissingProjectKeyError: Must set one of $WEB3_ALCHEMY_PROJECT_ID, $WEB3_ALCHEMY_API_KEY, $WEB3_ETHEREUM_MAINNET_ALCHEMY_PROJECT_ID, $WEB3_ETHEREUM_MAINNET_ALCHEMY_API_KEY. diff --git a/docs/userguides/development.md b/docs/userguides/development.md index fe5884ed..0a1431cd 100644 --- a/docs/userguides/development.md +++ b/docs/userguides/development.md @@ -7,6 +7,68 @@ In this guide, we are going to show you more details on how to build an applicat You should have a python project with Silverback installed. You can install Silverback via `pip install silverback` +## Project structure + +There are 3 suggested ways to structure your project. In the root directory of your project: + +1. Create a `bot.py` file. This is the simplest way to define your bot project. + +2. Create a `bots/` folder. Then develop bots in this folder as separate scripts (Do not include a __init__.py file). + +3. Create a `bot/` folder with a `__init__.py` file that will include the instantiation of your `SilverbackApp()` object. + +The `silverback` cli automatically searches for python scripts to run as bots in specific locations relative to the root of your project. +It will also be able to detect the scripts inside your `bots/` directory and let you run those by name (in case you have multiple bots in your project). + +If `silverback` finds a module named `bot` in the root directory of the project, then it will use that by default. + +```{note} +It is suggested that you create the instance of your `SilverbackApp()` object by naming the variable `bot`, since `silverback` will autodetect that variable name when loading your script file. +``` + +Another way you can structure your bot is to create a `bot` folder and define a runner inside of that folder as `__init__.py`. + +If you have a more complicated project that requires multiple bots, naming each bot their own individual name is okay to do, and we encourage you to locate them under the `bots/` folder relative to the root of your project. +This will work fairly seamlessly with the rest of the examples shown in this guide. + +To run a bot, as long as your project directory follows the suggestions above by using a `bot` module, you can run it easily with: + +```bash +silverback run --network your:network:of:choice +``` + +If your bot's module name is `example.py` (for example), you can run it like this: + +```bash +silverback run example --network your:network:of:choice +``` + +If the variable that you call the `SilverbackApp()` object is something other than `bot`, you can specific that by adding `:{variable-name}`: + +```bash +silverback run example:my_bot --network your:network:of:choice +``` + +We will automatically detect all scripts under the `bots/` folder automatically, but if your bot resides in a location other than `bots/` then you can use this to run it: + +```bash +silverback run folder.example:app --network your:network:of:choice +``` + +Note that with a `bot/__init__.py` setup, silverback will also autodetect it, and you can run it with: + +```bash +silverback run --network your:network:of:choice +``` + +```{note} +It is suggested that you develop your bots as scripts to keep your deployments simple. +If you have a deep understanding of containerization, and have specific needs, you can set your bots up however you'd like, and then create your own container definitions for deployments to publish to your reqistry of choice. +For the most streamlined experience, develop your bots as scripts, and avoid relying on local packages +(e.g. do not include an `__init__.py` file inside your `bots/` directory, and do not use local modules inside `bots/` for reusable code). +If you follow these suggestions, your Silverback deployments will be easy to use and require almost no thought. +``` + ## Creating an Application Creating a Silverback Application is easy, to do so initialize the `silverback.SilverbackApp` class: @@ -14,7 +76,7 @@ Creating a Silverback Application is easy, to do so initialize the `silverback.S ```py from silverback import SilverbackApp -app = SilverbackApp() +bot = SilverbackApp() ``` The SilverbackApp class handles state and configuration. @@ -22,7 +84,7 @@ Through this class, we can hook up event handlers to be executed each time we en Initializing the app creates a network connection using the Ape configuration of your local project, making it easy to add a Silverback bot to your project in order to perform automation of necessary on-chain interactions required. However, by default an app has no configured event handlers, so it won't be very useful. -This is where adding event handlers is useful via the `app.on_` method. +This is where adding event handlers is useful via the `bot.on_` method. This method lets us specify which event will trigger the execution of our handler as well as which handler to execute. ## New Block Events @@ -32,7 +94,7 @@ To add a block handler, you will do the following: ```py from ape import chain -@app.on_(chain.blocks) +@bot.on_(chain.blocks) def handle_new_block(block): ... ``` @@ -50,7 +112,7 @@ from ape import Contract TOKEN = Contract() -@app.on_(TOKEN.Transfer) +@bot.on_(TOKEN.Transfer) def handle_token_transfer_events(transfer): ... ``` @@ -66,12 +128,12 @@ Any errors you raise during this function will get captured by the client, and r If you have heavier resources you want to load during startup, or want to initialize things like database connections, you can add a worker startup function like so: ```py -@app.on_worker_startup() +@bot.on_worker_startup() def handle_on_worker_startup(state): # Connect to DB, set initial state, etc ... -@app.on_worker_shutdown() +@bot.on_worker_shutdown() def handle_on_worker_shutdown(state): # cleanup resources, close connections cleanly, etc ... @@ -93,7 +155,7 @@ To access the state from a handler, you must annotate `context` as a dependency from typing import Annotated from taskiq import Context, TaskiqDepends -@app.on_(chain.blocks) +@bot.on_(chain.blocks) def block_handler(block, context: Annotated[Context, TaskiqDepends()]): # Access state via context.state ... @@ -104,7 +166,7 @@ def block_handler(block, context: Annotated[Context, TaskiqDepends()]): You can also add an application startup and shutdown handler that will be **executed once upon every application startup**. This may be useful for things like processing historical events since the application was shutdown or other one-time actions to perform at startup. ```py -@app.on_startup() +@bot.on_startup() def handle_on_startup(startup_state): # Process missed events, etc # process_history(start_block=startup_state.last_block_seen) @@ -112,17 +174,17 @@ def handle_on_startup(startup_state): ... -@app.on_shutdown() +@bot.on_shutdown() def handle_on_shutdown(): # Record final state, etc ... ``` -*Changed in 0.2.0*: The behavior of the `@app.on_startup()` decorator and handler signature have changed. It is now executed only once upon application startup and worker events have moved on `@app.on_worker_startup()`. +*Changed in 0.2.0*: The behavior of the `@bot.on_startup()` decorator and handler signature have changed. It is now executed only once upon application startup and worker events have moved on `@bot.on_worker_startup()`. ### Signing Transactions -If configured, your bot with have `app.signer` which is an Ape account that can sign arbitrary transactions you ask it to. +If configured, your bot with have `bot.signer` which is an Ape account that can sign arbitrary transactions you ask it to. To learn more about signing transactions with Ape, see the [documentation](https://docs.apeworx.io/ape/stable/userguides/transactions.html). ```{warning} @@ -137,15 +199,19 @@ To run your bot locally, we have included a really useful cli command [`run`](.. ```sh # Run your bot on the Ethereum Sepolia testnet, with your own signer: -$ silverback run my_bot:app --network :sepolia --account acct-name +$ silverback run my_bot --network :sepolia --account acct-name +``` + +```{note} +`my_bot:bot` is not required for silverback run if you follow the suggested folder structure at the start of this page, you can just call it via `my_bot`. ``` -It's important to note that signers are optional, if not configured in the application then `app.signer` will be `None`. +It's important to note that signers are optional, if not configured in the application then `bot.signer` will be `None`. You can use this in your application to enable a "test execution" mode, something like this: ```py # Compute some metric that might lead to creating a transaction -if app.signer: +if bot.signer: # Execute a transaction via `sender=app.signer` else: # Log what the transaction *would* have done, had a signer been enabled @@ -183,7 +249,7 @@ export SILVERBACK_BROKER_KWARGS='{"queue_name": "taskiq", "url": "redis://127.0. export SILVERBACK_RESULT_BACKEND_CLASS="taskiq_redis:RedisAsyncResultBackend" export SILVERBACK_RESULT_BACKEND_URI="redis://127.0.0.1:6379" -silverback run "example:app" --network :mainnet:alchemy +silverback run --network :mainnet:alchemy ``` And then the worker process with 2 worker subprocesses: @@ -194,7 +260,7 @@ export SILVERBACK_BROKER_KWARGS='{"url": "redis://127.0.0.1:6379"}' export SILVERBACK_RESULT_BACKEND_CLASS="taskiq_redis:RedisAsyncResultBackend" export SILVERBACK_RESULT_BACKEND_URI="redis://127.0.0.1:6379" -silverback worker -w 2 "example:app" +silverback worker -w 2 ``` The client will send tasks to the 2 worker subprocesses, and all task queue and results data will be go through Redis. diff --git a/docs/userguides/platform.md b/docs/userguides/platform.md index b7d55917..f4e405fa 100644 --- a/docs/userguides/platform.md +++ b/docs/userguides/platform.md @@ -35,7 +35,51 @@ Building a container for your application can be an advanced topic, we have incl ## Building your Bot -TODO: Add build process and describe `silverback build --autogen` and `silverback build --upgrade` +To build your container definition(s) for your bot(s), you can use the `silverback build` command. This command searches your `bots` directory for python modules, then auto-generates Dockerfiles. + +For example, if your directory is structured as suggested in [development](./development), and your `bots/` directory looks like this: + +``` +bots/ +├── botA.py +├── botB.py +├── botC.py +``` + +Then you can use `silverback build --generate` to generate 3 separate Dockerfiles for those bots, and start trying to build them. + +Those Dockerfiles will appear under `.silverback-images/` as follows: + +```bash +silverback build --generate +``` + +This method will generate 3 Dockerfiles: + +``` +.silverback-images/ +├── Dockerfile.botA +├── Dockerfile.botB +├── Dockerfile.botC +``` + +You can retry you builds using the following (assuming you don't modify the structure of your project): + +```bash +silverback build +``` + +You can then push your image to your registry using: + +```bash +docker push your-registry-url/project/botA:latest +``` + +TODO: The ApeWorX team has github actions definitions for building, pushing and deploying. + +If you are unfamiliar with docker and container registries, you can use the \[\[github-action\]\]. + +You do not need to build using this command if you use the github action, but it is there to help you if you are having problems figuring out how to build and run your bot images on the cluster successfully. TODO: Add how to debug containers using `silverback run` w/ `taskiq-redis` broker diff --git a/silverback/_cli.py b/silverback/_cli.py index 191cc7a6..67a5ace7 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -1,5 +1,8 @@ import asyncio import os +import shlex +import subprocess +from pathlib import Path import click import yaml # type: ignore[import-untyped] @@ -15,17 +18,29 @@ from silverback._click_ext import ( SectionedHelpGroup, auth_required, + bot_path_callback, cls_import_callback, cluster_client, display_login_message, platform_client, ) -from silverback._importer import import_from_string from silverback.cluster.client import ClusterClient, PlatformClient from silverback.cluster.types import ClusterTier from silverback.runner import PollingRunner, WebsocketRunner from silverback.worker import run_worker +DOCKERFILE_CONTENT = """ +FROM ghcr.io/apeworx/silverback:stable +USER root +WORKDIR /app +RUN chown harambe:harambe /app +USER harambe +COPY ape-config.yaml . +COPY requirements.txt . +RUN pip install --upgrade pip && pip install -r requirements.txt +RUN ape plugins install . +""" + @click.group(cls=SectionedHelpGroup) def cli(): @@ -87,8 +102,8 @@ def _network_callback(ctx, param, val): callback=cls_import_callback, ) @click.option("-x", "--max-exceptions", type=int, default=3) -@click.argument("path") -def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): +@click.argument("bot", required=False, callback=bot_path_callback) +def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, bot): """Run Silverback application""" if not runner_class: @@ -102,15 +117,69 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): option_name="network", message="Network choice cannot support running app" ) - app = import_from_string(path) runner = runner_class( - app, + bot, recorder=recorder_class() if recorder_class else None, max_exceptions=max_exceptions, ) asyncio.run(runner.run()) +@cli.command(section="Local Commands") +@click.option("--generate", is_flag=True, default=False) +@click.argument("path", required=False, type=str, default="bots") +def build(generate, path): + """Generate Dockerfiles and build bot images""" + if generate: + if not (path := Path.cwd() / path).exists(): + raise FileNotFoundError( + f"The bots directory '{path}' does not exist. " + "You should have a `{path}/` folder in the root of your project." + ) + files = {file for file in path.iterdir() if file.is_file()} + bots = [] + for file in files: + if "__init__" in file.name: + bots = [file] + break + bots.append(file) + for bot in bots: + dockerfile_content = DOCKERFILE_CONTENT + if "__init__" in bot.name: + docker_filename = f"Dockerfile.{bot.parent.name}" + dockerfile_content += f"COPY {path.name}/ /app/bot" + else: + docker_filename = f"Dockerfile.{bot.name.replace('.py', '')}" + dockerfile_content += f"COPY {path.name}/{bot.name} /app/bot.py" + dockerfile_path = Path.cwd() / ".silverback-images" / docker_filename + dockerfile_path.parent.mkdir(exist_ok=True) + dockerfile_path.write_text(dockerfile_content.strip() + "\n") + click.echo(f"Generated {dockerfile_path}") + return + + if not (path := Path.cwd() / ".silverback-images").exists(): + raise FileNotFoundError( + f"The dockerfile directory '{path}' does not exist. " + "You should have a `{path}/` folder in the root of your project." + ) + dockerfiles = {file for file in path.iterdir() if file.is_file()} + for file in dockerfiles: + try: + command = shlex.split( + "docker build -f " + f"./{file.parent.name}/{file.name} " + f"-t {file.name.split('.')[1]}:latest ." + ) + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True + ) + click.echo(result.stdout) + except subprocess.CalledProcessError as e: + click.echo("Error during docker build:") + click.echo(e.stderr) + raise + + @cli.command(cls=ConnectedProviderCommand, section="Local Commands") @ape_cli_context() @network_option( @@ -121,12 +190,10 @@ def run(cli_ctx, account, runner_class, recorder_class, max_exceptions, path): @click.option("-w", "--workers", type=int, default=2) @click.option("-x", "--max-exceptions", type=int, default=3) @click.option("-s", "--shutdown_timeout", type=int, default=90) -@click.argument("path") -def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, path): +@click.argument("bot", required=False, callback=bot_path_callback) +def worker(cli_ctx, account, workers, max_exceptions, shutdown_timeout, bot): """Run Silverback task workers (advanced)""" - - app = import_from_string(path) - asyncio.run(run_worker(app.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) + asyncio.run(run_worker(bot.broker, worker_count=workers, shutdown_timeout=shutdown_timeout)) @cli.command(section="Cloud Commands (https://silverback.apeworx.io)") diff --git a/silverback/_click_ext.py b/silverback/_click_ext.py index d526e499..e849f3dc 100644 --- a/silverback/_click_ext.py +++ b/silverback/_click_ext.py @@ -15,6 +15,8 @@ ProfileSettings, ) +from .exceptions import ImportFromStringError + # NOTE: only load once settings = ProfileSettings.from_config_file() @@ -254,3 +256,15 @@ def get_cluster_client(ctx: click.Context, *args, **kwargs): return ctx.invoke(f, *args, **kwargs) return update_wrapper(get_cluster_client, f) + + +def bot_path_callback(ctx: click.Context, param: click.Parameter, path: str | None): + if not path: + path = "bot:bot" + elif ":" not in path: + path += ":bot" + + try: + return import_from_string(path) + except ImportFromStringError: + return import_from_string(f"bots.{path}")