From 17c5e606c527a1e951ad88df4f1ee497adfae892 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Mon, 19 Feb 2024 10:54:12 +0100 Subject: [PATCH 01/61] feat(frontend): add messages page --- .../frontend/templates/messages.html | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 src/zulip_write_only_proxy/frontend/templates/messages.html diff --git a/src/zulip_write_only_proxy/frontend/templates/messages.html b/src/zulip_write_only_proxy/frontend/templates/messages.html new file mode 100644 index 00000000..5481d358 --- /dev/null +++ b/src/zulip_write_only_proxy/frontend/templates/messages.html @@ -0,0 +1,120 @@ +{% extends 'base.html' %} +{% block content %} +

Message History - {{client.proposal_no}} / {{client.stream}}

+ +
+ +
+ + + + + + + + + + + + + {% for message in messages %} + + + + + + + + {% endfor %} + +
+ + TopicIDContent
+ + {{ message.topic }}{{ message.id }} + + + + +
+ + +
+ +
+ + +
+
+ + + +{% endblock %} From 4fab6b342d214b7d082aba98daf3e54135fd6212 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:13:56 +0100 Subject: [PATCH 02/61] refactor: split models out into module --- src/zulip_write_only_proxy/models/__init__.py | 12 ++++++ .../{models.py => models/client.py} | 38 ++++++++++++++----- src/zulip_write_only_proxy/models/zulip.py | 34 +++++++++++++++++ 3 files changed, 75 insertions(+), 9 deletions(-) create mode 100644 src/zulip_write_only_proxy/models/__init__.py rename src/zulip_write_only_proxy/{models.py => models/client.py} (73%) create mode 100644 src/zulip_write_only_proxy/models/zulip.py diff --git a/src/zulip_write_only_proxy/models/__init__.py b/src/zulip_write_only_proxy/models/__init__.py new file mode 100644 index 00000000..8759ccb1 --- /dev/null +++ b/src/zulip_write_only_proxy/models/__init__.py @@ -0,0 +1,12 @@ +from .client import ScopedClient, ScopedClientCreate, ScopedClientWithKey +from .zulip import BotConfig, Message, Messages, PropagateMode + +__all__ = [ + "ScopedClient", + "ScopedClientCreate", + "ScopedClientWithKey", + "BotConfig", + "Message", + "Messages", + "PropagateMode", +] diff --git a/src/zulip_write_only_proxy/models.py b/src/zulip_write_only_proxy/models/client.py similarity index 73% rename from src/zulip_write_only_proxy/models.py rename to src/zulip_write_only_proxy/models/client.py index 3ed12fc4..41b92b1f 100644 --- a/src/zulip_write_only_proxy/models.py +++ b/src/zulip_write_only_proxy/models/client.py @@ -1,22 +1,22 @@ import datetime -import enum import secrets from typing import IO, TYPE_CHECKING, Any -from pydantic import BaseModel, Field, PrivateAttr, SecretStr, field_validator +from pydantic import ( + BaseModel, + Field, + PrivateAttr, + SecretStr, + field_validator, +) -from zulip_write_only_proxy import logger +from .. import logger +from .zulip import PropagateMode if TYPE_CHECKING: import zulip -class PropagateMode(str, enum.Enum): - change_one = "change_one" - change_all = "change_all" - change_later = "change_later" - - class ScopedClientCreate(BaseModel): proposal_no: int stream: str | None = None @@ -30,6 +30,7 @@ class ScopedClient(BaseModel): proposal_no: int stream: str # type: ignore [reportIncompatibleVariableOverride] bot_name: str + bot_id: int key: SecretStr = Field(default_factory=lambda: SecretStr(secrets.token_urlsafe())) created_at: datetime.datetime = Field(default_factory=datetime.datetime.now) @@ -82,10 +83,29 @@ def update_message( return self._client.update_message(request) + def get_messages(self): + request = { + "anchor": "newest", + "num_before": 100, + "num_after": 0, + "apply_markdown": "false", + "narrow": [ + {"operator": "sender", "operand": self.bot_id}, + {"operator": "stream", "operand": self.stream}, + ], + } + # result should be success, if found oldest and found newest both true no more + # messages to fetch + return self._client.get_messages(request) + + def get_me(self): + return self._client.get_profile() + class ScopedClientWithKey(ScopedClient): key: str # type: ignore[assignment] @field_validator("key") + @classmethod def _set_key(cls, v: str | SecretStr) -> str: return v.get_secret_value() if isinstance(v, SecretStr) else v diff --git a/src/zulip_write_only_proxy/models/zulip.py b/src/zulip_write_only_proxy/models/zulip.py new file mode 100644 index 00000000..bf78445c --- /dev/null +++ b/src/zulip_write_only_proxy/models/zulip.py @@ -0,0 +1,34 @@ +import enum +from typing import Annotated + +from pydantic import BaseModel, EmailStr, Field, HttpUrl, SecretStr + + +class PropagateMode(str, enum.Enum): + change_one = "change_one" + change_all = "change_all" + change_later = "change_later" + + +class BotConfig(BaseModel): + name: str + email: EmailStr + api_key: SecretStr + site: Annotated[HttpUrl, Field(default="https://mylog.connect.xfel.eu/")] + id: int + + +MessageID = int + + +class Message(BaseModel): + topic: str + id: MessageID + content: str + + +class Messages(BaseModel): + found_newest: bool + found_oldest: bool + messages: list[Message] + client: str From 7c8997574cbdef31193f817c737dbf5252290465 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:15:46 +0100 Subject: [PATCH 03/61] feat(repositories): generalise repos, make async --- .gitignore | 1 + .../templates/fragments/create-success.html | 4 +- src/zulip_write_only_proxy/repositories.py | 85 +++++++++-------- src/zulip_write_only_proxy/routers/api.py | 6 +- .../routers/frontend.py | 16 ++-- src/zulip_write_only_proxy/services.py | 95 +++++++++++++------ src/zulip_write_only_proxy/settings.py | 8 +- 7 files changed, 129 insertions(+), 86 deletions(-) diff --git a/.gitignore b/.gitignore index 84b502e1..18edaf28 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ *.zuliprc .env clients.json +zuliprc.json node_modules src/zulip_write_only_proxy/_version.py src/zulip_write_only_proxy/frontend/static diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/create-success.html b/src/zulip_write_only_proxy/frontend/templates/fragments/create-success.html index c7921bce..d1923d03 100644 --- a/src/zulip_write_only_proxy/frontend/templates/fragments/create-success.html +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/create-success.html @@ -14,8 +14,8 @@

Bot Created

{{ client.bot_name }}
- Bot URL: - {{ bot_url }} + Bot Site: + {{ bot_site }}
Key: diff --git a/src/zulip_write_only_proxy/repositories.py b/src/zulip_write_only_proxy/repositories.py index 9a79c4e4..3dff7f6d 100644 --- a/src/zulip_write_only_proxy/repositories.py +++ b/src/zulip_write_only_proxy/repositories.py @@ -1,55 +1,62 @@ -import threading +import asyncio +from dataclasses import dataclass, field +from pathlib import Path +from typing import Generic, TypeVar import orjson -import zulip -from pydantic import BaseModel, DirectoryPath, FilePath, SecretStr, validate_call +import pydantic +from anyio import Path as APath +from pydantic import BaseModel -from . import models +T = TypeVar("T", bound=BaseModel) -file_lock = threading.Lock() +@dataclass +class BaseRepository(Generic[T]): + file: Path + index: str + model: T -class ZuliprcRepository(BaseModel): - directory: DirectoryPath + lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False, repr=False) + data: dict[str, T] = field(default_factory=dict, init=False, repr=False) - def get(self, key: str) -> zulip.Client: - return zulip.Client(config_file=str(self.directory / f"{key}.zuliprc")) + @staticmethod + def _serialize_pydantic(obj): + if type(obj) is pydantic.AnyUrl: + return str(obj) + if type(obj) is pydantic.SecretStr: + return obj.get_secret_value() + raise TypeError - @validate_call - def put(self, name: str, email: str, key: str, site: str) -> zulip.Client: - (self.directory / f"{name}.zuliprc").write_text( - f"""[api] -email={email} -key={key} -site={site} -""" - ) - return zulip.Client(config_file=str(self.directory / f"{name}.zuliprc")) + async def load(self): + if not await APath(self.file).exists(): + return - def list(self): - return [p.stem for p in self.directory.iterdir() if p.suffix == ".zuliprc"] + self.data = orjson.loads(await APath(self.file).read_bytes()) + async def write(self): + async with self.lock: + await APath(self.file).write_bytes( + orjson.dumps( + self.data, + option=orjson.OPT_INDENT_2, + default=self._serialize_pydantic, + ) + ) -class ClientRepository(BaseModel): - """A basic file/JSON-based repository for storing client entries.""" + async def get(self, key: str) -> T: + return self.model.model_validate(self.data.get(key)) - path: FilePath + async def insert(self, item: T): + _item = item.model_dump() + key = _item[self.index] - def get(self, key: str) -> models.ScopedClient: - data = orjson.loads(self.path.read_bytes()) - client_data = data[key] + if type(key) is pydantic.SecretStr: + key = key.get_secret_value() - return models.ScopedClient(key=SecretStr(key), **client_data) + self.data[key] = _item - def put(self, client: models.ScopedClient) -> None: - with file_lock: - data: dict[str, dict] = orjson.loads(self.path.read_bytes()) - data[client.key.get_secret_value()] = client.model_dump(exclude={"key"}) - self.path.write_bytes(orjson.dumps(data, option=orjson.OPT_INDENT_2)) + await self.write() - def list(self) -> list[models.ScopedClientWithKey]: - data = orjson.loads(self.path.read_bytes()) - - return [ - models.ScopedClientWithKey(key=key, **value) for key, value in data.items() - ] + async def list(self) -> list[T]: + return [self.model.model_validate(item) for item in self.data.values()] diff --git a/src/zulip_write_only_proxy/routers/api.py b/src/zulip_write_only_proxy/routers/api.py index c3dd06d4..91221a0c 100644 --- a/src/zulip_write_only_proxy/routers/api.py +++ b/src/zulip_write_only_proxy/routers/api.py @@ -15,14 +15,14 @@ api_key_header = APIKeyHeader(name="X-API-key", auto_error=False) -def get_client( +async def get_client( key: Annotated[str, fastapi.Security(api_key_header)] ) -> models.ScopedClient: if key is None: raise fastapi.HTTPException(status_code=403, detail="Not authenticated") try: - return services.get_client(key) + return await services.get_client(key) except KeyError as e: raise fastapi.HTTPException( status_code=401, detail="Unauthorised", headers={"HX-Location": "/"} @@ -66,7 +66,7 @@ def update_message( content: Annotated[str | None, fastapi.Body(media_type="text/plain")] = None, topic: Annotated[str | None, fastapi.Query()] = None, ): - if not (content or topic): + if not (content or topic): # sourcery skip raise fastapi.HTTPException( status_code=400, detail=( diff --git a/src/zulip_write_only_proxy/routers/frontend.py b/src/zulip_write_only_proxy/routers/frontend.py index b7eb1093..827c0956 100644 --- a/src/zulip_write_only_proxy/routers/frontend.py +++ b/src/zulip_write_only_proxy/routers/frontend.py @@ -52,6 +52,8 @@ async def check_auth(request: Request): detail=f"Forbidden - `{user.get('preferred_username')}` not allowed access", ) + logger.debug("Authenticated", user=user) + async def auth_redirect(request: Request, exc: AuthException): logger.info("Redirecting to login", status_code=exc.status_code, detail=exc.detail) @@ -73,13 +75,13 @@ async def auth_redirect(request: Request, exc: AuthException): @router.get("/") -def root(request: Request): - return client_list(request) +async def root(request: Request): + return await client_list(request) @router.get("/client/list") -def client_list(request: Request): - clients = services.list_clients() +async def client_list(request: Request): + clients = await services.list_clients() clients.reverse() return TEMPLATES.TemplateResponse( "list.html", @@ -93,7 +95,7 @@ def client_list(request: Request): @router.get("/client/create") -def client_create(request: Request): +async def client_create(request: Request): schema = models.ScopedClientCreate.model_json_schema() optional = schema["properties"] required = {field: optional.pop(field) for field in schema["required"]} @@ -113,13 +115,13 @@ async def client_create_post(request: Request): ) dump = client.model_dump() dump["key"] = client.key.get_secret_value() - bot = services.get_bot(client.bot_name) + bot = await services.get_bot(client.bot_name) return TEMPLATES.TemplateResponse( "fragments/create-success.html", { "request": request, "client": models.ScopedClientWithKey(**dump), - "bot_url": bot.base_url, + "bot_site": bot.site, }, ) except Exception as e: diff --git a/src/zulip_write_only_proxy/services.py b/src/zulip_write_only_proxy/services.py index 4d233100..307a5378 100644 --- a/src/zulip_write_only_proxy/services.py +++ b/src/zulip_write_only_proxy/services.py @@ -1,27 +1,41 @@ +import asyncio from typing import TYPE_CHECKING, Annotated import fastapi +import zulip from . import logger, models, mymdc, repositories if TYPE_CHECKING: from .settings import Settings -CLIENT_REPO: repositories.ClientRepository = None # type: ignore[assignment] -ZULIPRC_REPO: repositories.ZuliprcRepository = None # type: ignore[assignment] +CLIENT_REPO: repositories.BaseRepository = None # type: ignore[assignment] +ZULIPRC_REPO: repositories.BaseRepository = None # type: ignore[assignment] def configure(settings: "Settings", _: fastapi.FastAPI): """Set up the repositories for the services. This should be called before any of the other functions in this module.""" global CLIENT_REPO, ZULIPRC_REPO + + ZULIPRC_REPO = repositories.BaseRepository( + file=settings.config_dir / "zuliprc.json", + index="name", + model=models.BotConfig, + ) + + CLIENT_REPO = repositories.BaseRepository( + file=settings.config_dir / "clients.json", + index="key", + model=models.ScopedClient, + ) + logger.info( - "Setting up repositories", - client_repo=settings.clients.path, - zuliprc_repo=settings.zuliprcs.directory, + "Setting up repositories", client_repo=CLIENT_REPO, zuliprc_repo=ZULIPRC_REPO ) - CLIENT_REPO = repositories.ClientRepository(path=settings.clients.path) - ZULIPRC_REPO = repositories.ZuliprcRepository(directory=settings.zuliprcs.directory) + + asyncio.create_task(CLIENT_REPO.load()) # noqa: RUF006 + asyncio.create_task(ZULIPRC_REPO.load()) # noqa: RUF006 async def create_client( @@ -36,56 +50,77 @@ async def create_client( ) logger.debug("Stream name from MyMdC", stream=new_client.stream) - if new_client.bot_name is None: - new_client.bot_name = str(new_client.proposal_no) - logger.debug("Bot name from proposal number", bot_name=new_client.bot_name) + name = new_client.bot_name or new_client.proposal_no + key, email, site = (new_client.bot_key, new_client.bot_email, new_client.bot_site) - bot_name = new_client.bot_name - key, email, site = new_client.bot_key, new_client.bot_email, new_client.bot_site - - if bot_name not in ZULIPRC_REPO.list(): + if name not in await ZULIPRC_REPO.list(): logger.debug("Bot zuliprc not present") if not key or not email: - key, email = await mymdc.CLIENT.get_zulip_bot_credentials( + _id, key, email = await mymdc.CLIENT.get_zulip_bot_credentials( new_client.proposal_no ) - logger.debug("Bot credentials from MyMdC", bot_email=email, bot_key=key) + logger.debug( + "Bot credentials from MyMdC", + bot_name=name, + bot_email=email, + bot_key=key, + bot_id=_id, + ) if not key or not email: raise fastapi.HTTPException( status_code=422, detail=( - f"bot '{bot_name}' does not exist, and a bot could not " + f"bot '{name}' does not exist, and a bot could not " f"be found for proposal '{new_client.proposal_no}' via MyMdC. To " "add a client with a new bot provide both bot_email bot_key." ), ) - ZULIPRC_REPO.put(bot_name, email, key, site) + bot = models.BotConfig( + name=str(name), + id=_id, + api_key=key, # type: ignore[arg-type] + email=email, + site=site, # type: ignore[arg-type] + ) - _ = new_client.model_dump() - _["created_by"] = created_by - client = models.ScopedClient.model_validate(_) + await ZULIPRC_REPO.insert(bot) + else: + bot = await ZULIPRC_REPO.get(str(name)) + + client = models.ScopedClient.model_validate( + { + **new_client.model_dump(), + "created_by": created_by, + "bot_id": bot.id, + "bot_name": bot.name, + } + ) - CLIENT_REPO.put(client) + await CLIENT_REPO.insert(client) logger.info("Created client", client=client) return client -def get_client(key: str) -> models.ScopedClient: - client = CLIENT_REPO.get(key) +async def get_client(key: str) -> models.ScopedClient: + client = await CLIENT_REPO.get(key) + bot_config = await ZULIPRC_REPO.get(client.bot_name) - if isinstance(client, models.ScopedClient): - client._client = ZULIPRC_REPO.get(client.bot_name) + client._client = zulip.Client( + email=bot_config.email, + api_key=bot_config.api_key.get_secret_value(), + site=str(bot_config.site), + ) return client -def get_bot(bot_name: str): - return ZULIPRC_REPO.get(bot_name) +async def get_bot(bot_name: str) -> models.BotConfig: + return await ZULIPRC_REPO.get(bot_name) -def list_clients() -> list[models.ScopedClientWithKey]: - return CLIENT_REPO.list() +async def list_clients() -> list[models.ScopedClientWithKey]: + return await CLIENT_REPO.list() diff --git a/src/zulip_write_only_proxy/settings.py b/src/zulip_write_only_proxy/settings.py index 7f4be726..bf0198f1 100644 --- a/src/zulip_write_only_proxy/settings.py +++ b/src/zulip_write_only_proxy/settings.py @@ -4,8 +4,6 @@ from pydantic import AnyUrl, DirectoryPath, HttpUrl, SecretStr from pydantic_settings import BaseSettings, SettingsConfigDict -from .repositories import ClientRepository, ZuliprcRepository - class Auth(BaseSettings): client_id: str @@ -27,7 +25,9 @@ class MyMdCCredentials(BaseSettings): token_url: HttpUrl _access_token: str = "" - _expires_at: dt.datetime = dt.datetime.fromisocalendar(1970, 1, 1) + _expires_at: dt.datetime = dt.datetime.fromisocalendar(1970, 1, 1).astimezone( + dt.timezone.utc + ) class Settings(BaseSettings): @@ -40,8 +40,6 @@ class Settings(BaseSettings): auth: Auth mymdc: MyMdCCredentials - clients: ClientRepository - zuliprcs: ZuliprcRepository model_config = SettingsConfigDict( env_prefix="ZWOP_", env_file=[".env"], env_nested_delimiter="__" From 10406bae6d95e5a9b2a990e670d108218bd397fb Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:17:12 +0100 Subject: [PATCH 04/61] feat(mymdc): return bot id with credentials --- src/zulip_write_only_proxy/mymdc.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/zulip_write_only_proxy/mymdc.py b/src/zulip_write_only_proxy/mymdc.py index c9af65ae..2a5e86f0 100644 --- a/src/zulip_write_only_proxy/mymdc.py +++ b/src/zulip_write_only_proxy/mymdc.py @@ -138,12 +138,10 @@ async def get_zulip_stream_name(self, proposal_no: int) -> str: return res - async def get_zulip_bot_credentials( - self, proposal_no: int - ) -> tuple[str | None, str | None]: + async def get_zulip_bot_credentials(self, proposal_no: int) -> tuple[int, str, str]: res = (await self.get(f"/api/proposals/{proposal_no}/logbook_bot")).json() if res is None: raise NoStreamForProposalError(proposal_no) - return res.get("bot_key", None), res.get("bot_email", None) + return res.get("bot_id"), res.get("bot_key"), res.get("bot_email") From e4d1c97f194baa001d255c63be92aa79ea609978 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:20:53 +0100 Subject: [PATCH 05/61] feat(frontend): set htmx/css path in Jinja env --- src/zulip_write_only_proxy/frontend/templates/base.html | 4 ++-- src/zulip_write_only_proxy/routers/frontend.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/zulip_write_only_proxy/frontend/templates/base.html b/src/zulip_write_only_proxy/frontend/templates/base.html index 21ebe86d..83ea17d3 100644 --- a/src/zulip_write_only_proxy/frontend/templates/base.html +++ b/src/zulip_write_only_proxy/frontend/templates/base.html @@ -5,11 +5,11 @@ - + diff --git a/src/zulip_write_only_proxy/routers/frontend.py b/src/zulip_write_only_proxy/routers/frontend.py index 827c0956..01c9b5aa 100644 --- a/src/zulip_write_only_proxy/routers/frontend.py +++ b/src/zulip_write_only_proxy/routers/frontend.py @@ -29,6 +29,13 @@ def configure(_: "Settings", app: fastapi.FastAPI): ) TEMPLATES = Jinja2Templates(directory=templates_dir) + TEMPLATES.env.globals["static_main_css"] = app.url_path_for( + "static", path="main.css" + ) + TEMPLATES.env.globals["static_htmx"] = app.url_path_for( + "static", path="htmx.min.js" + ) + class AuthException(exceptions.ZwopException): pass From f4702e5a4eb93518df3f6cd7d264570855fddb9f Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:22:07 +0100 Subject: [PATCH 06/61] feat(frontend): add client/messages route --- .../routers/frontend.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/zulip_write_only_proxy/routers/frontend.py b/src/zulip_write_only_proxy/routers/frontend.py index 01c9b5aa..40fde2bb 100644 --- a/src/zulip_write_only_proxy/routers/frontend.py +++ b/src/zulip_write_only_proxy/routers/frontend.py @@ -136,3 +136,29 @@ async def client_create_post(request: Request): "fragments/alert-error.html", {"request": request, "message": e.__repr__()}, ) + + +@router.get("/client/messages") +async def client_messages(request: Request): + client_key = request.headers.get("X-API-Key") + if not client_key: + raise exceptions.ZwopException( + status_code=400, + detail="Bad Request - missing X-API-Key header", + ) + client = await services.get_client(client_key) + _messages = client.get_messages() + logger.debug("Messages", messages=_messages) + messages = [ + models.Message(topic=m["subject"], id=m["id"], content=m["content"]) + for m in _messages["messages"] + ] + return TEMPLATES.TemplateResponse( + "messages.html", + {"request": request, "messages": messages, "client": client}, + headers={ + "HX-Retarget": "#content", + "HX-Reselect": "#content", + "HX-Swap": "outerHTML", + }, + ) From 757ea5d326b8cc91b4682db8f5fc4286495f8091 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 09:27:44 +0100 Subject: [PATCH 07/61] feat(frontend): make navbar sticky --- .../frontend/templates/fragments/navbar.html | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html index 4f1fe97a..d6bcbdc7 100644 --- a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html @@ -1,4 +1,8 @@ -
+ -
+
Manually configure
@@ -44,7 +44,7 @@
diff --git a/src/zulip_write_only_proxy/frontend/templates/list.html b/src/zulip_write_only_proxy/frontend/templates/list.html index ed1feba1..65860217 100644 --- a/src/zulip_write_only_proxy/frontend/templates/list.html +++ b/src/zulip_write_only_proxy/frontend/templates/list.html @@ -71,7 +71,7 @@ function viewMessages(element) { let key = element.parentElement.parentElement.getAttribute("key"); - htmx.ajax('GET', '/client/messages', {headers: {'X-API-key': key}}) + htmx.ajax("GET", "/client/messages", { headers: { "X-API-key": key } }); } {% endblock %} diff --git a/src/zulip_write_only_proxy/frontend/templates/messages.html b/src/zulip_write_only_proxy/frontend/templates/messages.html index 360e3cf7..cb491df5 100644 --- a/src/zulip_write_only_proxy/frontend/templates/messages.html +++ b/src/zulip_write_only_proxy/frontend/templates/messages.html @@ -1,29 +1,35 @@ {% block disabled %} -{# NB: button for deleting message removed as message deletion not allowed on elog zulip#} - + {# NB: button for deleting message removed as message deletion not allowed on elog zulip#} + {% endblock %} - {% extends 'base.html' %} {% block content %} -

Message History - {{client.proposal_no}} / {{client.stream}}

+

+ Message History - {{ client.proposal_no }} / {{ client.stream }} +

- +
@@ -51,8 +57,13 @@ + + + + + +{% endfor %} diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html index 1f57109e..a7d969eb 100644 --- a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html @@ -2,23 +2,13 @@ id="navbar" class="navbar sticky top-0 z-30 w-full justify-center bg-opacity-90 backdrop-blur" hx-boost="true" + hx-push-url="false" + hx-target="#content" >
- ZWOP + ZWOP
-
+
  • List diff --git a/src/zulip_write_only_proxy/frontend/templates/list.html b/src/zulip_write_only_proxy/frontend/templates/list.html index 65860217..035889ff 100644 --- a/src/zulip_write_only_proxy/frontend/templates/list.html +++ b/src/zulip_write_only_proxy/frontend/templates/list.html @@ -1,6 +1,11 @@ {% extends 'base.html' %} {% block content %} -
{{ message.topic }} {{ message.id }} - + + @@ -97,7 +110,8 @@ function showModal(button) { modalRow = button.parentNode.parentNode; modalMessageID = modalRow.getAttribute("message_id"); - document.getElementById("editText").value = modalRow.getElementsByTagName("textarea")[0].value + document.getElementById("editText").value = + modalRow.getElementsByTagName("textarea")[0].value; document.getElementById("editModal").showModal(); } @@ -105,39 +119,49 @@ document.getElementById("editModal").close(); } - async function updateMessage(button){ - await fetch('/api/update_message?' + new URLSearchParams({message_id: modalMessageID}), { - method: 'PATCH', - body: document.getElementById("editText").value, - headers: { - 'Content-Type': 'text/plain', - 'X-API-key': '{{ client.key.get_secret_value() }}', + async function updateMessage(button) { + await fetch( + "/api/update_message?" + + new URLSearchParams({ message_id: modalMessageID }), + { + method: "PATCH", + body: document.getElementById("editText").value, + headers: { + "Content-Type": "text/plain", + "X-API-key": "{{ client.key.get_secret_value() }}", + }, }, - }).then(response => { + ).then((response) => { if (response.ok) { - modalRow.getElementsByTagName("textarea")[0].value = document.getElementById("editText").value; + modalRow.getElementsByTagName("textarea")[0].value = + document.getElementById("editText").value; closeModal(); } else { - alert('Error updating message: ' + response.status + ' ' + response.statusText); + alert( + "Error updating message: " + + response.status + + " " + + response.statusText, + ); } }); } - function filterTable() { - // Declare variables - var input, filter, table, tr, td, i, txtValue; - input = document.getElementById("filterText"); - filter = input.value.toUpperCase(); - table = document.getElementById("table"); - tr = table.getElementsByTagName("tr"); - for (i = 1; i < tr.length; i++) { - txtValue = tr[i].textContent || tr[i].innerText; - if (txtValue.toUpperCase().indexOf(filter) > -1) { - tr[i].classList.remove("hidden"); - } else { - tr[i].classList.add("hidden"); + function filterTable() { + // Declare variables + var input, filter, table, tr, td, i, txtValue; + input = document.getElementById("filterText"); + filter = input.value.toUpperCase(); + table = document.getElementById("table"); + tr = table.getElementsByTagName("tr"); + for (i = 1; i < tr.length; i++) { + txtValue = tr[i].textContent || tr[i].innerText; + if (txtValue.toUpperCase().indexOf(filter) > -1) { + tr[i].classList.remove("hidden"); + } else { + tr[i].classList.add("hidden"); + } } } - } {% endblock %} diff --git a/src/zulip_write_only_proxy/models/__init__.py b/src/zulip_write_only_proxy/models/__init__.py index 8759ccb1..7a0ef4b2 100644 --- a/src/zulip_write_only_proxy/models/__init__.py +++ b/src/zulip_write_only_proxy/models/__init__.py @@ -2,11 +2,11 @@ from .zulip import BotConfig, Message, Messages, PropagateMode __all__ = [ - "ScopedClient", - "ScopedClientCreate", - "ScopedClientWithKey", "BotConfig", "Message", "Messages", "PropagateMode", + "ScopedClient", + "ScopedClientCreate", + "ScopedClientWithKey", ] diff --git a/src/zulip_write_only_proxy/routers/api.py b/src/zulip_write_only_proxy/routers/api.py index 01a9b260..c2c69f2f 100644 --- a/src/zulip_write_only_proxy/routers/api.py +++ b/src/zulip_write_only_proxy/routers/api.py @@ -16,7 +16,7 @@ async def get_client( - key: Annotated[str, fastapi.Security(api_key_header)] + key: Annotated[str, fastapi.Security(api_key_header)], ) -> models.ScopedClient: return await services.get_client(key) @@ -101,7 +101,7 @@ def get_stream_topics( @router.get("/me", response_model_exclude={"key"}) def get_me( - client: Annotated[models.ScopedClient, fastapi.Depends(get_client)] + client: Annotated[models.ScopedClient, fastapi.Depends(get_client)], ) -> models.ScopedClient: return client diff --git a/src/zulip_write_only_proxy/routers/frontend.py b/src/zulip_write_only_proxy/routers/frontend.py index 40fde2bb..eff8efc2 100644 --- a/src/zulip_write_only_proxy/routers/frontend.py +++ b/src/zulip_write_only_proxy/routers/frontend.py @@ -148,7 +148,6 @@ async def client_messages(request: Request): ) client = await services.get_client(client_key) _messages = client.get_messages() - logger.debug("Messages", messages=_messages) messages = [ models.Message(topic=m["subject"], id=m["id"], content=m["content"]) for m in _messages["messages"] diff --git a/src/zulip_write_only_proxy/services.py b/src/zulip_write_only_proxy/services.py index cbad5691..5a6c01cf 100644 --- a/src/zulip_write_only_proxy/services.py +++ b/src/zulip_write_only_proxy/services.py @@ -101,14 +101,12 @@ async def create_client( else: bot = await ZULIPRC_REPO.get(str(name)) - client = models.ScopedClient.model_validate( - { - **new_client.model_dump(), - "created_by": created_by, - "bot_id": bot.id, - "bot_name": bot.name, - } - ) + client = models.ScopedClient.model_validate({ + **new_client.model_dump(), + "created_by": created_by, + "bot_id": bot.id, + "bot_name": bot.name, + }) await CLIENT_REPO.insert(client) From 63d8d439d19290b03c04e4a3e6ad3d2645232bdb Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 12:25:24 +0100 Subject: [PATCH 22/61] style: typing fixes --- src/zulip_write_only_proxy/main.py | 2 +- src/zulip_write_only_proxy/repositories.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zulip_write_only_proxy/main.py b/src/zulip_write_only_proxy/main.py index 96886559..3d2aa8b3 100644 --- a/src/zulip_write_only_proxy/main.py +++ b/src/zulip_write_only_proxy/main.py @@ -99,4 +99,4 @@ def get_trusted_hosts(logger): logger.info("Starting uvicorn", **args) - uvicorn.run(**args) + uvicorn.run(**args) # type: ignore[arg-type] diff --git a/src/zulip_write_only_proxy/repositories.py b/src/zulip_write_only_proxy/repositories.py index 1b5e48a7..fb20e42b 100644 --- a/src/zulip_write_only_proxy/repositories.py +++ b/src/zulip_write_only_proxy/repositories.py @@ -1,7 +1,7 @@ import asyncio from dataclasses import dataclass, field from pathlib import Path -from typing import Generic, TypeVar +from typing import Any, Generic, TypeVar import orjson import pydantic @@ -20,7 +20,7 @@ class BaseRepository(Generic[T]): model: T lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False, repr=False) - data: dict[str, T] = field(default_factory=dict, init=False, repr=False) + data: dict[str, Any] = field(default_factory=dict, init=False, repr=False) @staticmethod def _serialize_pydantic(obj): @@ -62,7 +62,7 @@ async def insert(self, item: T): if type(key) is pydantic.SecretStr: key = key.get_secret_value() - self.data[key] = _item + self.data[str(key)] = _item await self.write() From 41db021aafc1a27adedfb634444cac40edbb088e Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:13:00 +0100 Subject: [PATCH 23/61] feat(frontend): allow for true partial loading of pages --- .../frontend/templates/base.html | 67 ++++++++++++------- .../templates/fragments/list-table-rows.html | 45 +++++++++++++ .../frontend/templates/fragments/navbar.html | 18 ++--- .../frontend/templates/list.html | 56 ++-------------- .../routers/frontend.py | 25 ++++--- 5 files changed, 114 insertions(+), 97 deletions(-) create mode 100644 src/zulip_write_only_proxy/frontend/templates/fragments/list-table-rows.html diff --git a/src/zulip_write_only_proxy/frontend/templates/base.html b/src/zulip_write_only_proxy/frontend/templates/base.html index 61e31e7e..a58e8923 100644 --- a/src/zulip_write_only_proxy/frontend/templates/base.html +++ b/src/zulip_write_only_proxy/frontend/templates/base.html @@ -1,26 +1,43 @@ - - - - ZWOP - - - - - - +{% if not request.headers.get("HX-request") %} + {# Return the whole page on normal requests #} + + + + ZWOP + + + + + + - - {% block navbar %}{% include 'fragments/navbar.html' %}{% endblock %} -
- {% block content %}{% endblock %} -
- - - + + {% block navbar %} + {% include 'fragments/navbar.html' %} + {% endblock %} +
+ {% block content %} + {% endblock %} +
+ + + +{% else %} + {# Return only the subpage content on HTMX requests #} +
+ {{ self.content() }} +
+{% endif %} diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/list-table-rows.html b/src/zulip_write_only_proxy/frontend/templates/fragments/list-table-rows.html new file mode 100644 index 00000000..eca4b484 --- /dev/null +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/list-table-rows.html @@ -0,0 +1,45 @@ +{% for client in clients %} +
{{ client.proposal_no }}{{ client.stream }}{{ client.bot_name }} + + +
+
@@ -9,55 +14,8 @@ - - {% for client in clients %} - - - - - - - {% endfor %} - +
Proposal
{{ client.proposal_no }}{{ client.stream }}{{ client.bot_name }} - - -
- diff --git a/src/zulip_write_only_proxy/frontend/templates/list.html b/src/zulip_write_only_proxy/frontend/templates/list.html index 035889ff..7cf574d0 100644 --- a/src/zulip_write_only_proxy/frontend/templates/list.html +++ b/src/zulip_write_only_proxy/frontend/templates/list.html @@ -1,35 +1,28 @@ {% extends 'base.html' %} {% block content %} - - - - + + + + - +
ProposalStreamNameProposalStreamNameActions
{% endblock %} diff --git a/src/zulip_write_only_proxy/routers/frontend.py b/src/zulip_write_only_proxy/routers/frontend.py index 755c7f5b..57236edc 100644 --- a/src/zulip_write_only_proxy/routers/frontend.py +++ b/src/zulip_write_only_proxy/routers/frontend.py @@ -97,15 +97,20 @@ async def root(request: Request): @router.get("/client/list") async def client_list(request: Request): - if request.headers.get("HX-Target") == "rows": + if request.headers.get("HX-Current-URL", "").endswith("/client/list"): clients = await services.list_clients() clients.reverse() return TEMPLATES.TemplateResponse( "fragments/list-table-rows.html", {"request": request, "clients": clients}, + headers={"HX-Retarget": "#rows"}, ) - return TEMPLATES.TemplateResponse("list.html", {"request": request}) + return TEMPLATES.TemplateResponse( + "list.html", + {"request": request}, + headers={"HX-Replace-Url": str(request.url_for("client_list"))}, + ) @router.get("/client/create") From bc272c984252fbc0252d31cc4a9b6eba06d59e20 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:47:20 +0100 Subject: [PATCH 29/61] feat(frontend): split out message table row to fragments, partial load via htmx --- .../fragments/list-messages-rows.html | 73 +++++++++++++++++++ .../frontend/templates/messages.html | 73 +------------------ .../routers/frontend.py | 37 +++++----- 3 files changed, 95 insertions(+), 88 deletions(-) create mode 100644 src/zulip_write_only_proxy/frontend/templates/fragments/list-messages-rows.html diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/list-messages-rows.html b/src/zulip_write_only_proxy/frontend/templates/fragments/list-messages-rows.html new file mode 100644 index 00000000..6f9e40da --- /dev/null +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/list-messages-rows.html @@ -0,0 +1,73 @@ +{% for message in messages %} + + {{ message.topic }} + {{ message.id }} + + + + + + + +{% endfor %} + + diff --git a/src/zulip_write_only_proxy/frontend/templates/messages.html b/src/zulip_write_only_proxy/frontend/templates/messages.html index cb491df5..99b40cc3 100644 --- a/src/zulip_write_only_proxy/frontend/templates/messages.html +++ b/src/zulip_write_only_proxy/frontend/templates/messages.html @@ -46,49 +46,7 @@ - - {% for message in messages %} - - - - - {{ message.topic }} - {{ message.id }} - - - - - - - - {% endfor %} - + @@ -96,6 +54,7 @@
@@ -119,34 +78,6 @@ document.getElementById("editModal").close(); } - async function updateMessage(button) { - await fetch( - "/api/update_message?" + - new URLSearchParams({ message_id: modalMessageID }), - { - method: "PATCH", - body: document.getElementById("editText").value, - headers: { - "Content-Type": "text/plain", - "X-API-key": "{{ client.key.get_secret_value() }}", - }, - }, - ).then((response) => { - if (response.ok) { - modalRow.getElementsByTagName("textarea")[0].value = - document.getElementById("editText").value; - closeModal(); - } else { - alert( - "Error updating message: " + - response.status + - " " + - response.statusText, - ); - } - }); - } - function filterTable() { // Declare variables var input, filter, table, tr, td, i, txtValue; diff --git a/src/zulip_write_only_proxy/routers/frontend.py b/src/zulip_write_only_proxy/routers/frontend.py index 57236edc..39d04386 100644 --- a/src/zulip_write_only_proxy/routers/frontend.py +++ b/src/zulip_write_only_proxy/routers/frontend.py @@ -152,24 +152,27 @@ async def client_create_post(request: Request): @router.get("/client/messages") async def client_messages(request: Request): - client_key = request.headers.get("X-API-Key") - if not client_key: - raise exceptions.ZwopException( - status_code=400, - detail="Bad Request - missing X-API-Key header", + if request.headers.get("HX-Current-URL", "").endswith("/client/messages"): + client_key = request.headers.get("X-API-Key") + if not client_key: + raise exceptions.ZwopException( + status_code=400, + detail="Bad Request - missing X-API-Key header", + ) + client = await services.get_client(client_key) + _messages = client.get_messages() + messages = [ + models.Message(topic=m["subject"], id=m["id"], content=m["content"]) + for m in _messages["messages"] + ] + return TEMPLATES.TemplateResponse( + "fragments/list-messages-rows.html", + {"request": request, "messages": messages, "client": client}, + headers={"HX-Retarget": "#rows"}, ) - client = await services.get_client(client_key) - _messages = client.get_messages() - messages = [ - models.Message(topic=m["subject"], id=m["id"], content=m["content"]) - for m in _messages["messages"] - ] + return TEMPLATES.TemplateResponse( "messages.html", - {"request": request, "messages": messages, "client": client}, - headers={ - "HX-Retarget": "#content", - "HX-Reselect": "#content", - "HX-Swap": "outerHTML", - }, + {"request": request}, + headers={"HX-Retarget": "#content"}, ) From 216dbaf138d61ee4ee706d91c12c0280b1de04e2 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:49:32 +0100 Subject: [PATCH 30/61] feat(frontend): fade animations on component load, update navbar style --- .../frontend/templates/fragments/navbar.html | 40 ++++++++++++++----- .../frontend/templates/messages.html | 19 ++++----- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html index a7d969eb..db5c0825 100644 --- a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html @@ -8,17 +8,39 @@
ZWOP
-
+
+ List + Create +
-
+ diff --git a/src/zulip_write_only_proxy/frontend/templates/messages.html b/src/zulip_write_only_proxy/frontend/templates/messages.html index 99b40cc3..bc950a00 100644 --- a/src/zulip_write_only_proxy/frontend/templates/messages.html +++ b/src/zulip_write_only_proxy/frontend/templates/messages.html @@ -35,15 +35,10 @@ - - - - - + + + + @@ -94,5 +89,11 @@ } } } + + document.body.addEventListener("htmx:afterOnLoad", function (event) { + if (event.detail.target.id === "rows") { + event.detail.target.classList.remove("opacity-0"); + } + }); {% endblock %} From df2588fd1b2e0c687145b0cc878644bb8cc7f0ee Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Thu, 22 Feb 2024 10:52:29 +0100 Subject: [PATCH 31/61] feat(frontend): adjust htmx load style for better partial updates --- .../frontend/templates/fragments/navbar.html | 12 +++++----- .../frontend/templates/messages.html | 19 ++++++++++------ .../routers/frontend.py | 22 +++++-------------- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html index db5c0825..527dc271 100644 --- a/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html +++ b/src/zulip_write_only_proxy/frontend/templates/fragments/navbar.html @@ -2,7 +2,7 @@ id="navbar" class="navbar sticky top-0 z-30 w-full justify-center bg-opacity-90 backdrop-blur" hx-boost="true" - hx-push-url="false" + hx-push-url="true" hx-target="#content" >
@@ -12,8 +12,8 @@ List Create
-
    -
  • +
      +
    • -
      • -
    - +
    • +
+ diff --git a/src/zulip_write_only_proxy/frontend/templates/messages.html b/src/zulip_write_only_proxy/frontend/templates/messages.html index 2bc79aa0..47b0c574 100644 --- a/src/zulip_write_only_proxy/frontend/templates/messages.html +++ b/src/zulip_write_only_proxy/frontend/templates/messages.html @@ -66,6 +66,43 @@ var modalRow = null; var modalMessageID = null; + async function updateMessage(button) { + const currentText = modalRow.getElementsByTagName("textarea")[0].value; + const updatedText = document.getElementById("editText").value; + + if (currentText == updatedText) { + closeModal(); + } else { + document.getElementById("spinner").classList.remove("hidden"); + await fetch( + "{{ url_for('update_message') }}?" + + new URLSearchParams({ message_id: modalMessageID }), + { + method: "PATCH", + body: updatedText, + headers: { + "Content-Type": "text/plain", + "X-API-key": "{{ client.token.get_secret_value() }}", + }, + }, + ).then((response) => { + if (response.ok) { + modalRow.getElementsByTagName("textarea")[0].value = updatedText; + document.getElementById("spinner").classList.add("hidden"); + closeModal(); + } else { + alert( + "Error updating message: " + + response.status + + " " + + response.statusText, + ); + document.getElementById("spinner").classList.add("hidden"); + } + }); + } + } + function showModal(button) { modalRow = button.parentNode.parentNode; modalMessageID = modalRow.getAttribute("message_id"); From 059927e1d0d8276e18d119a709b7c4ac30ca4746 Mon Sep 17 00:00:00 2001 From: Robert Rosca <32569096+RobertRosca@users.noreply.github.com> Date: Mon, 11 Mar 2024 11:22:00 +0100 Subject: [PATCH 46/61] feat(frontend): add delete button to edit modal --- .../frontend/templates/messages.html | 38 +++++++++---------- .../routers/frontend.py | 19 +++++----- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/src/zulip_write_only_proxy/frontend/templates/messages.html b/src/zulip_write_only_proxy/frontend/templates/messages.html index 47b0c574..be480ae2 100644 --- a/src/zulip_write_only_proxy/frontend/templates/messages.html +++ b/src/zulip_write_only_proxy/frontend/templates/messages.html @@ -1,22 +1,3 @@ -{% if False %} - {# NB: button for deleting message removed as message deletion not allowed on elog zulip#} - -{% endif %} {% extends 'base.html' %} {% block content %} @@ -53,6 +34,25 @@
+
- - TopicIDContentTopicIDContent