From 75b35ac7e14d6e8617d6dc926448300f60c2cfde Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 30 Nov 2023 12:15:57 +0100 Subject: [PATCH 01/20] test --- etc/motley_cue.sudo | 1 + motley_cue/mapper/authorisation.py | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 etc/motley_cue.sudo diff --git a/etc/motley_cue.sudo b/etc/motley_cue.sudo new file mode 100644 index 0000000..f217bbb --- /dev/null +++ b/etc/motley_cue.sudo @@ -0,0 +1 @@ +motley_cue ALL=(root) NOPASSWD: /usr/sbin/useradd, /usr/sbin/userdel, /usr/sbin/groupadd, /usr/bin/chage, /usr/bin/pkill, /usr/sbin/usermod \ No newline at end of file diff --git a/motley_cue/mapper/authorisation.py b/motley_cue/mapper/authorisation.py index ccf6401..c56902c 100644 --- a/motley_cue/mapper/authorisation.py +++ b/motley_cue/mapper/authorisation.py @@ -206,6 +206,11 @@ def _check_request(user_infos: UserInfos, *_, **kwargs) -> CheckResult: if not op_authz.authorise_admins_for_all_ops and canonical_url( op_authz.op_url ) != canonical_url(user_iss): + logger.info( + "Admin from issuer %s is not authorised to manage users of issuer '%s'", + op_authz.op_url, + user_iss, + ) return CheckResult( False, f"Admin from issuer {op_authz.op_url} is not authorised to manage " From 8670a8741bc26f94fbcfc2459a9446cf53ff2c0c Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 7 Dec 2023 10:11:12 +0100 Subject: [PATCH 02/20] add api versioning --- motley_cue/api/__init__.py | 41 ++++++++++++++++ .../{routers => api/api_v1}/__init__.py | 0 motley_cue/api/api_v1/api.py | 6 +++ .../{api.py => api/api_v1/endpoints.py} | 47 ++++++------------- motley_cue/api/api_v1/routers/__init__.py | 0 motley_cue/{ => api/api_v1}/routers/admin.py | 4 +- motley_cue/{ => api/api_v1}/routers/user.py | 4 +- motley_cue/api/utils.py | 27 +++++++++++ motley_cue/dependencies.py | 2 + 9 files changed, 95 insertions(+), 36 deletions(-) create mode 100644 motley_cue/api/__init__.py rename motley_cue/{routers => api/api_v1}/__init__.py (100%) create mode 100644 motley_cue/api/api_v1/api.py rename motley_cue/{api.py => api/api_v1/endpoints.py} (73%) create mode 100644 motley_cue/api/api_v1/routers/__init__.py rename motley_cue/{ => api/api_v1}/routers/admin.py (96%) rename motley_cue/{ => api/api_v1}/routers/user.py (96%) create mode 100644 motley_cue/api/utils.py diff --git a/motley_cue/api/__init__.py b/motley_cue/api/__init__.py new file mode 100644 index 0000000..157d627 --- /dev/null +++ b/motley_cue/api/__init__.py @@ -0,0 +1,41 @@ +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError, ResponseValidationError +from pydantic import ValidationError + +from .api_v1.api import api_router as api_router_v1 +from ..dependencies import mapper, Settings +from ..mapper.exceptions import ( + validation_exception_handler, + request_validation_exception_handler, +) + + +def create_app(): + """Create motley_cue api.""" + + settings = Settings(docs_url=mapper.config.docs_url) + app = FastAPI( + title=settings.title, + description=settings.description, + version=settings.version, + openapi_url=settings.openapi_url, + docs_url=settings.docs_url, + redoc_url=settings.redoc_url, + ) + app.add_exception_handler( + RequestValidationError, request_validation_exception_handler + ) + app.add_exception_handler(RequestValidationError, request_validation_exception_handler) + app.add_exception_handler(ValidationError, validation_exception_handler) + app.add_exception_handler(ResponseValidationError, validation_exception_handler) + + app.include_router(api_router_v1, prefix="", tags=["API"]) + app.include_router(api_router_v1, prefix=settings.API_V1_STR, tags=["API v1"]) + app.include_router( + api_router_v1, prefix=settings.API_LATEST_STR, tags=["API latest"] + ) + + return app + + +api = create_app() diff --git a/motley_cue/routers/__init__.py b/motley_cue/api/api_v1/__init__.py similarity index 100% rename from motley_cue/routers/__init__.py rename to motley_cue/api/api_v1/__init__.py diff --git a/motley_cue/api/api_v1/api.py b/motley_cue/api/api_v1/api.py new file mode 100644 index 0000000..ae9bfe5 --- /dev/null +++ b/motley_cue/api/api_v1/api.py @@ -0,0 +1,6 @@ +from motley_cue.api.utils import APIRouter +from . import endpoints + + +api_router = APIRouter() +api_router.include_router(endpoints.router) diff --git a/motley_cue/api.py b/motley_cue/api/api_v1/endpoints.py similarity index 73% rename from motley_cue/api.py rename to motley_cue/api/api_v1/endpoints.py index fcccf57..cd46b31 100644 --- a/motley_cue/api.py +++ b/motley_cue/api/api_v1/endpoints.py @@ -1,37 +1,20 @@ """ -This module contains the definition of motley_cue's REST API. +This module contains the definition of motley_cue's REST router. """ -from fastapi import FastAPI, Depends, Request, Query, Header -from fastapi.exceptions import RequestValidationError, ResponseValidationError +from fastapi import Depends, Request, Query, Header from fastapi.responses import HTMLResponse -from pydantic import ValidationError -from .dependencies import mapper, Settings +from motley_cue.api.utils import APIRouter +from motley_cue.dependencies import mapper from .routers import user, admin -from .models import Info, InfoAuthorisation, InfoOp, VerifyUser, responses, ClientError -from .mapper.exceptions import ( - validation_exception_handler, - request_validation_exception_handler, -) - -settings = Settings(docs_url=mapper.config.docs_url) -api = FastAPI( - title=settings.title, - description=settings.description, - version=settings.version, - openapi_url=settings.openapi_url, - docs_url=settings.docs_url, - redoc_url=settings.redoc_url, -) +from motley_cue.models import Info, InfoAuthorisation, InfoOp, VerifyUser, responses -api.include_router(user.api, tags=["user"]) -api.include_router(admin.api, tags=["admin"]) -api.add_exception_handler(RequestValidationError, request_validation_exception_handler) -api.add_exception_handler(ValidationError, validation_exception_handler) -api.add_exception_handler(ResponseValidationError, validation_exception_handler) +router = APIRouter() +router.include_router(user.api, tags=["user"]) +router.include_router(admin.api, tags=["admin"]) -@api.get("/") +@router.get("/") async def read_root(): """Retrieve general API information: @@ -63,7 +46,7 @@ async def read_root(): } -@api.get("/info", response_model=Info, response_model_exclude_unset=True) +@router.get("/info", response_model=Info, response_model_exclude_unset=True) async def info(): """Retrieve service-specific information: @@ -74,7 +57,7 @@ async def info(): return mapper.info() -@api.get( +@router.get( "/info/authorisation", dependencies=[Depends(mapper.user_security)], response_model=InfoAuthorisation, @@ -96,7 +79,7 @@ async def info_authorisation( return mapper.info_authorisation(request) -@api.get( +@router.get( "/info/op", response_model=InfoOp, response_model_exclude_unset=True, @@ -114,7 +97,7 @@ async def info_op( return mapper.info_op(url) -@api.get( +@router.get( "/verify_user", dependencies=[Depends(mapper.user_security)], response_model=VerifyUser, @@ -143,13 +126,13 @@ async def verify_user( return mapper.verify_user(request, username) -@api.get("/privacy", response_class=HTMLResponse) +@router.get("/privacy", response_class=HTMLResponse) async def privacy(): return mapper.get_privacy_policy() # Logo for redoc (currently disabled). # This must be at the end after all the routes have been set! -# api.openapi()["info"]["x-logo"] = { +# router.openapi()["info"]["x-logo"] = { # "url": "https://motley-cue.readthedocs.io/en/latest/_static/logos/motley-cue.png" # } diff --git a/motley_cue/api/api_v1/routers/__init__.py b/motley_cue/api/api_v1/routers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/motley_cue/routers/admin.py b/motley_cue/api/api_v1/routers/admin.py similarity index 96% rename from motley_cue/routers/admin.py rename to motley_cue/api/api_v1/routers/admin.py index 03deacd..b7c24b8 100644 --- a/motley_cue/routers/admin.py +++ b/motley_cue/api/api_v1/routers/admin.py @@ -3,8 +3,8 @@ """ from fastapi import APIRouter, Request, Depends, Query, Header -from ..dependencies import mapper -from ..models import FeudalResponse, responses +from motley_cue.dependencies import mapper +from motley_cue.models import FeudalResponse, responses api = APIRouter(prefix="/admin") diff --git a/motley_cue/routers/user.py b/motley_cue/api/api_v1/routers/user.py similarity index 96% rename from motley_cue/routers/user.py rename to motley_cue/api/api_v1/routers/user.py index 19da1d3..3c3c7a2 100644 --- a/motley_cue/routers/user.py +++ b/motley_cue/api/api_v1/routers/user.py @@ -3,8 +3,8 @@ """ from fastapi import APIRouter, Request, Depends, Header -from ..dependencies import mapper -from ..models import FeudalResponse, OTPResponse, responses +from motley_cue.dependencies import mapper +from motley_cue.models import FeudalResponse, OTPResponse, responses api = APIRouter(prefix="/user") diff --git a/motley_cue/api/utils.py b/motley_cue/api/utils.py new file mode 100644 index 0000000..0b88ced --- /dev/null +++ b/motley_cue/api/utils.py @@ -0,0 +1,27 @@ +import typing as t +from fastapi import APIRouter as FastAPIRouter +from fastapi.types import DecoratedCallable + + +class APIRouter(FastAPIRouter): + """Overwrite APIRouter class from fastapi to remove trailing slashes from paths.""" + def api_route( + self, path: str, *, include_in_schema: bool = True, **kwargs: t.Any + ) -> t.Callable[[DecoratedCallable], DecoratedCallable]: + if path.endswith("/") and len(path) > 1: + path = path[:-1] + + add_path = super().api_route( + path, include_in_schema=include_in_schema, **kwargs + ) + + alternate_path = path + "/" + add_alternate_path = super().api_route( + alternate_path, include_in_schema=False, **kwargs + ) + + def decorator(func: DecoratedCallable) -> DecoratedCallable: + add_alternate_path(func) + return add_path(func) + + return decorator diff --git a/motley_cue/dependencies.py b/motley_cue/dependencies.py index 5b50568..deeb2d5 100644 --- a/motley_cue/dependencies.py +++ b/motley_cue/dependencies.py @@ -18,6 +18,8 @@ class Settings(BaseSettings): openapi_url: str = "/openapi.json" docs_url: Optional[str] = None redoc_url: Optional[str] = None + API_V1_STR: str = "/v1" + API_LATEST_STR: str = "/latest" @field_validator("openapi_url") @classmethod From cc02136e3ff43ddb10955ac49df1635f872c12ad Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 7 Dec 2023 10:15:31 +0100 Subject: [PATCH 03/20] use absolute path module imports --- motley_cue/api/__init__.py | 6 +++--- motley_cue/api/api_v1/api.py | 2 +- motley_cue/api/api_v1/endpoints.py | 2 +- motley_cue/dependencies.py | 4 ++-- motley_cue/mapper/__init__.py | 12 ++++++------ motley_cue/mapper/authorisation.py | 4 ++-- motley_cue/mapper/config.py | 2 +- motley_cue/mapper/local_user_management.py | 2 +- motley_cue/mapper/token_manager.py | 4 ++-- motley_cue/models.py | 2 +- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/motley_cue/api/__init__.py b/motley_cue/api/__init__.py index 157d627..1360448 100644 --- a/motley_cue/api/__init__.py +++ b/motley_cue/api/__init__.py @@ -2,9 +2,9 @@ from fastapi.exceptions import RequestValidationError, ResponseValidationError from pydantic import ValidationError -from .api_v1.api import api_router as api_router_v1 -from ..dependencies import mapper, Settings -from ..mapper.exceptions import ( +from motley_cue.api.api_v1.api import api_router as api_router_v1 +from motley_cue.dependencies import mapper, Settings +from motley_cue.mapper.exceptions import ( validation_exception_handler, request_validation_exception_handler, ) diff --git a/motley_cue/api/api_v1/api.py b/motley_cue/api/api_v1/api.py index ae9bfe5..74c253a 100644 --- a/motley_cue/api/api_v1/api.py +++ b/motley_cue/api/api_v1/api.py @@ -1,5 +1,5 @@ from motley_cue.api.utils import APIRouter -from . import endpoints +from motley_cue.api.api_v1 import endpoints api_router = APIRouter() diff --git a/motley_cue/api/api_v1/endpoints.py b/motley_cue/api/api_v1/endpoints.py index cd46b31..4c144be 100644 --- a/motley_cue/api/api_v1/endpoints.py +++ b/motley_cue/api/api_v1/endpoints.py @@ -6,7 +6,7 @@ from motley_cue.api.utils import APIRouter from motley_cue.dependencies import mapper -from .routers import user, admin +from motley_cue.api.api_v1.routers import user, admin from motley_cue.models import Info, InfoAuthorisation, InfoOp, VerifyUser, responses router = APIRouter() diff --git a/motley_cue/dependencies.py b/motley_cue/dependencies.py index deeb2d5..8bf2d03 100644 --- a/motley_cue/dependencies.py +++ b/motley_cue/dependencies.py @@ -3,8 +3,8 @@ from typing import Optional from pydantic import field_validator -from ._version import __version__ -from .mapper import Mapper, Config +from motley_cue._version import __version__ +from motley_cue.mapper import Mapper, Config from pydantic_settings import BaseSettings diff --git a/motley_cue/mapper/__init__.py b/motley_cue/mapper/__init__.py index 8916353..a638a92 100644 --- a/motley_cue/mapper/__init__.py +++ b/motley_cue/mapper/__init__.py @@ -8,12 +8,12 @@ from fastapi.security import HTTPBearer from fastapi.responses import HTMLResponse -from .config import Config -from .authorisation import Authorisation -from .local_user_management import LocalUserManager -from .exceptions import Unauthorised, NotFound -from .token_manager import TokenManager -from ..static import md_to_html +from motley_cue.mapper.config import Config +from motley_cue.mapper.authorisation import Authorisation +from motley_cue.mapper.local_user_management import LocalUserManager +from motley_cue.mapper.exceptions import Unauthorised, NotFound +from motley_cue.mapper.token_manager import TokenManager +from motley_cue.static import md_to_html class Mapper: diff --git a/motley_cue/mapper/authorisation.py b/motley_cue/mapper/authorisation.py index c56902c..d77918a 100644 --- a/motley_cue/mapper/authorisation.py +++ b/motley_cue/mapper/authorisation.py @@ -13,8 +13,8 @@ from flaat.user_infos import UserInfos from flaat.exceptions import FlaatException -from .config import Config, ConfigAuthorisation, canonical_url -from .exceptions import Unauthorised +from motley_cue.mapper.config import Config, ConfigAuthorisation, canonical_url +from motley_cue.mapper.exceptions import Unauthorised logger = logging.getLogger(__name__) diff --git a/motley_cue/mapper/config.py b/motley_cue/mapper/config.py index 09af107..42cdcf3 100644 --- a/motley_cue/mapper/config.py +++ b/motley_cue/mapper/config.py @@ -17,7 +17,7 @@ get_audience_requirement, ) -from .exceptions import InternalException +from motley_cue.mapper.exceptions import InternalException class Config: diff --git a/motley_cue/mapper/local_user_management.py b/motley_cue/mapper/local_user_management.py index da3a123..a520547 100644 --- a/motley_cue/mapper/local_user_management.py +++ b/motley_cue/mapper/local_user_management.py @@ -11,7 +11,7 @@ from ldf_adapter.results import ExceptionalResult, Rejection from ldf_adapter import User -from .exceptions import Unauthorised, InternalServerError +from motley_cue.mapper.exceptions import Unauthorised, InternalServerError class States(Enum): diff --git a/motley_cue/mapper/token_manager.py b/motley_cue/mapper/token_manager.py index 72855a4..639abbe 100644 --- a/motley_cue/mapper/token_manager.py +++ b/motley_cue/mapper/token_manager.py @@ -14,8 +14,8 @@ from cryptography.fernet import Fernet from pathlib import Path -from .config import ConfigOTP -from .exceptions import InternalException +from motley_cue.mapper.config import ConfigOTP +from motley_cue.mapper.exceptions import InternalException logger = logging.getLogger(__name__) diff --git a/motley_cue/models.py b/motley_cue/models.py index c3350f0..5c4ac73 100644 --- a/motley_cue/models.py +++ b/motley_cue/models.py @@ -4,7 +4,7 @@ from pydantic.dataclasses import dataclass from pydantic import Field -from .mapper.authorisation import AuthorisationType +from motley_cue.mapper.authorisation import AuthorisationType @dataclass From 9029d5d69106f006e144f7e5de3d0adf35eda784 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 7 Dec 2023 14:50:54 +0100 Subject: [PATCH 04/20] refactor api v1 routers and add support for redoc docs --- etc/motley_cue.conf | 2 ++ motley_cue/api/__init__.py | 11 ++++++----- motley_cue/api/api_v1/__init__.py | 6 ++++++ motley_cue/api/api_v1/{routers => }/admin.py | 4 ++-- motley_cue/api/api_v1/api.py | 6 ------ .../api/api_v1/{endpoints.py => root.py} | 19 ++++++++++--------- motley_cue/api/api_v1/routers/__init__.py | 0 motley_cue/api/api_v1/{routers => }/user.py | 6 +++--- motley_cue/dependencies.py | 4 ++-- motley_cue/mapper/config.py | 7 +++++++ 10 files changed, 38 insertions(+), 27 deletions(-) rename motley_cue/api/api_v1/{routers => }/admin.py (96%) delete mode 100644 motley_cue/api/api_v1/api.py rename motley_cue/api/api_v1/{endpoints.py => root.py} (89%) delete mode 100644 motley_cue/api/api_v1/routers/__init__.py rename motley_cue/api/api_v1/{routers => }/user.py (95%) diff --git a/etc/motley_cue.conf b/etc/motley_cue.conf index 63961a6..c56f299 100644 --- a/etc/motley_cue.conf +++ b/etc/motley_cue.conf @@ -14,6 +14,8 @@ log_level = WARNING # enable_docs = False ## location of swagger docs -- default: /docs # docs_url = /docs +## location of redoc docs -- default: /redoc +# redoc_url = /redoc ############ [mapper.otp] diff --git a/motley_cue/api/__init__.py b/motley_cue/api/__init__.py index 1360448..9d9bb81 100644 --- a/motley_cue/api/__init__.py +++ b/motley_cue/api/__init__.py @@ -2,7 +2,7 @@ from fastapi.exceptions import RequestValidationError, ResponseValidationError from pydantic import ValidationError -from motley_cue.api.api_v1.api import api_router as api_router_v1 +from motley_cue.api.api_v1 import api_router as api_router_v1 from motley_cue.dependencies import mapper, Settings from motley_cue.mapper.exceptions import ( validation_exception_handler, @@ -13,7 +13,7 @@ def create_app(): """Create motley_cue api.""" - settings = Settings(docs_url=mapper.config.docs_url) + settings = Settings(docs_url=mapper.config.docs_url, redoc_url=mapper.config.redoc_url) app = FastAPI( title=settings.title, description=settings.description, @@ -29,11 +29,12 @@ def create_app(): app.add_exception_handler(ValidationError, validation_exception_handler) app.add_exception_handler(ResponseValidationError, validation_exception_handler) + # for compatibility with old API, include all endpoints in the root app.include_router(api_router_v1, prefix="", tags=["API"]) + # latest API version + app.include_router(api_router_v1, prefix=settings.API_LATEST_STR, tags=["API latest"]) + # all API versions app.include_router(api_router_v1, prefix=settings.API_V1_STR, tags=["API v1"]) - app.include_router( - api_router_v1, prefix=settings.API_LATEST_STR, tags=["API latest"] - ) return app diff --git a/motley_cue/api/api_v1/__init__.py b/motley_cue/api/api_v1/__init__.py index e69de29..c62e3b8 100644 --- a/motley_cue/api/api_v1/__init__.py +++ b/motley_cue/api/api_v1/__init__.py @@ -0,0 +1,6 @@ +from motley_cue.api.utils import APIRouter +from motley_cue.api.api_v1 import root + + +api_router = APIRouter() +api_router.include_router(root.router) \ No newline at end of file diff --git a/motley_cue/api/api_v1/routers/admin.py b/motley_cue/api/api_v1/admin.py similarity index 96% rename from motley_cue/api/api_v1/routers/admin.py rename to motley_cue/api/api_v1/admin.py index b7c24b8..53cec00 100644 --- a/motley_cue/api/api_v1/routers/admin.py +++ b/motley_cue/api/api_v1/admin.py @@ -1,17 +1,17 @@ """ This module contains the definition of motley_cue's admin API. """ -from fastapi import APIRouter, Request, Depends, Query, Header +from fastapi import Request, Depends, Query, Header from motley_cue.dependencies import mapper from motley_cue.models import FeudalResponse, responses +from motley_cue.api.utils import APIRouter api = APIRouter(prefix="/admin") @api.get("") -@api.get("/", include_in_schema=False) async def read_root(): """Retrieve admin API information: diff --git a/motley_cue/api/api_v1/api.py b/motley_cue/api/api_v1/api.py deleted file mode 100644 index 74c253a..0000000 --- a/motley_cue/api/api_v1/api.py +++ /dev/null @@ -1,6 +0,0 @@ -from motley_cue.api.utils import APIRouter -from motley_cue.api.api_v1 import endpoints - - -api_router = APIRouter() -api_router.include_router(endpoints.router) diff --git a/motley_cue/api/api_v1/endpoints.py b/motley_cue/api/api_v1/root.py similarity index 89% rename from motley_cue/api/api_v1/endpoints.py rename to motley_cue/api/api_v1/root.py index 4c144be..4f54fe5 100644 --- a/motley_cue/api/api_v1/endpoints.py +++ b/motley_cue/api/api_v1/root.py @@ -6,12 +6,13 @@ from motley_cue.api.utils import APIRouter from motley_cue.dependencies import mapper -from motley_cue.api.api_v1.routers import user, admin from motley_cue.models import Info, InfoAuthorisation, InfoOp, VerifyUser, responses +from motley_cue.api.api_v1 import user, admin router = APIRouter() -router.include_router(user.api, tags=["user"]) -router.include_router(admin.api, tags=["admin"]) + +router.include_router(user.api) +router.include_router(admin.api) @router.get("/") @@ -26,20 +27,20 @@ async def read_root(): "description": "This is the user API for mapping remote identities to local identities.", "usage": "All endpoints are available via a bearer token.", "endpoints": { - "/info": "Service-specific information.", - "/info/authorisation": ( + f"{router.prefix}/info": "Service-specific information.", + f"{router.prefix}/info/authorisation": ( "Authorisation information for specific OP; " "requires valid access token from a supported OP." ), - "/info/op": ( + f"{router.prefix}/info/op": ( "Information about a specific OP specified via a query parameter 'url'; " "does not require an access token." ), - "/user": "User API; requires valid access token of an authorised user.", - "/admin": ( + f"{router.prefix}/user": "User API; requires valid access token of an authorised user.", + f"{router.prefix}/admin": ( "Admin API; requires valid access token of an authorised user with admin role." ), - "/verify_user": ( + f"{router.prefix}/verify_user": ( "Verifies if a given token belongs to a given local account via 'username'." ), }, diff --git a/motley_cue/api/api_v1/routers/__init__.py b/motley_cue/api/api_v1/routers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/motley_cue/api/api_v1/routers/user.py b/motley_cue/api/api_v1/user.py similarity index 95% rename from motley_cue/api/api_v1/routers/user.py rename to motley_cue/api/api_v1/user.py index 3c3c7a2..1621f6b 100644 --- a/motley_cue/api/api_v1/routers/user.py +++ b/motley_cue/api/api_v1/user.py @@ -1,17 +1,17 @@ """ This module contains the definition of motley_cue's user API. """ -from fastapi import APIRouter, Request, Depends, Header +from fastapi import Request, Depends, Header from motley_cue.dependencies import mapper from motley_cue.models import FeudalResponse, OTPResponse, responses +from motley_cue.api.utils import APIRouter -api = APIRouter(prefix="/user") +api = APIRouter(prefix="/user", redirect_slashes=True) @api.get("") -@api.get("/", include_in_schema=False) async def read_root(): """Retrieve user API information: diff --git a/motley_cue/dependencies.py b/motley_cue/dependencies.py index 8bf2d03..6e461e4 100644 --- a/motley_cue/dependencies.py +++ b/motley_cue/dependencies.py @@ -18,8 +18,8 @@ class Settings(BaseSettings): openapi_url: str = "/openapi.json" docs_url: Optional[str] = None redoc_url: Optional[str] = None - API_V1_STR: str = "/v1" - API_LATEST_STR: str = "/latest" + API_V1_STR: str = "/api/v1" + API_LATEST_STR: str = "/api" @field_validator("openapi_url") @classmethod diff --git a/motley_cue/mapper/config.py b/motley_cue/mapper/config.py index 42cdcf3..c59be2f 100644 --- a/motley_cue/mapper/config.py +++ b/motley_cue/mapper/config.py @@ -1,6 +1,7 @@ """Module for loading and describing motley_cue configuration. """ from configparser import ConfigParser +import re from typing import List, Optional, Dict import logging import os @@ -78,6 +79,11 @@ def docs_url(self): """return url to be used as location for swagger docs""" return self.CONFIG.mapper.docs_url if self.CONFIG.mapper.enable_docs else None + @property + def redoc_url(self): + """return url to be used as location for redoc docs""" + return self.CONFIG.mapper.redoc_url if self.CONFIG.mapper.enable_docs else None + @property def otp(self): """Return OTP configuration""" @@ -285,6 +291,7 @@ class ConfigMapper(ConfigSection): log_file: Optional[str] = None # equivalent to /dev/stderr enable_docs: bool = False docs_url: str = "/docs" + redoc_url: str = "/redoc" @classmethod def __section__name__(cls): From 5a1ade7fa99f9503620d3f09f300ce9356ae8efb Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 7 Dec 2023 17:54:24 +0100 Subject: [PATCH 05/20] add possibility to configure api version in conf file, add more generic api include --- etc/motley_cue.conf | 4 +++ motley_cue/api/__init__.py | 47 ++++++++++++++++++++++---- motley_cue/api/api_v1/__init__.py | 6 ---- motley_cue/api/utils.py | 1 + motley_cue/api/v1/__init__.py | 8 +++++ motley_cue/api/{api_v1 => v1}/admin.py | 14 ++++---- motley_cue/api/{api_v1 => v1}/root.py | 24 +++++-------- motley_cue/api/{api_v1 => v1}/user.py | 24 +++++++------ motley_cue/dependencies.py | 11 ++++-- motley_cue/mapper/config.py | 7 ++++ 10 files changed, 100 insertions(+), 46 deletions(-) delete mode 100644 motley_cue/api/api_v1/__init__.py create mode 100644 motley_cue/api/v1/__init__.py rename motley_cue/api/{api_v1 => v1}/admin.py (89%) rename motley_cue/api/{api_v1 => v1}/root.py (87%) rename motley_cue/api/{api_v1 => v1}/user.py (83%) diff --git a/etc/motley_cue.conf b/etc/motley_cue.conf index c56f299..d529707 100644 --- a/etc/motley_cue.conf +++ b/etc/motley_cue.conf @@ -16,6 +16,10 @@ log_level = WARNING # docs_url = /docs ## location of redoc docs -- default: /redoc # redoc_url = /redoc +## +## API version -- default: v1 +## supported versions: v1 +# api_version = v1 ############ [mapper.otp] diff --git a/motley_cue/api/__init__.py b/motley_cue/api/__init__.py index 9d9bb81..28789ce 100644 --- a/motley_cue/api/__init__.py +++ b/motley_cue/api/__init__.py @@ -1,10 +1,13 @@ +import os +import pkgutil +import importlib from fastapi import FastAPI from fastapi.exceptions import RequestValidationError, ResponseValidationError from pydantic import ValidationError -from motley_cue.api.api_v1 import api_router as api_router_v1 from motley_cue.dependencies import mapper, Settings from motley_cue.mapper.exceptions import ( + InternalException, validation_exception_handler, request_validation_exception_handler, ) @@ -13,7 +16,11 @@ def create_app(): """Create motley_cue api.""" - settings = Settings(docs_url=mapper.config.docs_url, redoc_url=mapper.config.redoc_url) + settings = Settings( + docs_url=mapper.config.docs_url, + redoc_url=mapper.config.redoc_url, + api_version=mapper.config.api_version, + ) app = FastAPI( title=settings.title, description=settings.description, @@ -25,16 +32,42 @@ def create_app(): app.add_exception_handler( RequestValidationError, request_validation_exception_handler ) - app.add_exception_handler(RequestValidationError, request_validation_exception_handler) + app.add_exception_handler( + RequestValidationError, request_validation_exception_handler + ) app.add_exception_handler(ValidationError, validation_exception_handler) app.add_exception_handler(ResponseValidationError, validation_exception_handler) + # get routers for all api versions + api_routers = {} + for version_submodule in pkgutil.iter_modules([os.path.dirname(__file__)]): + if version_submodule.name.startswith("v"): + api_routers[version_submodule.name] = importlib.import_module( + f"motley_cue.api.{version_submodule.name}" + ).router + + try: + current_api_router = api_routers[settings.api_version] + except KeyError as exc: + raise InternalException( + f"API version {settings.api_version} does not exist." + ) from exc + + # current API version + app.include_router(current_api_router, prefix="/api", tags=["API"]) # for compatibility with old API, include all endpoints in the root - app.include_router(api_router_v1, prefix="", tags=["API"]) - # latest API version - app.include_router(api_router_v1, prefix=settings.API_LATEST_STR, tags=["API latest"]) + app.include_router(current_api_router, prefix="", include_in_schema=False) + # all API versions - app.include_router(api_router_v1, prefix=settings.API_V1_STR, tags=["API v1"]) + for api_version, api_router in api_routers.items(): + app.include_router( + api_router, prefix=f"/api/{api_version}", tags=[f"API {api_version}"] + ) + + # Logo for redoc. This must be at the end after all the routes have been set! + app.openapi()["info"]["x-logo"] = { + "url": "https://motley-cue.readthedocs.io/en/latest/_static/logos/motley-cue.png" + } return app diff --git a/motley_cue/api/api_v1/__init__.py b/motley_cue/api/api_v1/__init__.py deleted file mode 100644 index c62e3b8..0000000 --- a/motley_cue/api/api_v1/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from motley_cue.api.utils import APIRouter -from motley_cue.api.api_v1 import root - - -api_router = APIRouter() -api_router.include_router(root.router) \ No newline at end of file diff --git a/motley_cue/api/utils.py b/motley_cue/api/utils.py index 0b88ced..8fcec42 100644 --- a/motley_cue/api/utils.py +++ b/motley_cue/api/utils.py @@ -5,6 +5,7 @@ class APIRouter(FastAPIRouter): """Overwrite APIRouter class from fastapi to remove trailing slashes from paths.""" + def api_route( self, path: str, *, include_in_schema: bool = True, **kwargs: t.Any ) -> t.Callable[[DecoratedCallable], DecoratedCallable]: diff --git a/motley_cue/api/v1/__init__.py b/motley_cue/api/v1/__init__.py new file mode 100644 index 0000000..20a22a5 --- /dev/null +++ b/motley_cue/api/v1/__init__.py @@ -0,0 +1,8 @@ +from motley_cue.api.utils import APIRouter +from motley_cue.api.v1 import root, user, admin + + +router = APIRouter() +router.include_router(root.router) +router.include_router(user.router) +router.include_router(admin.router) diff --git a/motley_cue/api/api_v1/admin.py b/motley_cue/api/v1/admin.py similarity index 89% rename from motley_cue/api/api_v1/admin.py rename to motley_cue/api/v1/admin.py index 53cec00..0cdf6f4 100644 --- a/motley_cue/api/api_v1/admin.py +++ b/motley_cue/api/v1/admin.py @@ -8,10 +8,10 @@ from motley_cue.api.utils import APIRouter -api = APIRouter(prefix="/admin") +router = APIRouter(prefix="/admin") -@api.get("") +@router.get("", summary="Admin: API info") async def read_root(): """Retrieve admin API information: @@ -24,14 +24,15 @@ async def read_root(): "usage": "All endpoints are available using an OIDC Access Token as a bearer token and " "need subject and issuer of account to be modified, via 'sub' and 'iss' variables.", "endpoints": { - f"{api.prefix}/suspend": "Suspends a local account.", - f"{api.prefix}/resume": "Restores a suspended local account.", + f"{router.prefix}/suspend": "Suspends a local account.", + f"{router.prefix}/resume": "Restores a suspended local account.", }, } -@api.get( +@router.get( "/suspend", + summary="Admin: suspend user", dependencies=[Depends(mapper.admin_security)], response_model=FeudalResponse, response_model_exclude_unset=True, @@ -61,8 +62,9 @@ async def suspend( return mapper.admin_suspend(sub, iss) -@api.get( +@router.get( "/resume", + summary="Admin: resume user", dependencies=[Depends(mapper.admin_security)], response_model=FeudalResponse, response_model_exclude_unset=True, diff --git a/motley_cue/api/api_v1/root.py b/motley_cue/api/v1/root.py similarity index 87% rename from motley_cue/api/api_v1/root.py rename to motley_cue/api/v1/root.py index 4f54fe5..26f99d7 100644 --- a/motley_cue/api/api_v1/root.py +++ b/motley_cue/api/v1/root.py @@ -7,16 +7,13 @@ from motley_cue.api.utils import APIRouter from motley_cue.dependencies import mapper from motley_cue.models import Info, InfoAuthorisation, InfoOp, VerifyUser, responses -from motley_cue.api.api_v1 import user, admin -router = APIRouter() -router.include_router(user.api) -router.include_router(admin.api) +router = APIRouter() -@router.get("/") -async def read_root(): +@router.get("/", summary="API info") +async def root_api(): """Retrieve general API information: * description @@ -47,7 +44,7 @@ async def read_root(): } -@router.get("/info", response_model=Info, response_model_exclude_unset=True) +@router.get("/info", summary="Login info", response_model=Info, response_model_exclude_unset=True) async def info(): """Retrieve service-specific information: @@ -60,6 +57,7 @@ async def info(): @router.get( "/info/authorisation", + summary="Authorisation info by OP", dependencies=[Depends(mapper.user_security)], response_model=InfoAuthorisation, response_model_exclude_unset=True, @@ -82,6 +80,7 @@ async def info_authorisation( @router.get( "/info/op", + summary="OP info", response_model=InfoOp, response_model_exclude_unset=True, responses={**responses, 200: {"model": InfoOp}}, @@ -100,6 +99,7 @@ async def info_op( @router.get( "/verify_user", + summary="Verify user", dependencies=[Depends(mapper.user_security)], response_model=VerifyUser, responses={**responses, 200: {"model": VerifyUser}}, @@ -127,13 +127,7 @@ async def verify_user( return mapper.verify_user(request, username) -@router.get("/privacy", response_class=HTMLResponse) +@router.get("/privacy", summary="Privacy policy", response_class=HTMLResponse) async def privacy(): + """Retrieve privacy policy.""" return mapper.get_privacy_policy() - - -# Logo for redoc (currently disabled). -# This must be at the end after all the routes have been set! -# router.openapi()["info"]["x-logo"] = { -# "url": "https://motley-cue.readthedocs.io/en/latest/_static/logos/motley-cue.png" -# } diff --git a/motley_cue/api/api_v1/user.py b/motley_cue/api/v1/user.py similarity index 83% rename from motley_cue/api/api_v1/user.py rename to motley_cue/api/v1/user.py index 1621f6b..0dbfe82 100644 --- a/motley_cue/api/api_v1/user.py +++ b/motley_cue/api/v1/user.py @@ -8,10 +8,10 @@ from motley_cue.api.utils import APIRouter -api = APIRouter(prefix="/user", redirect_slashes=True) +router = APIRouter(prefix="/user") -@api.get("") +@router.get("", summary="User: API info") async def read_root(): """Retrieve user API information: @@ -23,16 +23,17 @@ async def read_root(): "description": "This is the user API for mapping remote identities to local identities.", "usage": "All endpoints are available using an OIDC Access Token as a bearer token.", "endpoints": { - f"{api.prefix}/get_status": "Get information about your local account.", - f"{api.prefix}/deploy": "Provision local account.", - f"{api.prefix}/suspend": "Suspend local account.", - f"{api.prefix}/generate_otp": "Generates a one-time token for given access token.", + f"{router.prefix}/get_status": "Get information about your local account.", + f"{router.prefix}/deploy": "Provision local account.", + f"{router.prefix}/suspend": "Suspend local account.", + f"{router.prefix}/generate_otp": "Generates a one-time token for given access token.", }, } -@api.get( +@router.get( "/get_status", + summary="User: get status", dependencies=[Depends(mapper.user_security)], response_model=FeudalResponse, response_model_exclude_unset=True, @@ -53,8 +54,9 @@ async def get_status( return mapper.get_status(request) -@api.get( +@router.get( "/deploy", + summary="User: deploy", dependencies=[Depends(mapper.user_security)], response_model=FeudalResponse, response_model_exclude_unset=True, @@ -72,8 +74,9 @@ async def deploy( return mapper.deploy(request) -@api.get( +@router.get( "/suspend", + summary="User: suspend", dependencies=[Depends(mapper.user_security)], response_model=FeudalResponse, response_model_exclude_unset=True, @@ -91,8 +94,9 @@ async def suspend( return mapper.suspend(request) -@api.get( +@router.get( "/generate_otp", + summary="User: generate one-time token", dependencies=[Depends(mapper.user_security)], response_model=OTPResponse, response_model_exclude_unset=True, diff --git a/motley_cue/dependencies.py b/motley_cue/dependencies.py index 6e461e4..0eb2ebb 100644 --- a/motley_cue/dependencies.py +++ b/motley_cue/dependencies.py @@ -18,8 +18,7 @@ class Settings(BaseSettings): openapi_url: str = "/openapi.json" docs_url: Optional[str] = None redoc_url: Optional[str] = None - API_V1_STR: str = "/api/v1" - API_LATEST_STR: str = "/api" + api_version: str = "v1" @field_validator("openapi_url") @classmethod @@ -37,5 +36,13 @@ def must_start_with_slash_or_none(cls, url): raise ValueError("Routed paths must start with '/'") return url + @field_validator("api_version") + @classmethod + def must_start_with_v(cls, version): + """validate API version: must start with a 'v'""" + if not version.startswith("v"): + raise ValueError("API version must start with 'v'") + return version + mapper = Mapper(Config.from_files([])) diff --git a/motley_cue/mapper/config.py b/motley_cue/mapper/config.py index c59be2f..73e7e13 100644 --- a/motley_cue/mapper/config.py +++ b/motley_cue/mapper/config.py @@ -2,6 +2,7 @@ """ from configparser import ConfigParser import re +from sys import api_version from typing import List, Optional, Dict import logging import os @@ -84,6 +85,11 @@ def redoc_url(self): """return url to be used as location for redoc docs""" return self.CONFIG.mapper.redoc_url if self.CONFIG.mapper.enable_docs else None + @property + def api_version(self): + """return api version""" + return self.CONFIG.mapper.api_version + @property def otp(self): """Return OTP configuration""" @@ -292,6 +298,7 @@ class ConfigMapper(ConfigSection): enable_docs: bool = False docs_url: str = "/docs" redoc_url: str = "/redoc" + api_version: str = "v1" @classmethod def __section__name__(cls): From 25ec8579da6122f6dff6f9b63ed18097538ec362 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 7 Dec 2023 18:08:20 +0100 Subject: [PATCH 06/20] add error handling for submodule imports for api version --- motley_cue/api/__init__.py | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/motley_cue/api/__init__.py b/motley_cue/api/__init__.py index 28789ce..579e969 100644 --- a/motley_cue/api/__init__.py +++ b/motley_cue/api/__init__.py @@ -1,3 +1,4 @@ +import logging import os import pkgutil import importlib @@ -12,6 +13,8 @@ request_validation_exception_handler, ) +logger = logging.getLogger(__name__) + def create_app(): """Create motley_cue api.""" @@ -42,9 +45,28 @@ def create_app(): api_routers = {} for version_submodule in pkgutil.iter_modules([os.path.dirname(__file__)]): if version_submodule.name.startswith("v"): - api_routers[version_submodule.name] = importlib.import_module( - f"motley_cue.api.{version_submodule.name}" - ).router + try: + api_routers[version_submodule.name] = importlib.import_module( + f"motley_cue.api.{version_submodule.name}" + ).router + except AttributeError as exc: + logger.error( + "API version %s does not have a router", + version_submodule.name, + exc_info=exc, + ) + raise InternalException( + f"API version {version_submodule.name} does not have a router." + ) from exc + except Exception as exc: + logger.error( + "Could not import API version %s", + version_submodule.name, + exc_info=exc, + ) + raise InternalException( + f"Could not import API version {version_submodule.name}" + ) from exc try: current_api_router = api_routers[settings.api_version] From 5f66a09930a57f066d468175b37f3c6d943d9004 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Fri, 8 Dec 2023 11:59:54 +0100 Subject: [PATCH 07/20] fix api-related import errors in tests --- motley_cue/{api/__init__.py => api.py} | 5 +++-- motley_cue/apis/__init__.py | 0 motley_cue/{api => apis}/utils.py | 0 motley_cue/apis/v1/__init__.py | 0 motley_cue/{api => apis}/v1/admin.py | 2 +- motley_cue/{api/v1/__init__.py => apis/v1/api.py} | 4 ++-- motley_cue/{api => apis}/v1/root.py | 11 ++++++++--- motley_cue/{api => apis}/v1/user.py | 2 +- motley_cue/mapper/config.py | 2 -- tests/conftest.py | 4 ++-- 10 files changed, 17 insertions(+), 13 deletions(-) rename motley_cue/{api/__init__.py => api.py} (94%) create mode 100644 motley_cue/apis/__init__.py rename motley_cue/{api => apis}/utils.py (100%) create mode 100644 motley_cue/apis/v1/__init__.py rename motley_cue/{api => apis}/v1/admin.py (98%) rename motley_cue/{api/v1/__init__.py => apis/v1/api.py} (58%) rename motley_cue/{api => apis}/v1/root.py (94%) rename motley_cue/{api => apis}/v1/user.py (98%) diff --git a/motley_cue/api/__init__.py b/motley_cue/api.py similarity index 94% rename from motley_cue/api/__init__.py rename to motley_cue/api.py index 579e969..fecd626 100644 --- a/motley_cue/api/__init__.py +++ b/motley_cue/api.py @@ -12,6 +12,7 @@ validation_exception_handler, request_validation_exception_handler, ) +import motley_cue.apis logger = logging.getLogger(__name__) @@ -43,11 +44,11 @@ def create_app(): # get routers for all api versions api_routers = {} - for version_submodule in pkgutil.iter_modules([os.path.dirname(__file__)]): + for version_submodule in pkgutil.iter_modules(motley_cue.apis.__path__): if version_submodule.name.startswith("v"): try: api_routers[version_submodule.name] = importlib.import_module( - f"motley_cue.api.{version_submodule.name}" + f"motley_cue.apis.{version_submodule.name}.api" ).router except AttributeError as exc: logger.error( diff --git a/motley_cue/apis/__init__.py b/motley_cue/apis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/motley_cue/api/utils.py b/motley_cue/apis/utils.py similarity index 100% rename from motley_cue/api/utils.py rename to motley_cue/apis/utils.py diff --git a/motley_cue/apis/v1/__init__.py b/motley_cue/apis/v1/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/motley_cue/api/v1/admin.py b/motley_cue/apis/v1/admin.py similarity index 98% rename from motley_cue/api/v1/admin.py rename to motley_cue/apis/v1/admin.py index 0cdf6f4..c5fa670 100644 --- a/motley_cue/api/v1/admin.py +++ b/motley_cue/apis/v1/admin.py @@ -5,7 +5,7 @@ from motley_cue.dependencies import mapper from motley_cue.models import FeudalResponse, responses -from motley_cue.api.utils import APIRouter +from motley_cue.apis.utils import APIRouter router = APIRouter(prefix="/admin") diff --git a/motley_cue/api/v1/__init__.py b/motley_cue/apis/v1/api.py similarity index 58% rename from motley_cue/api/v1/__init__.py rename to motley_cue/apis/v1/api.py index 20a22a5..b976750 100644 --- a/motley_cue/api/v1/__init__.py +++ b/motley_cue/apis/v1/api.py @@ -1,5 +1,5 @@ -from motley_cue.api.utils import APIRouter -from motley_cue.api.v1 import root, user, admin +from motley_cue.apis.utils import APIRouter +from motley_cue.apis.v1 import root, user, admin router = APIRouter() diff --git a/motley_cue/api/v1/root.py b/motley_cue/apis/v1/root.py similarity index 94% rename from motley_cue/api/v1/root.py rename to motley_cue/apis/v1/root.py index 26f99d7..65fb4ab 100644 --- a/motley_cue/api/v1/root.py +++ b/motley_cue/apis/v1/root.py @@ -1,10 +1,10 @@ """ This module contains the definition of motley_cue's REST router. """ -from fastapi import Depends, Request, Query, Header +from fastapi import Depends, Request, Query, Header from fastapi.responses import HTMLResponse -from motley_cue.api.utils import APIRouter +from motley_cue.apis.utils import APIRouter from motley_cue.dependencies import mapper from motley_cue.models import Info, InfoAuthorisation, InfoOp, VerifyUser, responses @@ -44,7 +44,12 @@ async def root_api(): } -@router.get("/info", summary="Login info", response_model=Info, response_model_exclude_unset=True) +@router.get( + "/info", + summary="Login info", + response_model=Info, + response_model_exclude_unset=True, +) async def info(): """Retrieve service-specific information: diff --git a/motley_cue/api/v1/user.py b/motley_cue/apis/v1/user.py similarity index 98% rename from motley_cue/api/v1/user.py rename to motley_cue/apis/v1/user.py index 0dbfe82..25ca4dd 100644 --- a/motley_cue/api/v1/user.py +++ b/motley_cue/apis/v1/user.py @@ -5,7 +5,7 @@ from motley_cue.dependencies import mapper from motley_cue.models import FeudalResponse, OTPResponse, responses -from motley_cue.api.utils import APIRouter +from motley_cue.apis.utils import APIRouter router = APIRouter(prefix="/user") diff --git a/motley_cue/mapper/config.py b/motley_cue/mapper/config.py index 73e7e13..60e5faf 100644 --- a/motley_cue/mapper/config.py +++ b/motley_cue/mapper/config.py @@ -1,8 +1,6 @@ """Module for loading and describing motley_cue configuration. """ from configparser import ConfigParser -import re -from sys import api_version from typing import List, Optional, Dict import logging import os diff --git a/tests/conftest.py b/tests/conftest.py index 3e39f78..17717ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -35,8 +35,8 @@ def test_api( # import FastAPI object here to make sure its decorators are based on monkeypatched mapper from motley_cue.api import api - test_api = TestClient(api) - yield test_api + test_client = TestClient(api) + yield test_client # unload all motley_cue modules for m in [x for x in sys.modules if x.startswith("motley_cue")]: From 9bf1f990c0333a7aca3ce9418e1688fa968faab5 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Mon, 11 Dec 2023 11:11:06 +0100 Subject: [PATCH 08/20] fix pyright error: return type __section__name___ --- motley_cue/mapper/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/motley_cue/mapper/config.py b/motley_cue/mapper/config.py index 60e5faf..884847e 100644 --- a/motley_cue/mapper/config.py +++ b/motley_cue/mapper/config.py @@ -234,7 +234,7 @@ def canonical_url(url: str) -> str: @dataclass class ConfigSection: @classmethod - def __section__name__(cls): + def __section__name__(cls) -> str: return "DEFAULT" @classmethod From 47d0462504a3b10c52a1546f1476d0103354d581 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Mon, 11 Dec 2023 13:24:23 +0100 Subject: [PATCH 09/20] =?UTF-8?q?Bump=20version:=200.6.1=20=E2=86=92=200.7?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- debian/changelog | 2 +- motley_cue/VERSION | 2 +- rpm/motley-cue.spec | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index b91ad92..80aa735 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.6.1 +current_version = 0.7.0 commit = True tag = False diff --git a/debian/changelog b/debian/changelog index ea4deaa..21a7292 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,4 @@ -motley-cue (0.6.1-1) UNRELEASED; urgency=medium +motley-cue (0.7.0-1) UNRELEASED; urgency=medium [ Marcus Hardt ] * Initial go for packageing diff --git a/motley_cue/VERSION b/motley_cue/VERSION index 7ceb040..bcaffe1 100644 --- a/motley_cue/VERSION +++ b/motley_cue/VERSION @@ -1 +1 @@ -0.6.1 \ No newline at end of file +0.7.0 \ No newline at end of file diff --git a/rpm/motley-cue.spec b/rpm/motley-cue.spec index 3e25ef6..af29572 100644 --- a/rpm/motley-cue.spec +++ b/rpm/motley-cue.spec @@ -1,5 +1,5 @@ Name: motley-cue -Version: 0.6.1 +Version: 0.7.0 Release: 1%{?dist} Summary: Mapper Oidc To Local idEntitY with loCal User managEment From d851fa9b4e86024df9d41de41d714eddaa9aff31 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Wed, 13 Dec 2023 18:01:36 +0100 Subject: [PATCH 10/20] add github action to mirror to gitlab --- .github/workflows/mirror-repo.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 .github/workflows/mirror-repo.yml diff --git a/.github/workflows/mirror-repo.yml b/.github/workflows/mirror-repo.yml new file mode 100644 index 0000000..1075329 --- /dev/null +++ b/.github/workflows/mirror-repo.yml @@ -0,0 +1,19 @@ +name: Mirror Repository + +on: + - push + - delete + +jobs: + sync: + runs-on: ubuntu-latest + name: Git Repo Sync + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: wangchucheng/git-repo-sync@v0.1.0 + with: + target-url: secrets.GITLAB_MIRROR_URL + target-username: secrets.GITLAB_MIRROR_USERNAME + target-token: secrets.GITLAB_MIRROR_TOKEN From 1737d10c51f7b65db53e1d7988581a52ac945475 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 14 Dec 2023 11:39:09 +0100 Subject: [PATCH 11/20] use secrets as vars in gitlab sync --- .github/workflows/mirror-repo.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/mirror-repo.yml b/.github/workflows/mirror-repo.yml index 1075329..009872e 100644 --- a/.github/workflows/mirror-repo.yml +++ b/.github/workflows/mirror-repo.yml @@ -14,6 +14,6 @@ jobs: fetch-depth: 0 - uses: wangchucheng/git-repo-sync@v0.1.0 with: - target-url: secrets.GITLAB_MIRROR_URL - target-username: secrets.GITLAB_MIRROR_USERNAME - target-token: secrets.GITLAB_MIRROR_TOKEN + target-url: ${{ secrets.GITLAB_MIRROR_URL }} + target-username: ${{ secrets.GITLAB_MIRROR_USERNAME }} + target-token: ${{ secrets.GITLAB_MIRROR_TOKEN }} From 2bf2417f9cf8eb3c227b58256fa9ff5a3f747479 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 28 Dec 2023 17:56:00 +0100 Subject: [PATCH 12/20] add gitlab ci target to report status back to github --- .gitlab-ci.yml | 46 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0b02b80..f11e633 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,17 +1,19 @@ --- include: - - 'https://git.scc.kit.edu/m-team/ci-voodoo/raw/master/ci-include/generic-ci.yml' - - 'https://git.scc.kit.edu/m-team/ci-voodoo/raw/master/ci-include/pipeline-jobs.yml' - - 'https://git.scc.kit.edu/m-team/ci-voodoo/raw/master/ci-include/pipeline-jobs-publish-to-repo.yml' + - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/generic-ci.yml" + - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/pipeline-jobs.yml" + - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/pipeline-jobs-publish-to-repo.yml" + - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/github-status/ci-include/github-status-sync.yml" default: tags: - linux variables: - STAGING_BRANCH_NAME: 'staging' - DOCKER_IMAGE_NAMESPACE: 'marcvs/build' - DOCKER_IMAGE_NAME: 'motley-cue' + STATUS_PROJECT: dianagudu/motley_cue + STAGING_BRANCH_NAME: "staging" + DOCKER_IMAGE_NAMESPACE: "marcvs/build" + DOCKER_IMAGE_NAME: "motley-cue" #PREREL_BRANCH_NAME: 'ci/adapt-to-pam-ssh-oidc' #TARGET_REPO: 'devel' # The following varialbes can be overwritten only in specific targets @@ -55,4 +57,36 @@ integration-tests: extends: - .trigger-integration-tests-ssh-oidc +########################################################################## +.report-status: + variables: + STATUS_PROJECT: dianagudu/motley_cue + STATUS_NAME: Codebase CI + + script: + # For complete details on the GitHub API please see: + # https://developer.github.com/v3/repos/statuses + - 'curl -L + -X POST + -H "Accept: application/vnd.github+json" + -H "Authorization: Bearer ${GH_STATUS_TOKEN}" + -H "X-GitHub-Api-Version: 2022-11-28" + https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA} + -d "{\"state\": \"${CI_JOB_NAME}\",\"context\": \"${STATUS_NAME}\", \"target_url\": \"${CI_PIPELINE_URL}\"}"' + +pending: + stage: .pre + extends: + - .report-status + +success: + stage: .post + extends: + - .report-status +failed: + stage: .post + extends: + - .report-status + rules: + - when: on_failure From d5f9a3e32cf556be905bb65d0b512e851af5ac24 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 28 Dec 2023 20:29:29 +0100 Subject: [PATCH 13/20] fix pyright errors for custom exception handlers --- motley_cue/api.py | 165 +++++++++++++++++--------------- motley_cue/mapper/exceptions.py | 51 +++++----- 2 files changed, 112 insertions(+), 104 deletions(-) diff --git a/motley_cue/api.py b/motley_cue/api.py index fecd626..fe2f210 100644 --- a/motley_cue/api.py +++ b/motley_cue/api.py @@ -1,98 +1,113 @@ import logging -import os import pkgutil import importlib -from fastapi import FastAPI -from fastapi.exceptions import RequestValidationError, ResponseValidationError +from typing import Union from pydantic import ValidationError +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError, ResponseValidationError from motley_cue.dependencies import mapper, Settings from motley_cue.mapper.exceptions import ( InternalException, - validation_exception_handler, - request_validation_exception_handler, + InvalidResponse, + MissingParameter, ) import motley_cue.apis logger = logging.getLogger(__name__) +# API settings +settings = Settings( + docs_url=mapper.config.docs_url, + redoc_url=mapper.config.redoc_url, + api_version=mapper.config.api_version, +) -def create_app(): - """Create motley_cue api.""" +# create FastAPI app +api = FastAPI( + title=settings.title, + description=settings.description, + version=settings.version, + openapi_url=settings.openapi_url, + docs_url=settings.docs_url, + redoc_url=settings.redoc_url, +) - settings = Settings( - docs_url=mapper.config.docs_url, - redoc_url=mapper.config.redoc_url, - api_version=mapper.config.api_version, - ) - app = FastAPI( - title=settings.title, - description=settings.description, - version=settings.version, - openapi_url=settings.openapi_url, - docs_url=settings.docs_url, - redoc_url=settings.redoc_url, - ) - app.add_exception_handler( - RequestValidationError, request_validation_exception_handler - ) - app.add_exception_handler( - RequestValidationError, request_validation_exception_handler - ) - app.add_exception_handler(ValidationError, validation_exception_handler) - app.add_exception_handler(ResponseValidationError, validation_exception_handler) +# get routers for all api versions +api_routers = {} +for version_submodule in pkgutil.iter_modules(motley_cue.apis.__path__): + if version_submodule.name.startswith("v"): + try: + api_routers[version_submodule.name] = importlib.import_module( + f"motley_cue.apis.{version_submodule.name}.api" + ).router + except AttributeError as exc: + logger.error( + "API version %s does not have a router", + version_submodule.name, + exc_info=exc, + ) + raise InternalException( + f"API version {version_submodule.name} does not have a router." + ) from exc + except Exception as exc: + logger.error( + "Could not import API version %s", + version_submodule.name, + exc_info=exc, + ) + raise InternalException( + f"Could not import API version {version_submodule.name}" + ) from exc + +try: + current_api_router = api_routers[settings.api_version] +except KeyError as exc: + raise InternalException( + f"API version {settings.api_version} does not exist." + ) from exc - # get routers for all api versions - api_routers = {} - for version_submodule in pkgutil.iter_modules(motley_cue.apis.__path__): - if version_submodule.name.startswith("v"): - try: - api_routers[version_submodule.name] = importlib.import_module( - f"motley_cue.apis.{version_submodule.name}.api" - ).router - except AttributeError as exc: - logger.error( - "API version %s does not have a router", - version_submodule.name, - exc_info=exc, - ) - raise InternalException( - f"API version {version_submodule.name} does not have a router." - ) from exc - except Exception as exc: - logger.error( - "Could not import API version %s", - version_submodule.name, - exc_info=exc, - ) - raise InternalException( - f"Could not import API version {version_submodule.name}" - ) from exc +# current API version +api.include_router(current_api_router, prefix="/api", tags=["API"]) +# for compatibility with old API, include all endpoints in the root +api.include_router(current_api_router, prefix="", include_in_schema=False) + +# all API versions +for api_version, api_router in api_routers.items(): + api.include_router( + api_router, prefix=f"/api/{api_version}", tags=[f"API {api_version}"] + ) - try: - current_api_router = api_routers[settings.api_version] - except KeyError as exc: - raise InternalException( - f"API version {settings.api_version} does not exist." - ) from exc +# Logo for redoc. This must be at the end after all the routes have been set! +api.openapi()["info"]["x-logo"] = { + "url": "https://motley-cue.readthedocs.io/en/latest/_static/logos/motley-cue.png" +} - # current API version - app.include_router(current_api_router, prefix="/api", tags=["API"]) - # for compatibility with old API, include all endpoints in the root - app.include_router(current_api_router, prefix="", include_in_schema=False) - # all API versions - for api_version, api_router in api_routers.items(): - app.include_router( - api_router, prefix=f"/api/{api_version}", tags=[f"API {api_version}"] - ) +# Exception handlers +@api.exception_handler(ResponseValidationError) +@api.exception_handler(ValidationError) +async def validation_exception_handler( + request: Request, validation_exc: Union[ResponseValidationError, ValidationError] +): + """Replacement callback for handling ResponseValidationError exceptions. - # Logo for redoc. This must be at the end after all the routes have been set! - app.openapi()["info"]["x-logo"] = { - "url": "https://motley-cue.readthedocs.io/en/latest/_static/logos/motley-cue.png" - } + :param request: request object that caused the ResponseValidationError + :param validation_exc: ResponseValidationError containing validation errors + """ + _ = request + _ = validation_exc + return InvalidResponse(validation_exc) - return app +@api.exception_handler(RequestValidationError) +async def request_validation_exception_handler( + request: Request, validation_exc: RequestValidationError +): + """Replacement callback for handling RequestValidationError exceptions. -api = create_app() + :param request: request object that caused the RequestValidationError + :param validation_exc: RequestValidationError containing validation errors + """ + _ = request + return MissingParameter(validation_exc) diff --git a/motley_cue/mapper/exceptions.py b/motley_cue/mapper/exceptions.py index bcf0aea..b055df6 100644 --- a/motley_cue/mapper/exceptions.py +++ b/motley_cue/mapper/exceptions.py @@ -1,9 +1,13 @@ """exceptions definitions for motley_cue """ +from typing import Union from pydantic import ValidationError -from fastapi import Request -from fastapi.exceptions import HTTPException, RequestValidationError -from starlette.responses import JSONResponse +from fastapi.exceptions import ( + HTTPException, + RequestValidationError, + ResponseValidationError, +) +from fastapi.responses import JSONResponse from starlette.status import ( HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED, @@ -60,35 +64,24 @@ def __init__(self, exc: RequestValidationError): super().__init__(status_code=HTTP_400_BAD_REQUEST, content={"detail": message}) +class InvalidResponse(JSONResponse): + """Response for invalid response model. + + Returns HTTP 500 Internal Server Error status code + and informative message. + """ + + def __init__(self, exc: Union[ResponseValidationError, ValidationError]): + message = "Could not validate response model." + _ = exc + super().__init__( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, content={"detail": message} + ) + + class InternalException(Exception): """Wrapper for internal errors""" def __init__(self, message) -> None: self.message = message super().__init__(message) - - -async def validation_exception_handler(request: Request, exc: ValidationError): - """Replacement callback for handling RequestValidationError exceptions. - - :param request: request object that caused the RequestValidationError - :param exc: RequestValidationError containing validation errors - """ - _ = request - _ = exc - return JSONResponse( - status_code=HTTP_500_INTERNAL_SERVER_ERROR, - content={"detail": "Could not validate response model."}, - ) - - -async def request_validation_exception_handler( - request: Request, exc: RequestValidationError -): - """Replacement callback for handling RequestValidationError exceptions. - - :param request: request object that caused the RequestValidationError - :param exc: RequestValidationError containing validation errors - """ - _ = request - return MissingParameter(exc) From 2de10a73cd6271e506abc80a9531021089756212 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 28 Dec 2023 20:37:33 +0100 Subject: [PATCH 14/20] add docker image for report-status target in ci --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f11e633..28a47a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,7 +62,7 @@ integration-tests: variables: STATUS_PROJECT: dianagudu/motley_cue STATUS_NAME: Codebase CI - + image: debian:stable-slim script: # For complete details on the GitHub API please see: # https://developer.github.com/v3/repos/statuses From 2b004bc891b0e3618e99ba2e201bbb73ba5dd98c Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 28 Dec 2023 20:48:23 +0100 Subject: [PATCH 15/20] install curl in ci docker to report status --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 28a47a3..29b64ec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -62,11 +62,12 @@ integration-tests: variables: STATUS_PROJECT: dianagudu/motley_cue STATUS_NAME: Codebase CI - image: debian:stable-slim + image: alpine:latest script: # For complete details on the GitHub API please see: # https://developer.github.com/v3/repos/statuses - - 'curl -L + - 'apk --no-cache add curl && + curl -L -X POST -H "Accept: application/vnd.github+json" -H "Authorization: Bearer ${GH_STATUS_TOKEN}" From 297e96083f009ee2b602c916967edcf477d8f75f Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Fri, 29 Dec 2023 11:47:34 +0100 Subject: [PATCH 16/20] fix failure reporting status in ci --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 29b64ec..ff5a4f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -85,7 +85,7 @@ success: extends: - .report-status -failed: +failure: stage: .post extends: - .report-status From ef2db69c3100cbb47eaa37291c782aa58a33ca9c Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Mon, 8 Jan 2024 16:26:26 +0100 Subject: [PATCH 17/20] use github-status-sync from ci-voodoo --- .gitlab-ci.yml | 39 ++------------------------------------- 1 file changed, 2 insertions(+), 37 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff5a4f5..75e4df1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,14 +3,14 @@ include: - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/generic-ci.yml" - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/pipeline-jobs.yml" - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/pipeline-jobs-publish-to-repo.yml" - - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/github-status/ci-include/github-status-sync.yml" + - "https://codebase.helmholtz.cloud/m-team/tools/ci-voodoo/raw/master/ci-include/github-status-sync.yml" default: tags: - linux variables: - STATUS_PROJECT: dianagudu/motley_cue + UPSTREAM_PROJECT: dianagudu/motley_cue STAGING_BRANCH_NAME: "staging" DOCKER_IMAGE_NAMESPACE: "marcvs/build" DOCKER_IMAGE_NAME: "motley-cue" @@ -56,38 +56,3 @@ build-ubuntu-bionic: integration-tests: extends: - .trigger-integration-tests-ssh-oidc - -########################################################################## -.report-status: - variables: - STATUS_PROJECT: dianagudu/motley_cue - STATUS_NAME: Codebase CI - image: alpine:latest - script: - # For complete details on the GitHub API please see: - # https://developer.github.com/v3/repos/statuses - - 'apk --no-cache add curl && - curl -L - -X POST - -H "Accept: application/vnd.github+json" - -H "Authorization: Bearer ${GH_STATUS_TOKEN}" - -H "X-GitHub-Api-Version: 2022-11-28" - https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_COMMIT_SHA} - -d "{\"state\": \"${CI_JOB_NAME}\",\"context\": \"${STATUS_NAME}\", \"target_url\": \"${CI_PIPELINE_URL}\"}"' - -pending: - stage: .pre - extends: - - .report-status - -success: - stage: .post - extends: - - .report-status - -failure: - stage: .post - extends: - - .report-status - rules: - - when: on_failure From 4b0f0c0abe85bba587642812a11b608a184aa70f Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Fri, 12 Jan 2024 14:07:29 +0100 Subject: [PATCH 18/20] set upstream variables for downstream integration test --- .gitlab-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75e4df1..7876996 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,6 +28,9 @@ variables: # See generic-ci.yml -> .build-definition for the full list # DOCKER_IMAGE_VERSION # DOCKER_IMAGE_VERSION_WINDOWS + # These are needed to find the artifacts: + CI_UPSTREAM_JOB_ID: $CI_JOB_ID + CI_UPSTREAM_PROJECT_NAMESPACE: $CI_PROJECT_NAMESPACE build-centos-7: extends: From 21dcba6a7ba265ec9a329eab2d2bb5ac75f9bbde Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Fri, 12 Jan 2024 15:08:36 +0100 Subject: [PATCH 19/20] test ci --- .gitlab-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7876996..75e4df1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -28,9 +28,6 @@ variables: # See generic-ci.yml -> .build-definition for the full list # DOCKER_IMAGE_VERSION # DOCKER_IMAGE_VERSION_WINDOWS - # These are needed to find the artifacts: - CI_UPSTREAM_JOB_ID: $CI_JOB_ID - CI_UPSTREAM_PROJECT_NAMESPACE: $CI_PROJECT_NAMESPACE build-centos-7: extends: From 33b40671fb882c0ed7d769f15af7dfbdbbd558e5 Mon Sep 17 00:00:00 2001 From: Diana Gudu Date: Thu, 18 Jan 2024 13:25:53 +0100 Subject: [PATCH 20/20] fix gitlab script to set prerel version --- .gitlab-ci-scripts/set-prerelease-version.sh | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci-scripts/set-prerelease-version.sh b/.gitlab-ci-scripts/set-prerelease-version.sh index 06b6fae..93b874e 100755 --- a/.gitlab-ci-scripts/set-prerelease-version.sh +++ b/.gitlab-ci-scripts/set-prerelease-version.sh @@ -29,15 +29,29 @@ done # Get master branch name: # use origin if exists # else use last found remote -REMOTES=$(git remote show) -for R in $REMOTES; do - MASTER=$(git remote show "$R" 2>/dev/null \ - | sed -n '/HEAD branch/s/.*: //p') - MASTER_BRANCH="refs/remotes/${R}/${MASTER}" - #echo "Master-branch: ${MASTER_BRANCH}" - [ "x${R}" == "xorigin" ] && break -done +MASTER_BRANCH="" +get_master_branch_of_mteam() { + git remote -vv | awk -F[\\t@:] '{ print $1 " " $3 }' | while read REMOTE HOST; do + # echo " $HOST -- $REMOTE" + MASTER=$(git remote show "$REMOTE" 2>/dev/null \ + | sed -n '/HEAD branch/s/.*: //p') + MASTER_BRANCH="refs/remotes/${REMOTE}/${MASTER}" + [ "x${HOST}" == "xcodebase.helmholtz.cloud" ] && { + echo "${MASTER_BRANCH}" + break + } + [ "x${HOST}" == "xgit.scc.kit.edu" ] && { + echo "${MASTER_BRANCH}" + break + } + [ "x${REMOTE}" == "xorigin" ] && { + echo "${MASTER_BRANCH}" + break + } + done +} +MASTER_BRANCH=$(get_master_branch_of_mteam) PREREL=$(git rev-list --count HEAD ^"$MASTER_BRANCH") # if we use a version file, things are easy: