Merge The Palace Circulation v15.0.1 into E-kirjasto

.github/workflows/lint.yml

@@ -28,7 +28,7 @@ jobs:
           POETRY_VIRTUALENVS_CREATE: false
 
       - name: Restore pre-commit cache
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pre-commit
           key: pre-commit-${{ runner.os }}-py${{ env.PYTHON_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}

.github/workflows/test-build.yml

@@ -175,7 +175,7 @@ jobs:
       # using these changes throughout the rest of the build. If the base image
       # build wasn't changed, we don't use it and just rely on scheduled build.
       - name: Check if base image was changed by this branch
-        uses: dorny/paths-filter@v2
+        uses: dorny/paths-filter@v3
         id: changes
         with:
           filters: |
@@ -273,18 +273,6 @@ jobs:
       matrix:
         platform: ["linux/amd64", "linux/arm64"]
         image: ["scripts", "webapp"]
-    env:
-      POSTGRES_USER: palace_user
-      POSTGRES_PASSWORD: test
-      POSTGRES_DB: palace_circulation
-
-    services:
-      postgres:
-        image: postgres:12
-        env:
-          POSTGRES_USER: ${{ env.POSTGRES_USER }}
-          POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }}
-          POSTGRES_DB: ${{ env.POSTGRES_DB }}
 
     steps:
       - uses: actions/checkout@v4
@@ -301,36 +289,23 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build image
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./docker/Dockerfile
-          tags: test_image
-          load: true
-          target: ${{ matrix.image }}
-          cache-from: type=gha,scope=buildkit-${{ github.run_id }}
-          platforms: ${{ matrix.platform }}
-          build-args: |
-            BASE_IMAGE=${{ needs.docker-image-build.outputs.baseimage }}
-
-      - name: Start container
-        run: >
-          docker run --rm --name test_container -d --platform ${{ matrix.platform }}
-          --network ${{job.services.postgres.network}}
-          -e SIMPLIFIED_PRODUCTION_DATABASE="postgresql://${{ env.POSTGRES_USER }}:${{ env.POSTGRES_PASSWORD }}@postgres:5432/${{ env.POSTGRES_DB }}"
-          test_image
+      - name: Build & Start container
+        run: docker compose up -d --build ${{ matrix.image }}
+        env:
+          BUILD_PLATFORM: ${{ matrix.platform }}
+          BUILD_CACHE_FROM: type=gha,scope=buildkit-${{ github.run_id }}
+          BUILD_BASE_IMAGE: ${{ needs.docker-image-build.outputs.baseimage }}
 
       - name: Run tests
-        run: ./docker/ci/test_${{ matrix.image }}.sh test_container
+        run: ./docker/ci/test_${{ matrix.image }}.sh ${{ matrix.image }}
 
       - name: Output logs
         if: failure()
-        run: docker logs test_container
+        run: docker logs circulation-${{ matrix.image }}-1
 
       - name: Stop container
         if: always()
-        run: docker stop test_container
+        run: docker compose down
 
   docker-image-push:
     if: false # Disable temporarily

README.md

@@ -124,7 +124,7 @@ Elasticsearch is no longer supported.
 We recommend that you run OpenSearch with docker using the following docker commands:
 
 ```sh
-docker run --name opensearch -d --rm -p 9006:9200 -e "discovery.type=single-node" -e "plugins.security.disabled=true" "opensearchproject/opensearch:1"
+docker run --name opensearch -d --rm -p 9200:9200 -e "discovery.type=single-node" -e "plugins.security.disabled=true" "opensearchproject/opensearch:1"
 docker exec opensearch opensearch-plugin -s install analysis-icu
 docker restart opensearch
 ```
@@ -165,6 +165,22 @@ To let the application know which database to use, set the `SIMPLIFIED_PRODUCTIO
 export SIMPLIFIED_PRODUCTION_DATABASE="postgresql://palace:test@localhost:5432/circ"
 ```
 
+#### Opensearch
+
+To let the application know which Opensearch instance to use, you can set the following environment variables:
+
+- `PALACE_SEARCH_URL`: The url of the Opensearch instance (**required**).
+- `PALACE_SEARCH_INDEX_PREFIX`: The prefix to use for the Opensearch indices. The default is `circulation-works`.
+    This is useful if you want to use the same Opensearch instance for multiple CM (optional).
+- `PALACE_SEARCH_TIMEOUT`: The timeout in seconds to use when connecting to the Opensearch instance. The default is `20`
+  (optional).
+- `PALACE_SEARCH_MAXSIZE`: The maximum size of the connection pool to use when connecting to the Opensearch instance.
+  (optional).
+
+```sh
+export PALACE_SEARCH_URL="http://localhost:9200"
+```
+
 #### Storage Service
 
 The application optionally uses a s3 compatible storage service to store files. To configure the application to use
@@ -656,7 +672,7 @@ If you already have elastic search or postgres running locally, you can run them
 following environment variables:
 
 - `SIMPLIFIED_TEST_DATABASE`
-- `SIMPLIFIED_TEST_OPENSEARCH`
+- `PALACE_TEST_SEARCH_URL`
 
 Make sure the ports and usernames are updated to reflect the local configuration.
 

alembic/versions/20240124_993729d4bf97_migrate_metadata_services.py

@@ -0,0 +1,94 @@
+"""migrate metadata services
+
+Revision ID: 993729d4bf97
+Revises: 735bf6ced8b9
+Create Date: 2024-01-24 23:51:13.464107+00:00
+
+"""
+from alembic import op
+from api.integration.registry.metadata import MetadataRegistry
+from core.integration.base import HasLibraryIntegrationConfiguration
+from core.migration.migrate_external_integration import (
+    _migrate_external_integration,
+    _migrate_library_settings,
+    get_configuration_settings,
+    get_integrations,
+    get_library_for_integration,
+)
+from core.migration.util import pg_update_enum
+
+# revision identifiers, used by Alembic.
+revision = "993729d4bf97"
+down_revision = "735bf6ced8b9"
+branch_labels = None
+depends_on = None
+
+METADATA_GOAL = "METADATA_GOAL"
+old_goals_enum = ["PATRON_AUTH_GOAL", "LICENSE_GOAL", "DISCOVERY_GOAL", "CATALOG_GOAL"]
+new_goals_enum = old_goals_enum + [METADATA_GOAL]
+
+
+def upgrade() -> None:
+    # Add the new enum value to our goals enum
+    pg_update_enum(
+        op,
+        "integration_configurations",
+        "goal",
+        "goals",
+        old_goals_enum,
+        new_goals_enum,
+    )
+
+    # Migrate the existing metadata services to integration configurations
+    connection = op.get_bind()
+    registry = MetadataRegistry()
+    integrations = get_integrations(connection, "metadata")
+    for integration in integrations:
+        _id, protocol, name = integration
+        protocol_class = registry[protocol]
+
+        (
+            settings_dict,
+            libraries_settings,
+            self_test_result,
+        ) = get_configuration_settings(connection, integration)
+
+        updated_protocol = registry.get_protocol(protocol_class)
+        if updated_protocol is None:
+            raise RuntimeError(f"Unknown metadata service '{protocol}'")
+        integration_configuration_id = _migrate_external_integration(
+            connection,
+            integration.name,
+            updated_protocol,
+            protocol_class,
+            METADATA_GOAL,
+            settings_dict,
+            self_test_result,
+        )
+
+        integration_libraries = get_library_for_integration(connection, _id)
+        for library in integration_libraries:
+            if issubclass(protocol_class, HasLibraryIntegrationConfiguration):
+                _migrate_library_settings(
+                    connection,
+                    integration_configuration_id,
+                    library.library_id,
+                    libraries_settings[library.library_id],
+                    protocol_class,
+                )
+            else:
+                raise RuntimeError(
+                    f"Protocol not expected to have library settings '{protocol}'"
+                )
+
+
+def downgrade() -> None:
+    # Remove the new enum value from our goals enum.
+    pg_update_enum(
+        op,
+        "integration_configurations",
+        "goal",
+        "goals",
+        new_goals_enum,
+        old_goals_enum,
+    )

api/admin/config.py

@@ -16,7 +16,7 @@ class OperationalMode(str, Enum):
 class Configuration(LoggerMixin):
     APP_NAME = "E-kirjasto Collection Manager"
     PACKAGE_NAME = "@natlibfi/ekirjasto-circulation-admin"
-    PACKAGE_VERSION = "0.0.1"
+    PACKAGE_VERSION = "0.0.1-post.8"
 
     STATIC_ASSETS = {
         "admin_js": "circulation-admin.js",

api/admin/controller/__init__.py

@@ -13,7 +13,6 @@ def setup_admin_controllers(manager: CirculationManager):
     from api.admin.controller.admin_search import AdminSearchController
     from api.admin.controller.announcement_service import AnnouncementSettings
     from api.admin.controller.catalog_services import CatalogServicesController
-    from api.admin.controller.collection_self_tests import CollectionSelfTestsController
     from api.admin.controller.collection_settings import CollectionSettingsController
     from api.admin.controller.custom_lists import CustomListsController
     from api.admin.controller.dashboard import DashboardController
@@ -27,26 +26,11 @@ def setup_admin_controllers(manager: CirculationManager):
     )
     from api.admin.controller.lanes import LanesController
     from api.admin.controller.library_settings import LibrarySettingsController
-    from api.admin.controller.metadata_service_self_tests import (
-        MetadataServiceSelfTestsController,
-    )
     from api.admin.controller.metadata_services import MetadataServicesController
     from api.admin.controller.patron import PatronController
-    from api.admin.controller.patron_auth_service_self_tests import (
-        PatronAuthServiceSelfTestsController,
-    )
     from api.admin.controller.patron_auth_services import PatronAuthServicesController
     from api.admin.controller.reset_password import ResetPasswordController
-    from api.admin.controller.search_service_self_tests import (
-        SearchServiceSelfTestsController,
-    )
-    from api.admin.controller.self_tests import SelfTestsController
-    from api.admin.controller.settings import SettingsController
     from api.admin.controller.sign_in import SignInController
-    from api.admin.controller.sitewide_services import (
-        SearchServicesController,
-        SitewideServicesController,
-    )
     from api.admin.controller.sitewide_settings import (
         SitewideConfigurationSettingsController,
     )
@@ -63,41 +47,25 @@ def setup_admin_controllers(manager: CirculationManager):
     manager.admin_custom_lists_controller = CustomListsController(manager)
     manager.admin_lanes_controller = LanesController(manager)
     manager.admin_dashboard_controller = DashboardController(manager)
-    manager.admin_settings_controller = SettingsController(manager)
     manager.admin_patron_controller = PatronController(manager)
-    manager.admin_self_tests_controller = SelfTestsController(manager)
     manager.admin_discovery_services_controller = DiscoveryServicesController(manager)
     manager.admin_discovery_service_library_registrations_controller = (
         DiscoveryServiceLibraryRegistrationsController(manager)
     )
     manager.admin_metadata_services_controller = MetadataServicesController(manager)
-    manager.admin_metadata_service_self_tests_controller = (
-        MetadataServiceSelfTestsController(manager)
-    )
     manager.admin_patron_auth_services_controller = PatronAuthServicesController(
         manager
     )
 
-    manager.admin_patron_auth_service_self_tests_controller = (
-        PatronAuthServiceSelfTestsController(manager._db)
-    )
     manager.admin_collection_settings_controller = CollectionSettingsController(manager)
-    manager.admin_collection_self_tests_controller = CollectionSelfTestsController(
-        manager._db
-    )
     manager.admin_sitewide_configuration_settings_controller = (
         SitewideConfigurationSettingsController(manager)
     )
     manager.admin_library_settings_controller = LibrarySettingsController(manager)
     manager.admin_individual_admin_settings_controller = (
-        IndividualAdminSettingsController(manager)
-    )
-    manager.admin_sitewide_services_controller = SitewideServicesController(manager)
-    manager.admin_search_service_self_tests_controller = (
-        SearchServiceSelfTestsController(manager)
+        IndividualAdminSettingsController(manager._db)
     )
-    manager.admin_search_services_controller = SearchServicesController(manager)
     manager.admin_catalog_services_controller = CatalogServicesController(manager)
-    manager.admin_announcement_service = AnnouncementSettings(manager)
+    manager.admin_announcement_service = AnnouncementSettings(manager._db)
     manager.admin_search_controller = AdminSearchController(manager)
     manager.admin_quicksight_controller = QuickSightController(manager)

api/admin/controller/announcement_service.py

@@ -4,18 +4,21 @@
 from typing import Any
 
 import flask
+from sqlalchemy.orm import Session
 
 from api.admin.announcement_list_validator import AnnouncementListValidator
-from api.admin.controller.settings import SettingsController
 from api.config import Configuration
 from core.model.announcements import Announcement
 from core.problem_details import INVALID_INPUT
 from core.util.problem_detail import ProblemDetail, ProblemError
 
 
-class AnnouncementSettings(SettingsController):
+class AnnouncementSettings:
     """Controller that manages global announcements for all libraries"""
 
+    def __init__(self, db: Session) -> None:
+        self._db = db
+
     def _action(self) -> Callable:
         method = flask.request.method.lower()
         return getattr(self, method)