user: uses invenio access profile

* Replaces invenio-userprofile by the invenio access profile. * Updates several packages. * Moves `login_user_via_session` fonction in the tests as some problems occurs when a session merge is called between this function and the http request. * Removes useless babel mock in the tests. * Moves username from the invenio userprofile to the main user database. * Adds a record constructor for a record `replace_refs` as it is not done anymore by invenio. * Adds tests to check the existance of `_updated` and `_created` in the document index. * Removes document types for elasticsearch operations. * Adds marshmalow schema for user profile. * Uses node 18 for the production images. * Updates dependencies for Flask2. * Corrects problems from newer modules. * Closes #3405. Co-Authored-by: Peter Weber <peter.weber@rero.ch> Co-Authored-by: Johnny Mariéthoz <Johnny.Mariethoz@rero.ch>
rero · Dec 21, 2023 · 9338171 · 9338171
1 parent 0aef8e3
commit 9338171
Show file tree

Hide file tree

Showing 142 changed files with 2,060 additions and 1,810 deletions.
diff --git a/.github/workflows/continuous-integration-test.yml b/.github/workflows/continuous-integration-test.yml
@@ -13,18 +13,18 @@ jobs:
     #   with:
     #     access_token: ${{ github.token }}
 
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Setup node
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
-        node-version: '14'
+        node-version: '16'
 
     - name: Docker compose up
       run: docker-compose up -d
 
     - name: Set up Python 3.9
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
 

diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
@@ -1,7 +1,6 @@
 name: "PR Labeler"
 on:
   - pull_request_target
-
 jobs:
   triage:
     runs-on: ubuntu-latest

diff --git a/Dockerfile.base b/Dockerfile.base
@@ -25,13 +25,11 @@ RUN pip install --upgrade setuptools wheel pip poetry
 # # uwsgi uwsgitop uwsgi-tools
 
 # Install Node
-RUN curl -sL https://deb.nodesource.com/setup_14.x | bash -
-RUN apt-get install --no-install-recommends -y nodejs && rm -rf /var/lib/apt/lists/*
-
-# RUN npm update
-
-# RUN python -m site
-# RUN python -m site --user-site
+RUN apt-get update && apt-get install -y ca-certificates curl gnupg
+RUN mkdir -p /etc/apt/keyrings
+RUN curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg
+RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_18.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list
+RUN apt-get update && apt-get install nodejs -y
 
 # Install Invenio
 ENV WORKING_DIR=/invenio

diff --git a/data/users.json b/data/users.json
@@ -751,7 +751,7 @@
   },
   {
     "pid": "30",
-    "birth_date": "1800-01-01",
+    "birth_date": "1900-01-01",
     "city": "no indication",
     "email": "thelibrarian@example.org",
     "username": "thelibrarian",

diff --git a/docs/conf.py b/docs/conf.py
@@ -81,7 +81,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line for these cases.
-language = None
+language = 'english'
 
 # There are two options for replacing |today|: either, you set today to some
 # non-false value, then it is used:

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -30,22 +30,49 @@ uwsgitop = ">=0.11"
 
 ## Third party invenio modules used by RERO ILS
 invenio-oaiharvester = { git = "https://github.com/inveniosoftware/invenio-oaiharvester.git", tag = "v1.0.0a4" }
-invenio-circulation = { git = "https://github.com/inveniosoftware/invenio-circulation.git", tag = "v1.0.0a36" }
-
-## Invenio 3.4 base modules used by RERO ILS
-# same as invenio metadata extras without invenio-search-ui
-invenio-indexer = ">=1.2.0,<1.3.0"
-invenio-jsonschemas = ">=1.1.1,<1.2.0"
-invenio-pidstore = ">=1.2.1,<1.3.0"
-invenio-records-rest = ">=1.8.0,<1.9.0"
+invenio-circulation = {version = "^3.0.0a1", allow-prereleases = true}
+
+invenio-search = {version = ">=2.1.0,<3.0.0", extras = ["elasticsearch7"]}
+# Invenio core modules
+invenio-app = ">=1.3.4,<1.4.0"
+invenio-base = ">=1.2.16,<1.3.0"
+invenio-cache = ">=1.1.1,<1.2.0"
+invenio-celery = ">=1.2.5,<1.3.0"
+invenio-config = ">=1.0.3,<1.1.0"
+invenio-i18n = ">=2.0.0,<3.0.0"
+invenio-db = {version = ">=1.1.0,<1.2.0", extras = ["postgresql"]}
+# Invenio base bundle
+invenio-admin = ">=1.4.0,<1.5.0"
+# invenio-assets = ">=3.0.0,<4.0.0" Ç Error: Patch file found for package semantic-ui-less which is not present at node_modules/semantic-ui-less
+invenio-assets = ">=2.0.0,<3.0.0"
+invenio-formatter = ">=2.0.0,<3.0.0"
+invenio-logging = {version = ">=2.0.0,<3.0.0"}
+invenio-mail = ">=2.0.0,<3.0.0"
+invenio-rest = ">=1.3.0,<1.4.0"
+invenio-theme = ">=2.5.7,<3.0.0"
+# Invenio auth bundle
+invenio-access = ">=2.0.0,<3.0.0"
+invenio-accounts = ">=3.0.0,<4.0.0"
+invenio-oauth2server = ">=2.0.0,<3.0.0"
+invenio-oauthclient = ">=3.0.0,<4.0.0"
+# Invenio metadata bundle
+invenio-indexer = ">=2.2.0,<3.0.0"
+invenio-jsonschemas = ">=1.1.4,<1.2.0"
+invenio-oaiserver = ">=2.2.0,<2.3.0"
+invenio-pidstore = ">=1.3.0,<1.4.0"
+invenio-records-rest = ">=2.2.0,<2.3.0"
 invenio-records-ui = ">=1.2.0,<1.3.0"
-invenio-records = ">=1.4.0,<1.7.0"
+invenio-records = ">=2.1.0,<2.3.0"
+invenio-search-ui = ">=2.4.0,<3.0.0"
 
-## Default from Invenio
-invenio = {version = ">=3.4.0,<3.5.0", extras = ["base", "postgresql", "auth", "elasticsearch7", "docs", "tests" ]}
+invenio-records-permissions = ">=0.13.0"
 
-# TODO: jsonschema 4.0.0 is not working with invenio because there is a new parameter type!
-jsonschema = "<4.0.0"
+# Pinned due to before_first_request deprecation https://flask.palletsprojects.com/en/2.2.x/api/#flask.Flask.before_first_request
+Flask = ">=2.2.0,<2.3.0"
+sentry-sdk = ">=1.0.0" # normaly in invenio-logging = {version = ">=2.0.0,<3.0.0", extras = ["sentry_sdk"]}
+dojson = ">=1.4.0"
+# TODO: dojson problem = AttributeError: 'Group' object has no attribute 'resultcallback'
+click = "<8.1.0"
 
 ## RERO ILS specific python modules
 PyYAML = ">=5.3.1"
@@ -57,63 +84,44 @@ xmltodict = "*"
 redisbeat = ">1.2.5, <1.3.0"
 jsonpickle = ">=1.4.1"
 ciso8601 = "*"
-# TODO: to be removed when the thumbnail will be refactored
-invenio-userprofiles = {git = "https://github.com/rero/invenio-userprofiles.git", tag = "v1.2.1-rero2.0"}
 
 ## Additionnal constraints on python modules
 markdown-captions = "*"
 zipp = "*"
 
 ## Deployment
-# sentry
-invenio-logging = { version = ">=1.3.0,<1.4.0", extras = ["sentry-sdk", "sentry"] }
 python-dotenv = ">=0.13.0"
+pydocstyle = ">=6.1.1"
 
 ## Third party optional modules used by RERO ILS
-invenio-sip2 = ">=0.6.16"
-flask-cors = ">3.0.8"
-celery = ">=5.0.0"
-cryptography = ">38.0.2,<40.0"
+invenio-sip2 = ">=0.6.22"
 freezegun = "^1.1.0"
 lazyreader = ">1.0.0"
 jinja2 = ">2.11.2"
 jsonmerge = "^1.8.0"
 num2words = "^0.5.10"
-Flask = "<2.0.0"
 iso639 = "^0.1.4"
 dcxml = "^0.1.2"
 DeepDiff = "^5.5.0"
 docutils = "<0.18.0"
 wtforms = "<3.0.0"
-flask-wtf = "<1.0.0"
-# TODO: ImportError: cannot import name 'soft_unicode' from 'markupsafe'
-markupsafe = "<2.1.0"
 poethepoet = "^0.12.3"
-flask-wiki = "^0.2.2"
-pytest-invenio = ">=1.4.0,<1.4.12"
 # to avoid conflict for urllib3
-sentry-sdk = "<1.6.1"
 dparse = ">=0.5.2"
 Mako = ">=1.2.2"
-rero-invenio-base = "^0.2.1"
-jsonref = "<1.0.0"
-dojson = "^1.4.0"
-jsonresolver = "<0.3.2"
-pydocstyle = ">=6.1.1,<6.2"
-# to solve: pytz
-setuptools = "<67.0.0"
-invenio-records-permissions = "^0.13.0"
+rero-invenio-base = "^0.3.0"
+jsonresolver = "*"
 # needed for elasticsearch 7.13.4
 urllib3 = "<2.0.0"
+pyparsing = "^3.1.1"
+flask-wiki = "^0.3.1"
 
 [tool.poetry.dev-dependencies]
 ## Python packages development dependencies (order matters)
 #----------------------------------------------------------
 ## Default from Invenio
-pytest-invenio = ">=1.4.0,<1.5.0"
-# TODO: ther is a problem with sphinx 3.4.2
-# RuntimeError: Working outside of application context.
-Sphinx = ">=3,<3.4.2"
+pytest-invenio = ">=2.1.6,<3.0.0"
+Sphinx = ">=4.5.0"
 Flask-Debugtoolbar = ">=0.10.1"
 ## RERO ILS specific python packages
 safety = ">=1.8"
@@ -176,7 +184,6 @@ notifications = "rero_ils.modules.notifications.views:blueprint"
 patron_types = "rero_ils.modules.patron_types.views:blueprint"
 patrons = "rero_ils.modules.patrons.views:api_blueprint"
 permissions ="rero_ils.modules.views:api_blueprint"
-rero_ils = "rero_ils.modules.views:api_blueprint"
 users = "rero_ils.modules.users.api_views:api_blueprint"
 stats_cfg = "rero_ils.modules.stats_cfg.views:api_blueprint"
 
@@ -194,7 +201,6 @@ item_types = "rero_ils.modules.item_types.views:blueprint"
 notifications = "rero_ils.modules.notifications.views:blueprint"
 patron_types = "rero_ils.modules.patron_types.views:blueprint"
 patrons = "rero_ils.modules.patrons.views:blueprint"
-rero_ils = "rero_ils.theme.views:blueprint"
 stats = "rero_ils.modules.stats.views:blueprint"
 templates = "rero_ils.modules.templates.views:blueprint"
 theme = "rero_ils.theme.views:blueprint"

diff --git a/rero_ils/accounts_views.py b/rero_ils/accounts_views.py
@@ -20,7 +20,7 @@
 from flask import abort, after_this_request, current_app
 from flask import request as flask_request
 from flask.views import MethodView
-from flask_babelex import gettext as _
+from flask_babel import gettext as _
 from flask_security.confirmable import requires_confirmation
 from flask_security.utils import get_message, verify_and_update_password
 from invenio_accounts.utils import change_user_password

diff --git a/rero_ils/alembic/add75cbcad66_migrate_obsolete_country_codes.py b/rero_ils/alembic/add75cbcad66_migrate_obsolete_country_codes.py
@@ -21,12 +21,15 @@
 from logging import getLogger
 
 from invenio_db import db
-from invenio_userprofiles.models import UserProfile
 
 from rero_ils.dojson.utils import _OBSOLETE_COUNTRIES_MAPPING
 from rero_ils.modules.documents.api import Document, DocumentsSearch
 from rero_ils.modules.patrons.api import Patron, PatronsSearch
 
+# TODO: Flask2
+# from invenio_userprofiles.models import UserProfile
+
+
 # revision identifiers, used by Alembic.
 revision = 'add75cbcad66'
 down_revision = 'e3eb396b39bb'
@@ -94,10 +97,10 @@ def fix_users(query, old_country, new_country):
             LOGGER.info(f'Found {len(ptrn_pids)} patrons with {old_country}.')
             fix_patrons(ptrn_pids, old_country, new_country)
 
-        query = UserProfile.query.filter_by(country=old_country)
-        if query.count() > 0:
-            LOGGER.info(f'Found {query.count()} users with {old_country}.')
-            fix_users(query, old_country, new_country)
+        # query = UserProfile.query.filter_by(country=old_country)
+        # if query.count() > 0:
+        #     LOGGER.info(f'Found {query.count()} users with {old_country}.')
+        #     fix_users(query, old_country, new_country)
 
 
 def downgrade():