Merge branch 'main' into bugfix/render-cache-collision

.editorconfig

@@ -12,11 +12,20 @@ spaces_around_operators = true
 [*.bat]
 end_of_line = crlf
 
-[*.{py,rst,sh}]
+[*.py]
 indent_size = 4
 
-[*.{js}]
+[*.rst]
+indent_size = 3
+
+[*.js]
 quote_type = double
 
 [*.{markdown,md}]
 trim_trailing_whitespace = false
+
+[[shell]]
+indent_style = space
+indent_size = 4
+space_redirects = true
+simplify = true

.github/matchers/mypy.json

@@ -0,0 +1,18 @@
+{
+  "problemMatcher": [
+    {
+      "owner": "mypy",
+      "pattern": [
+        {
+          "regexp": "^([^:]*):(\\d+):(?:(\\d+):)? ([^:]*): (.*?)(?:  \\[(\\S+)\\])?$",
+          "file": 1,
+          "line": 2,
+          "column": 3,
+          "severity": 4,
+          "message": 5,
+          "code": 6
+        }
+      ]
+    }
+  ]
+}

.github/matchers/mypy.json.license

@@ -0,0 +1,5 @@
+Copyright © Michal Čihař <michal@weblate.org>
+
+SPDX-License-Identifier: CC0-1.0
+
+This file is maintained in https://github.com/WeblateOrg/meta/

.github/workflows/mypy.yml

@@ -21,6 +21,28 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 2
+    - name: Get changed files
+      if: github.event_name == 'pull_request'
+      id: changed-files
+      uses: tj-actions/changed-files@v45
+      with:
+        files: '**.py'
+    - name: List all changed files
+      if: github.event_name == 'pull_request'
+      env:
+        ALL_CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }}
+      run: |
+        for file in ${ALL_CHANGED_FILES}; do
+          echo "$file" >> changed-files.txt
+        done
+        if [ ! -f changed-files.txt ] ; then
+          echo "No changed files!"
+        else
+          cat changed-files.txt
+        fi
+
     - name: Install apt dependencies
       run: sudo ./ci/apt-install
     - uses: astral-sh/setup-uv@v3
@@ -36,9 +58,16 @@ jobs:
       run: ./ci/pip-install mypy
 
     - name: Run mypy
-      uses: liskin/gh-problem-matcher-wrap@v3
-      with:
-        linters: mypy
-        run: mypy --show-column-numbers weblate
+      run: mypy --show-column-numbers weblate > mypy.log
       # Temporary hack until we have this fully working
       continue-on-error: true
+    - name: Report mypy
+      if: always()
+      run: |
+        echo "::add-matcher::.github/matchers/mypy.json"
+        if [ -f changed-files.txt ] ; then
+          grep -F -f changed-files.txt mypy.log
+        else
+          cat mypy.log
+        fi
+        echo "::remove-matcher owner=mypy::"

.github/workflows/scorecards.yml

@@ -61,6 +61,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
     - name: Upload to code-scanning
-      uses: github/codeql-action/upload-sarif@v3.26.10
+      uses: github/codeql-action/upload-sarif@v3.26.11
       with:
         sarif_file: results.sarif

.github/workflows/setup.yml

@@ -107,9 +107,8 @@ jobs:
     - name: Install Python dependencies
       run: |
         uv pip install --system $(sed -n 's/.*"\(twine==\([^"]*\)\)".*/\1/p' pyproject.toml)
-        uv pip install --system $(sed -n 's/.*"\(build==\([^"]*\)\)".*/\1/p' pyproject.toml)
     - name: build
-      run: python -m build
+      run: uv build
     - name: Twine check
       run: twine check dist/*
     - name: Sign the dists with Sigstore
@@ -173,8 +172,8 @@ jobs:
     - name: Cleanup dist
       # Remove files not supported on PyPI (eg. Sigstore signatures)
       run: find dist -mindepth 1 -not -name '*.tar.gz' -not -name '*.whl' -delete
-    - name: Publish package distributions to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
+    - uses: astral-sh/setup-uv@v3
+    - run: uv publish --trusted-publishing always
 
   publish_github:
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/weblate')

.github/workflows/test.yml

@@ -51,6 +51,11 @@ jobs:
           requirements: edge
           database: postgresql
           experimental: true
+        # Python pre-release
+        - python-version: '3.13'
+          requirements: latest
+          database: postgresql
+          experimental: true
     name: py${{ matrix.python-version }}, ${{ matrix.database }}, ${{ matrix.requirements }} deps
     env:
       CI_DATABASE: ${{ matrix.database }}
@@ -78,6 +83,7 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
+        allow-prereleases: true
     - name: Used versions
       run: ./ci/print-versions
     - name: Install Python dependencies

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
   hooks:
   - id: yamllint
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.6.8
+  rev: v0.6.9
   hooks:
   - id: ruff
     args: [--fix, --exit-non-zero-on-fix]
@@ -56,6 +56,10 @@ repos:
     - mdformat-ruff==0.1.3
     - mdformat-shfmt==0.1.0
     - mdformat_tables==1.0.0
+- repo: https://github.com/scop/pre-commit-shfmt
+  rev: v3.9.0-1
+  hooks:
+  - id: shfmt
 - repo: https://github.com/awebdeveloper/pre-commit-stylelint
   rev: 0.0.2
   hooks:

ci/apt-install

@@ -48,8 +48,12 @@ apt-get install -y \
     gir1.2-rsvg-2.0 \
     gir1.2-pango-1.0
 
+# TODO: remove this once Python 3.13 wheels are available, see
+# https://github.com/sirfz/tesserocr/pull/357
+apt-get install -y libleptonica-dev libtesseract-dev
+
 # Remove MySQL 8.x client, use older MariaDB one to ensure compatibility
-if [ "$CI_DATABASE" = "mariadb" ] ; then
+if [ "$CI_DATABASE" = "mariadb" ]; then
     apt-get purge 'mysql-client.*'
     apt-get install -y mariadb-client
 fi

ci/docker-compose-postgresql.yml

@@ -8,7 +8,7 @@ services:
     ports:
     - 60000:5432
     # make postgres faster and non-durable, see https://www.postgresql.org/docs/current/non-durability.html
-    command: [postgres, -c, fsync=off, -c, synchronous_commit=off, -c, full_page_writes=off, -c, checkpoint_timeout=3600, -c, max_wal_size=4096]
+    command: [postgres, -c, fsync=off, -c, synchronous_commit=off, -c, full_page_writes=off, -c, checkpoint_timeout=7200, -c, wal_level=minimal, -c, max_wal_senders=0]
     environment:
       POSTGRES_USER: postgres
       POSTGRES_PASSWORD: weblate

ci/lib.sh

@@ -6,13 +6,13 @@
 
 # shellcheck shell=sh
 
-if [ -z "$TERM" ] ; then
+if [ -z "$TERM" ]; then
     export TERM=xterm-256color
 fi
 
 check() {
     RET=$?
-    if [ $RET -ne 0 ] ; then
+    if [ $RET -ne 0 ]; then
         exit $RET
     fi
 }
@@ -24,7 +24,7 @@ run_coverage() {
 get_mysql_args() {
     # shellcheck disable=SC2153
     args="--host=$CI_DB_HOST --user=root"
-    if [ -n "$CI_DB_PORT" ] ; then
+    if [ -n "$CI_DB_PORT" ]; then
         args="$args --port=$CI_DB_PORT"
     fi
     echo "$args"
@@ -33,8 +33,8 @@ get_mysql_args() {
 cleanup_database() {
     rm -f weblate.db
 
-    if [ "$CI_DATABASE" = "mysql" ] || [ "$CI_DATABASE" = 'mariadb' ] ; then
-        if [ -n "$CI_DB_PASSWORD" ] ; then
+    if [ "$CI_DATABASE" = "mysql" ] || [ "$CI_DATABASE" = 'mariadb' ]; then
+        if [ -n "$CI_DB_PASSWORD" ]; then
             export MYSQL_PWD="$CI_DB_PASSWORD"
         fi
         # shellcheck disable=SC2046
@@ -47,11 +47,11 @@ cleanup_database() {
         mysql $(get_mysql_args) -e 'CREATE DATABASE weblate CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;'
     fi
 
-    if [ "$CI_DATABASE" = "postgresql" ] ; then
-        if [ -n "$CI_DB_PASSWORD" ] ; then
+    if [ "$CI_DATABASE" = "postgresql" ]; then
+        if [ -n "$CI_DB_PASSWORD" ]; then
             export PGPASSWORD="$CI_DB_PASSWORD"
         fi
-        if [ -n "$CI_DB_PORT" ] ; then
+        if [ -n "$CI_DB_PORT" ]; then
             export PGPORT="$CI_DB_PORT"
         fi
         psql --host="$CI_DB_HOST" -c 'DROP DATABASE IF EXISTS weblate;' -U postgres
@@ -66,8 +66,8 @@ print_step() {
 }
 
 print_version() {
-    for cmd in "$@" ; do
-        if command -v "$cmd" ; then
+    for cmd in "$@"; do
+        if command -v "$cmd"; then
             $cmd --version
             return
         fi

ci/pip-install

@@ -6,20 +6,26 @@
 
 set -e -x
 
-if [ "${1:-latest}" = migrations ] ; then
+if [ "${1:-latest}" = migrations ]; then
     uv pip install --system -e ".[all,mysql,ci]"
 else
-    if [ "${1:-latest}" = minimal ] ; then
+    if [ "${1:-latest}" = minimal ]; then
         # Adjust deps to force minimal version
         sed -i '/^ *"/ s/>=/==/' pyproject.toml
     fi
 
-    if [ "${1:-latest}" = mypy ] ; then
-      uv pip install --system -e ".[all,mysql,ci,mypy,test]"
+    if [ "${1:-latest}" = mypy ]; then
+        uv pip install --system -e ".[all,mysql,ci,mypy,test]"
     else
-      uv pip install --system -e ".[all,mysql,ci,test]"
+        # TODO: lxml can use wheels once xmlsec has one
+        # see https://github.com/xmlsec/python-xmlsec/issues/327
+        if python -c 'import sys; sys.exit(0 if sys.version_info >= (3,13) else 1)'; then
+            uv pip install --system --no-binary=lxml -e ".[all,mysql,ci,test]"
+        else
+            uv pip install --system -e ".[all,mysql,ci,test]"
+        fi
     fi
-    if [ "${1:-latest}" = edge ] ; then
+    if [ "${1:-latest}" = edge ]; then
         uv pip install --system --upgrade -e ".[all,mysql,test,ci]"
         # Install from git / pre-release
         uv pip install --system --no-deps --upgrade --force-reinstall \
@@ -31,6 +37,6 @@ else
 fi
 
 # Verify that deps are consistent
-if [ "${1:-latest}" != edge ] ; then
+if [ "${1:-latest}" != edge ]; then
     uv pip check
 fi

ci/prepare-database

@@ -10,15 +10,15 @@ set -e
 
 cleanup_database
 
-if [ "$CI_DATABASE" = "mysql" ] || [ "$CI_DATABASE" = 'mariadb' ] ; then
+if [ "$CI_DATABASE" = "mysql" ] || [ "$CI_DATABASE" = 'mariadb' ]; then
     # shellcheck disable=SC2046
     mysql $(get_mysql_args) -e 'SHOW VARIABLES LIKE "%version%";'
 fi
-if [ "$CI_DATABASE" = "postgresql" ] ; then
-    if [ -n "$CI_DB_PASSWORD" ] ; then
+if [ "$CI_DATABASE" = "postgresql" ]; then
+    if [ -n "$CI_DB_PASSWORD" ]; then
         export PGPASSWORD="$CI_DB_PASSWORD"
     fi
-    if [ -n "$CI_DB_PORT" ] ; then
+    if [ -n "$CI_DB_PORT" ]; then
         export PGPORT="$CI_DB_PORT"
     fi
     psql --host="$CI_DB_HOST" -c 'SELECT version();' -U postgres

ci/run-checkmigrate

@@ -12,9 +12,9 @@ cleanup_database
 run_coverage ./manage.py migrate
 run_coverage ./manage.py makemigrations
 check
-if [ "$(git status -s -- '*/migrations/*' | wc -l)" -gt 0 ] ; then
+if [ "$(git status -s -- '*/migrations/*' | wc -l)" -gt 0 ]; then
     echo 'There are untracked migrations:'
-    git status -s --porcelain -- '*/migrations/*' | sed -n '/^??/ s/^?? \(.*\)/\1/p' | while read -r migration ; do
+    git status -s --porcelain -- '*/migrations/*' | sed -n '/^??/ s/^?? \(.*\)/\1/p' | while read -r migration; do
         echo "$migration"
         cat "$migration"
         echo

ci/run-docs

@@ -6,12 +6,10 @@
 
 set -e
 
-if ! command -v dot &> /dev/null
-then
+if ! command -v dot &> /dev/null; then
     echo "dot command could not be found, please install graphviz"
     exit 1
 fi
 
-
 # Build documentation with warnings treated as errors
 make -C docs "${1:-html}" SPHINXOPTS='-n -W -a --keep-going'

ci/run-migrate

@@ -8,7 +8,7 @@
 
 . ci/lib.sh
 
-if [ -n "$1" ] ; then
+if [ -n "$1" ]; then
     TAG="weblate-$1"
 else
     echo "Missing version to migrate from!"
@@ -20,7 +20,7 @@ HEAD_COMMIT=$(git rev-parse HEAD)
 print_step "Testing migration from $TAG on $CI_DATABASE..."
 cleanup_database
 check
-if ! git fetch --depth 1 origin tag "$TAG" ; then
+if ! git fetch --depth 1 origin tag "$TAG"; then
     git remote add upstream https://github.com/WeblateOrg/weblate.git
     git fetch --depth 1 upstream tag "$TAG"
 fi
@@ -36,11 +36,11 @@ uv pip install -e ".[all,mysql]"
 check
 echo "DATABASES['default']['HOST'] = '$CI_DB_HOST'" >> weblate/settings_test.py
 check
-if [ -n "$CI_DB_PASSWORD" ] ; then
+if [ -n "$CI_DB_PASSWORD" ]; then
     echo "DATABASES['default']['PASSWORD'] = '$CI_DB_PASSWORD'" >> weblate/settings_test.py
     check
 fi
-if [ -n "$CI_DB_PORT" ] ; then
+if [ -n "$CI_DB_PORT" ]; then
     echo "DATABASES['default']['PORT'] = '$CI_DB_PORT'" >> weblate/settings_test.py
     check
 fi
@@ -77,7 +77,6 @@ check
 ./manage.py shell -c 'from weblate.memory.models import Memory; from weblate.lang.models import Language; Memory.objects.create(source_language=Language.objects.get(code="en"), target_language=Language.objects.get(code="cs"), source="source"*1000, target="target"*1000, origin="origin"*1000)'
 check
 
-
 git reset --hard
 check
 git checkout "$HEAD_COMMIT"

ci/services-down

@@ -6,7 +6,7 @@
 
 set -e
 
-if [ ! -f services-down ] ; then
+if [ ! -f services-down ]; then
     cd ci
 fi
 

ci/services-up

@@ -6,7 +6,7 @@
 
 set -e
 
-if [ ! -f services-up ] ; then
+if [ ! -f services-up ]; then
     cd ci
 fi