Geopackage support (#55)

* Remove ConnectionNode.the_geom_linestring
* Enable reading of geopackage, besides spatialite

---------

Co-authored-by: jpprins1 <jpprins1@gmail.com>

.github/workflows/test.yml

@@ -18,19 +18,24 @@ jobs:
           # 2019
           - python: 3.8
             os: ubuntu-20.04
-            pins: "sqlalchemy==1.4.0 alembic==1.8.* geoalchemy2==0.9.*"
+            pins: "sqlalchemy==1.4.44 alembic==1.8.* geoalchemy2==0.14.0"
           # 2021
           - python: 3.9
             os: ubuntu-20.04
-            pins: "sqlalchemy==1.4.30 alembic==1.8.* geoalchemy2==0.10.*"
+            pins: "sqlalchemy==1.4.44 alembic==1.8.* geoalchemy2==0.14.0"
           # 2022
           - python: "3.10"
             os: ubuntu-22.04
-            pins: "sqlalchemy==1.4.44 alembic==1.8.* geoalchemy2==0.12.*"
-          # current
+            pins: "sqlalchemy==1.4.44 alembic==1.8.* geoalchemy2==0.14.0"
+          # 2023
           - python: "3.11"
+            os: ubuntu-22.04
+            pins: "sqlalchemy==2.0.24 alembic==1.13.1 geoalchemy2==0.14.3"
+            pytestargs: "-Werror"
+          # 2024
+          - python: "3.12"
             os: ubuntu-latest
-            pins: ""
+            pins: "sqlalchemy==2.0.* alembic==1.13.* geoalchemy2==0.14.*"
             pytestargs: "-Werror"
 
     steps:
@@ -47,7 +52,14 @@ jobs:
         run: |
           sudo apt update
           sudo apt install --yes --no-install-recommends sqlite3 libsqlite3-mod-spatialite
-  
+          sqlite3 --version
+
+      - name: Install gdal (for ogr2ogr)
+        run: |
+          sudo apt update          
+          sudo apt install --yes --no-install-recommends gdal-bin
+          ogr2ogr --version
+
       - name: Install python dependencies
         shell: bash
         run: |

CHANGES.rst

@@ -1,10 +1,10 @@
 Changelog of threedi-schema
 ===================================================
 
-0.219.2 (unreleased)
+0.220 (unreleased)
 --------------------
-
-- Nothing changed yet.
+- Add support for geopackage
+- Remove `the_geom_linestring` from `v2_connection_nodes` because geopackage does not support multiple geometry objects in one table
 
 
 0.219.1 (2024-01-30)

threedi_schema/__init__.py

@@ -2,5 +2,5 @@
 from .domain import constants, custom_types, models  # NOQA
 
 # fmt: off
-__version__ = '0.219.2.dev0'
+__version__ = '0.219.2.dev1'
 # fmt: on

threedi_schema/application/schema.py

@@ -1,11 +1,16 @@
+import re
+import subprocess
 import warnings
+from pathlib import Path
 
+# This import is needed for alembic to recognize the geopackage dialect
+import geoalchemy2.alembic_helpers  # noqa: F401
 from alembic import command as alembic_command
 from alembic.config import Config
 from alembic.environment import EnvironmentContext
 from alembic.migration import MigrationContext
 from alembic.script import ScriptDirectory
-from sqlalchemy import Column, Integer, MetaData, Table
+from sqlalchemy import Column, Integer, MetaData, Table, text
 from sqlalchemy.exc import IntegrityError
 
 from ..domain import constants, models, views
@@ -73,7 +78,6 @@ def get_version(self):
                 connection, opts={"version_table": constants.VERSION_TABLE_NAME}
             )
             version = context.get_current_revision()
-
         if version is not None:
             return int(version)
         else:
@@ -85,6 +89,7 @@ def upgrade(
         backup=True,
         set_views=True,
         upgrade_spatialite_version=False,
+        convert_to_geopackage=False,
     ):
         """Upgrade the database to the latest version.
 
@@ -104,8 +109,11 @@ def upgrade(
 
         Specify 'upgrade_spatialite_version=True' to also upgrade the
         spatialite file version after the upgrade.
+
+        Specify 'convert_to_geopackage=True' to also convert from spatialite
+        to geopackage file version after the upgrade.
         """
-        if upgrade_spatialite_version and not set_views:
+        if (upgrade_spatialite_version or convert_to_geopackage) and not set_views:
             raise ValueError(
                 "Cannot upgrade the spatialite version without setting the views."
             )
@@ -123,9 +131,10 @@ def upgrade(
                 _upgrade_database(work_db, revision=revision, unsafe=True)
         else:
             _upgrade_database(self.db, revision=revision, unsafe=False)
-
         if upgrade_spatialite_version:
             self.upgrade_spatialite_version()
+        elif convert_to_geopackage:
+            self.convert_to_geopackage()
         elif set_views:
             self.set_views()
 
@@ -207,3 +216,103 @@ def upgrade_spatialite_version(self):
                     copy_models(self.db, work_db, self.declared_models)
                 except IntegrityError as e:
                     raise UpgradeFailedError(e.orig.args[0])
+
+    def convert_to_geopackage(self):
+        """
+        Convert spatialite to geopackage using gdal's ogr2ogr.
+
+        Does nothing if the current database is already a geopackage.
+
+        Raises UpgradeFailedError if the conversion of spatialite to geopackage with ogr2ogr fails.
+        """
+        if self.db.get_engine().dialect.name == "geopackage":
+            return
+        # Check if ogr2ogr
+        result = subprocess.run(
+            "ogr2ogr --version",
+            shell=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            text=True,
+        )
+        ## ogr2ogr is installed; make sure the version is high enough and return if not
+        if result.returncode == 0:
+            # get version
+            version = re.findall(r"\b(\d+\.\d+\.\d+)\b", result.stdout)[0]
+            # trim patch version and convert to float
+            float_version = float(version[0 : version.rfind(".")])
+            if float_version < 3.4:
+                warnings.warn(
+                    f"ogr2ogr 3.4 (part of GDAL) or newer is needed to convert spatialite to geopackage "
+                    f"but ogr2ogr {version} was found. {self.db.path} will not be converted"
+                    f"to geopackage."
+                )
+                return
+        # ogr2ogr is not (properly) installed; return
+        elif result.returncode != 0:
+            warnings.warn(
+                f"ogr2ogr (part of GDAL) is needed to convert spatialite to geopackage but no working"
+                f"working installation was found:\n{result.stderr}"
+            )
+            return
+        # Ensure database is upgraded and views are recreated
+        self.upgrade()
+        self.validate_schema()
+        # Make necessary modifications for conversion on temporary database
+        with self.db.file_transaction(start_empty=False, copy_results=False) as work_db:
+            # remove spatialite specific tables that break conversion
+            with work_db.get_session() as session:
+                session.execute(text("DROP TABLE IF EXISTS spatialite_history;"))
+                session.execute(text("DROP TABLE IF EXISTS views_geometry_columns;"))
+            cmd = [
+                "ogr2ogr",
+                "-skipfailures",
+                "-f",
+                "gpkg",
+                str(Path(self.db.path).with_suffix(".gpkg")),
+                str(work_db.path),
+                "-oo",
+                "LIST_ALL_TABLES=YES",
+            ]
+            try:
+                p = subprocess.Popen(
+                    cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=-1
+                )
+            except Exception as e:
+                raise UpgradeFailedError(f"ogr2ogr failed conversion:\n{e}")
+            _, out = p.communicate()
+        # Error handling
+        # convert bytes to utf and split lines
+        out_list = out.decode("utf-8").split("\n")
+        # collect only errors and remove 'ERROR #:'
+        errors = [
+            [idx, ": ".join(item.split(": ")[1:])]
+            for idx, item in enumerate(out_list)
+            if item.lower().startswith("error")
+        ]
+        # While creating the geopackage with ogr2ogr an error occurs
+        # because ogr2ogr tries to create a table `sqlite_sequence`, which
+        # is reserved for internal use. The resulting database seems fine,
+        # so this specific error is ignored
+        # convert error output to list
+        expected_error = 'sqlite3_exec(CREATE TABLE "sqlite_sequence" ( "rowid" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, "name" TEXT, "seq" TEXT)) failed: object name reserved for internal use: sqlite_sequence'
+        unexpected_error_indices = [
+            idx for idx, error in errors if error.lower() != expected_error.lower()
+        ]
+        if len(unexpected_error_indices) > 0:
+            error_str = "\n".join(
+                [out_list[idx].decode("utf-8") for idx in unexpected_error_indices]
+            )
+            raise UpgradeFailedError(f"ogr2ogr didn't finish as expected:\n{error_str}")
+        # Correct path of current database
+        self.db.path = Path(self.db.path).with_suffix(".gpkg")
+        # Reset engine so new path is used on the next call of get_engine()
+        self.db._engine = None
+        # Recreate views_geometry_columns so set_views works as expected
+        with self.db.get_session() as session:
+            session.execute(
+                text(
+                    "CREATE TABLE views_geometry_columns(view_name TEXT, view_geometry TEXT, view_rowid TEXT, f_table_name VARCHAR(256), f_geometry_column VARCHAR(256))"
+                )
+            )
+        self.set_views()

threedi_schema/application/threedi_database.py

@@ -1,9 +1,11 @@
+import os
 import shutil
 import tempfile
 import uuid
 from contextlib import contextmanager
 from pathlib import Path
 
+from geoalchemy2 import load_spatialite, load_spatialite_gpkg
 from sqlalchemy import create_engine, event, inspect, text
 from sqlalchemy.engine import Engine
 from sqlalchemy.event import listen
@@ -12,6 +14,8 @@
 
 from .schema import ModelSchema
 
+os.environ["SPATIALITE_LIBRARY_PATH"] = "mod_spatialite.so"
+
 __all__ = ["ThreediDatabase"]
 
 
@@ -35,45 +39,10 @@ def set_sqlite_pragma(dbapi_connection, connection_record):
     cursor.close()
 
 
-def load_spatialite(con, connection_record):
-    """Load spatialite extension as described in
-    https://geoalchemy-2.readthedocs.io/en/latest/spatialite_tutorial.html"""
-    import sqlite3
-
-    con.enable_load_extension(True)
-    cur = con.cursor()
-    libs = [
-        # SpatiaLite >= 4.2 and Sqlite >= 3.7.17, should work on all platforms
-        ("mod_spatialite", "sqlite3_modspatialite_init"),
-        # SpatiaLite >= 4.2 and Sqlite < 3.7.17 (Travis)
-        ("mod_spatialite.so", "sqlite3_modspatialite_init"),
-        # SpatiaLite < 4.2 (linux)
-        ("libspatialite.so", "sqlite3_extension_init"),
-    ]
-    found = False
-    for lib, entry_point in libs:
-        try:
-            cur.execute("select load_extension('{}', '{}')".format(lib, entry_point))
-        except sqlite3.OperationalError:
-            continue
-        else:
-            found = True
-            break
-    try:
-        cur.execute("select EnableGpkgAmphibiousMode()")
-    except sqlite3.OperationalError:
-        pass
-    if not found:
-        raise RuntimeError("Cannot find any suitable spatialite module")
-    cur.close()
-    con.enable_load_extension(False)
-
-
 class ThreediDatabase:
     def __init__(self, path, echo=False):
         self.path = path
         self.echo = echo
-
         self._engine = None
         self._base_metadata = None
 
@@ -90,22 +59,27 @@ def base_path(self):
         return Path(self.path).absolute().parent
 
     def get_engine(self, get_seperate_engine=False):
+        # Ensure that path is a Path so checks below don't break
+        path = Path(self.path)
         if self._engine is None or get_seperate_engine:
-            if self.path == "":
+            if path == Path(""):
                 # Special case in-memory SQLite:
                 # https://docs.sqlalchemy.org/en/20/dialects/sqlite.html#threading-pooling-behavior
                 poolclass = None
             else:
                 poolclass = NullPool
-            engine = create_engine(
-                "sqlite:///{0}".format(self.path), echo=self.echo, poolclass=poolclass
-            )
-            listen(engine, "connect", load_spatialite)
+            if path.suffix.lower() == ".gpkg":
+                engine_path = f"gpkg:///{path}"
+                engine_fn = load_spatialite_gpkg
+            else:
+                engine_path = "sqlite:///" if path == Path("") else f"sqlite:///{path}"
+                engine_fn = load_spatialite
+            engine = create_engine(engine_path, echo=self.echo, poolclass=poolclass)
+            listen(engine, "connect", engine_fn)
             if get_seperate_engine:
                 return engine
             else:
                 self._engine = engine
-
         return self._engine
 
     def get_session(self, **kwargs):
@@ -133,7 +107,7 @@ def session_scope(self, **kwargs):
             session.close()
 
     @contextmanager
-    def file_transaction(self, start_empty=False):
+    def file_transaction(self, start_empty=False, copy_results=True):
         """Copy the complete database into a tmpdir and work on that one.
 
         On contextmanager exit, the database is copied back and the real
@@ -150,7 +124,8 @@ def file_transaction(self, start_empty=False):
             except Exception as e:
                 raise e
             else:
-                shutil.copy(str(work_file), self.path)
+                if copy_results:
+                    shutil.copy(str(work_file), self.path)
 
     def check_connection(self):
         """Check if there a connection can be started with the database

threedi_schema/domain/models.py

@@ -280,7 +280,6 @@ class ConnectionNode(Base):
     storage_area = Column(Float)
     initial_waterlevel = Column(Float)
     the_geom = Column(Geometry("POINT"), nullable=False)
-    the_geom_linestring = Column(Geometry("LINESTRING"))
     code = Column(String(100))
 
     manholes = relationship("Manhole", back_populates="connection_node")

threedi_schema/domain/views.py

@@ -36,7 +36,7 @@
         "f_geometry_column": "the_geom",
     },
     "v2_manhole_view": {
-        "definition": "SELECT manh.rowid AS rowid, manh.id AS manh_id, manh.display_name AS manh_display_name, manh.code AS manh_code, manh.connection_node_id AS manh_connection_node_id, manh.shape AS manh_shape, manh.width AS manh_width, manh.length AS manh_length, manh.manhole_indicator AS manh_manhole_indicator, manh.calculation_type AS manh_calculation_type, manh.bottom_level AS manh_bottom_level, manh.surface_level AS manh_surface_level, manh.drain_level AS manh_drain_level, manh.sediment_level AS manh_sediment_level, manh.zoom_category AS manh_zoom_category, manh.exchange_thickness AS manh_exchange_thickness, manh.hydraulic_conductivity_in AS manh_hydraulic_conductivity_in, manh.hydraulic_conductivity_out AS manh_hydraulic_conductivity_out, node.id AS node_id, node.storage_area AS node_storage_area, node.initial_waterlevel AS node_initial_waterlevel, node.code AS node_code, node.the_geom AS the_geom, node.the_geom_linestring AS node_the_geom_linestring FROM v2_manhole AS manh , v2_connection_nodes AS node WHERE manh.connection_node_id = node.id",
+        "definition": "SELECT manh.rowid AS rowid, manh.id AS manh_id, manh.display_name AS manh_display_name, manh.code AS manh_code, manh.connection_node_id AS manh_connection_node_id, manh.shape AS manh_shape, manh.width AS manh_width, manh.length AS manh_length, manh.manhole_indicator AS manh_manhole_indicator, manh.calculation_type AS manh_calculation_type, manh.bottom_level AS manh_bottom_level, manh.surface_level AS manh_surface_level, manh.drain_level AS manh_drain_level, manh.sediment_level AS manh_sediment_level, manh.zoom_category AS manh_zoom_category, manh.exchange_thickness AS manh_exchange_thickness, manh.hydraulic_conductivity_in AS manh_hydraulic_conductivity_in, manh.hydraulic_conductivity_out AS manh_hydraulic_conductivity_out, node.id AS node_id, node.storage_area AS node_storage_area, node.initial_waterlevel AS node_initial_waterlevel, node.code AS node_code, node.the_geom AS the_geom FROM v2_manhole AS manh , v2_connection_nodes AS node WHERE manh.connection_node_id = node.id",
         "view_geometry": "the_geom",
         "view_rowid": "rowid",
         "f_table_name": "v2_connection_nodes",

threedi_schema/infrastructure/views.py

@@ -18,7 +18,6 @@ def drop_view(connection, name):
 def recreate_views(db, file_version, all_views, views_to_delete):
     """Recreate predefined views in a ThreediDatabase instance"""
     engine = db.engine
-
     with engine.connect() as connection:
         with connection.begin():
             for name, view in all_views.items():