Add documentation for filters

Re spine-tools/Spine-Toolbox#1655

docs/source/conf.py

@@ -23,8 +23,8 @@
 # -- Project information -----------------------------------------------------
 
 project = "Spine Database API"
-author = 'Spine project consortium'
-copyright = '2017-2021 {}'.format(author)
+author = "Spine project consortium, Spine Database API contributors"
+copyright = "2017-2022 {}".format(author)
 
 # The short X.Y version
 from spinedb_api import __version__ as spinedb_api_version
@@ -44,15 +44,15 @@
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.todo',
-    'sphinx.ext.coverage',
-    'sphinx.ext.ifconfig',
-    'sphinx.ext.viewcode',
-    'sphinx.ext.githubpages',
-    'sphinx.ext.napoleon',
-    'sphinx.ext.intersphinx',
-    'recommonmark',
-    'autoapi.extension',
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.viewcode",
+    "sphinx.ext.githubpages",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.intersphinx",
+    "recommonmark",
+    "autoapi.extension",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
@@ -64,7 +64,7 @@
 # source_suffix = ['.rst', '.md']
 # source_suffix = ".rst"
 # (note: since Sphinx 1.8 this is a dict rather than a list)
-source_suffix = {'.rst': 'restructuredtext', '.md': 'markdown'}  # support provided via the 'recommonmark' extension
+source_suffix = {".rst": "restructuredtext", ".md": "markdown"}  # support provided via the 'recommonmark' extension
 
 
 # The master toctree document.
@@ -83,30 +83,29 @@
 exclude_patterns = []
 
 # The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
+pygments_style = "sphinx"
 
 # Settings for Sphinx AutoAPI
-autoapi_options = ['members', 'show-module-summary', 'show-inheritance']
+autoapi_options = ["members", "show-module-summary", "show-inheritance"]
 autoapi_python_class_content = "both"
 autoapi_add_toctree_entry = True
 autoapi_root = "autoapi"
-autoapi_dirs = ['../../spinedb_api']  # package to be documented
+autoapi_dirs = ["../../spinedb_api"]  # package to be documented
 autoapi_ignore = [
-    '*/spinedb_api/alembic/*',
-    '*/spinedb_api/export_mapping/*',
-    '*/spinedb_api/import_mapping/*',
-    '*/spinedb_api/spine_io/*',
-    '*/spinedb_api/compatibility*',
-    '*/spinedb_api/db_mapping_helpers*',
-    '*/spinedb_api/exception*',
-    '*/spinedb_api/export_functions*',
-    '*/spinedb_api/helpers*',
-    '*/spinedb_api/mapping*',
-    '*/spinedb_api/perfect_split*',
-    '*/spinedb_api/purge*',
-    '*/spinedb_api/query*',
-    '*/spinedb_api/spine_db_client*',
-    '*/spinedb_api/spine_db_server*',
+    "*/spinedb_api/alembic/*",
+    "*/spinedb_api/arrow_value*",
+    "*/spinedb_api/compatibility*",
+    "*/spinedb_api/db_mapping_helpers*",
+    "*/spinedb_api/exception*",
+    "*/spinedb_api/export_functions*",
+    "*/spinedb_api/graph_layout_generator*",
+    "*/spinedb_api/helpers*",
+    "*/spinedb_api/mapping*",
+    "*/spinedb_api/perfect_split*",
+    "*/spinedb_api/spine_db_client*",
+    "*/spinedb_api/spine_db_server*",
+    "*/spinedb_api/temp_id*",
+    "*/spinedb_api/version*",
 ]  # ignored modules
 
 
@@ -125,7 +124,7 @@ def _process_docstring(app, what, name, obj, options, lines):
 
 def _db_mapping_schema_lines():
     def type_(f_dict):
-        return f_dict['type'].__name__ + (', optional' if f_dict.get('optional', False) else '')
+        return f_dict["type"].__name__ + (", optional" if f_dict.get("optional", False) else "")
 
     lines = [
         ".. _db_mapping_schema:",
@@ -230,7 +229,13 @@ def setup(sphinx):
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, "SpineDatabaseAPI.tex", "Spine Database API Documentation", "Spine project consortium", "manual")
+    (
+        master_doc,
+        "SpineDatabaseAPI.tex",
+        "Spine Database API Documentation",
+        "Spine project consortium, Spine Database API contributors",
+        "manual",
+    )
 ]
 
 
@@ -253,8 +258,8 @@ def setup(sphinx):
         "Spine Database API Documentation",
         author,
         "SpineDatabaseAPI",
-        "One line description of project.",
-        "Miscellaneous",
+        "Database interface to Spine generic data model.",
+        "",
     )
 ]
 
@@ -283,8 +288,8 @@ def setup(sphinx):
 
 # Example configuration for intersphinx: refer to the Python standard library.
 intersphinx_mapping = {
-    "python": ("https://docs.python.org/", None),
-    "sqlalchemy": ("https://docs.sqlalchemy.org/en/13/", None)
+    "python": ("https://docs.python.org/3/", None),
+    "sqlalchemy": ("https://docs.sqlalchemy.org/en/14/", None),
 }
 
 # -- Options for todo extension ----------------------------------------------

docs/source/filters.rst

@@ -0,0 +1,68 @@
+*******
+Filters
+*******
+
+.. note::
+
+   There is no need to apply filters manually if you are executing scripts
+   as Spine Toolbox Tools.
+
+The :mod:`spinedb_api.filters` module provides filters that alter the queries of :class:`spinedb_api.DatabaseMapping` such
+that the queries return filtered or transformed data.
+The filters are transparent to use:
+once applied, the database mapping can be read exactly the same way as without filters.
+
+.. note::
+
+   A database mapping that has filters applied should be considered read-only
+   as any data modifying functionality is oblivious of filters.
+
+Filters are generated by the :literal:`*_config()` functions in the filter-specific submodules of :mod:`spinedb_api.filters`.
+The functions return *config dicts*, dictionaries that should be treated as opaque objects
+and not to be messed with directly.
+
+The usual way to apply a filter is to create a config dict, then add it to the database URL
+with :func:`spinedb_api.filters.tools.append_filter_config`
+before passing the URL to the :class:`spinedb_api.DatabaseMapping` constructor.
+The snipped below demonstrates the application of a scenario filter this way:
+
+.. code-block:: python
+
+   from spinedb_api import DatabaseMapping
+   from spinedb_api.filters.scenario_filter import scenario_filter_config
+   from spinedb_api.filters.tools import append_filter_config
+
+   url = ...
+   scenario_filter = scenario_filter_config("coal_chp")
+   url = append_filter_config(url, scenario_filter)
+   with DatabaseMapping(url) as db_map:
+      ...
+
+If you already have a :class:`spinedb_api.DatabaseMapping` instance,
+:func:`spinedb_api.filters.tools.apply_filter_stack` lets you apply all your filters to the
+database mapping.
+
+.. note::
+
+   Filters applied by :func:`spinedb_api.filters.tools.apply_filter_stack` cannot affect data
+   that has been fetched from the database before the application.
+
+:meth:`spinedb_api.DatabaseMapping.get_filter_configs` returns the current config dicts
+that have been applied to the database mapping.
+These can be inspected with the functions found in :mod:`spinedb_api.filters`.
+As an example, the code below checks if database mapping has a scenario filter:
+
+.. code-block:: python
+
+   import sys
+   from spinedb_api import DatabaseMapping
+   from spinedb_api.filters.scenario_filter import scenario_name_from_dict
+
+   url = sys.argv[1]
+   with DatabaseMapping(url) as db_map:
+       filters = db_map.get_filter_configs()
+       if filters:
+           scenario_names = (scenario_name_from_dict(config_dict) for config_dict in filters)
+           scenario_name = next(iter(name for name in scenario_names if name is not None), None)
+           if scenario_name:
+               print(f"Scenario filter detected: {scenario_name}")

docs/source/index.rst

@@ -16,9 +16,10 @@ Welcome to Spine Database API's documentation!
 
    front_matter
    tutorial
+   db_mapping_schema
+   filters
    parameter_value_format
    metadata
-   db_mapping_schema
    autoapi/index
 
 Indices and tables

docs/source/parameter_value_format.rst

@@ -9,7 +9,7 @@ Parameter value format
 
    Client code should almost never convert parameter values to JSON and back manually.
    For most cases, JSON should be considered an implementation detail.
-   Clients should rather use :func:`.to_database` and :func:`.from_database` which shield
+   Clients should rather use :func:`spinedb_api.parameter_value.to_database` and :func:`spinedb_api.parameter_value.from_database` which shield
    from abrupt changes in the database representation.
 
 Parameter values are specified using JSON in the ``value`` field of the ``parameter_value`` table.

spinedb_api/__init__.py

@@ -10,9 +10,7 @@
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
 
-"""
-A package to interact with Spine DBs.
-"""
+""" A package to interact with Spine DBs. """
 
 from .db_mapping import DatabaseMapping
 from .exception import (

spinedb_api/dataframes.py

@@ -80,7 +80,7 @@
 For example, using ``df`` from above::
 
     with DatabaseMapping(target_url) as db_map:
-        add_or_update_from(df, target_url)
+        add_or_update_from(df, db_map)
         db_map.commit_session("Added unit_capacity value.")
 
 .. warning::

spinedb_api/db_mapping.py

@@ -1011,12 +1011,8 @@ def remove_unused_metadata(self):
         unused_metadata_ids = {x["id"] for x in self.mapped_table("metadata").valid_values()} - used_metadata_ids
         self.remove_items("metadata", *unused_metadata_ids)
 
-    def get_filter_configs(self):
-        """Returns the filters from this mapping's URL.
-
-        Returns:
-            list(dict):
-        """
+    def get_filter_configs(self) -> list[dict]:
+        """Returns the config dicts of filters applied to this database mapping."""
         return self._filter_configs
 
 

spinedb_api/export_mapping/__init__.py

@@ -9,10 +9,6 @@
 # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
-"""
-This package contains facilities to map a Spine database into tables.
-
-"""
 
 from .generator import rows, titles
 from .settings import (

spinedb_api/filters/__init__.py

@@ -9,3 +9,5 @@
 # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
+
+"""A package to apply filters to database mappings."""

spinedb_api/filters/alternative_filter.py

@@ -9,12 +9,20 @@
 # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
-""" Provides functions to apply filtering based on alternatives to parameter value subqueries. """
+"""
+This module provides the alternative filter.
+
+Alternative filter is defined by selected alternatives.
+It lets everything depending on the selected alternatives through and filters out the rest.
+"""
+from collections.abc import Iterable
 from functools import partial
 from sqlalchemy import and_, or_
 from ..exception import SpineDBAPIError
 from .query_utils import filter_by_active_elements
 
+__all__ = ("alternative_filter_config",)
+
 ALTERNATIVE_FILTER_TYPE = "alternative_filter"
 ALTERNATIVE_FILTER_SHORTHAND_TAG = "alternatives"
 
@@ -46,16 +54,8 @@ def apply_alternative_filter_to_parameter_value_sq(db_map, alternatives):
     db_map.override_parameter_value_sq_maker(make_parameter_value_sq)
 
 
-def alternative_filter_config(alternatives):
-    """
-    Creates a config dict for alternative filter.
-
-    Args:
-        alternatives (Iterable of str): alternative names
-
-    Returns:
-        dict: filter configuration
-    """
+def alternative_filter_config(alternatives: Iterable[str]) -> dict:
+    """Creates a config dict for alternative filter from a list of selected alternative names."""
     return {"type": ALTERNATIVE_FILTER_TYPE, "alternatives": list(alternatives)}
 
 

spinedb_api/filters/execution_filter.py

@@ -9,8 +9,6 @@
 # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
-""" Provides functions to apply filtering based on scenarios to parameter value subqueries. """
-
 from functools import partial
 import json
 from sqlalchemy import func  # pylint: disable=unused-import

spinedb_api/filters/query_utils.py

@@ -9,7 +9,6 @@
 # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
-""" Provides utilities for database queries. """
 from sqlalchemy import and_, func, or_
 
 

spinedb_api/filters/renamer.py

@@ -9,10 +9,16 @@
 # Public License for more details. You should have received a copy of the GNU Lesser General Public License along with
 # this program. If not, see <http://www.gnu.org/licenses/>.
 ######################################################################################################################
-""" Provides a database query manipulator that renames database items. """
+"""
+This module provides a manipulator filter that renames database items.
+
+Currently, entity classes and parameter can be renamed.
+"""
 from functools import partial
 from sqlalchemy import case
 
+__all__ = ("entity_class_renamer_config", "parameter_renamer_config")
+
 ENTITY_CLASS_RENAMER_TYPE = "entity_class_renamer"
 ENTITY_CLASS_RENAMER_SHORTHAND_TAG = "entity_class_rename"
 PARAMETER_RENAMER_TYPE = "parameter_renamer"
@@ -32,15 +38,11 @@ def apply_renaming_to_entity_class_sq(db_map, name_map):
     db_map.override_entity_class_sq_maker(renaming)
 
 
-def entity_class_renamer_config(**renames):
+def entity_class_renamer_config(**renames) -> dict:
     """
-    Creates a config dict for renamer.
+    Creates a config dict for entity class renamer.
 
-    Args:
-        **renames: keyword is the old name, value is the new name
-
-    Returns:
-        dict: renamer configuration
+    ``renames`` is a mapping from old parameter name to the new one.
     """
     return {"type": ENTITY_CLASS_RENAMER_TYPE, "name_map": dict(renames)}
 
@@ -102,15 +104,11 @@ def apply_renaming_to_parameter_definition_sq(db_map, name_map):
     db_map.override_parameter_definition_sq_maker(renaming)
 
 
-def parameter_renamer_config(renames):
+def parameter_renamer_config(renames: dict[str, str]) -> dict:
     """
-    Creates a config dict for renamer.
-
-    Args:
-        renames (dict): mapping from entity class name to mapping from parameter name to new name
+    Creates a config dict for parameter definition renamer.
 
-    Returns:
-        dict: renamer configuration
+    ``renames`` is a mapping from old parameter name to the new one.
     """
     return {"type": PARAMETER_RENAMER_TYPE, "name_map": renames}