Merge pull request #497 from lmichel/feature_mivot

Feature MIVOT (Model Instance in VOTable)

.gitignore

@@ -54,9 +54,11 @@ pip-wheel-metadata
 .project
 .pydevproject
 .settings
+coverage.xml
 
 # Mac OSX
 .DS_Store
 
 # ipython
 .ipynb_checkpoints
+/.pytest_cache/

CHANGES.rst

@@ -22,6 +22,10 @@ Enhancements and Fixes
 - registry.Ivoid now accepts multiple ivoids and will then match any of
   them. [#517]
 
+- Introducing the new MIVOT module, enabling processed VOTable data mapped to
+  any model serialized in VO-DML. This package dynamically generates python objects 
+  whose structure corresponds to the classes of the mapped models. [#497]
+
 Deprecations and Removals
 -------------------------
 

conftest.py

@@ -54,6 +54,7 @@ def pytest_configure(config):
         # packages for which version numbers are displayed when running the tests.
         PYTEST_HEADER_MODULES['Astropy'] = 'astropy'  # noqa
         PYTEST_HEADER_MODULES['requests'] = 'requests'  # noqa
+        PYTEST_HEADER_MODULES['defusedxml'] = 'defusedxml'
 
         PYTEST_HEADER_MODULES.pop('Pandas', None)
         PYTEST_HEADER_MODULES.pop('h5py', None)

docs/index.rst

@@ -134,5 +134,6 @@ Using ``pyvo``
    registry/index
    io/index
    auth/index
+   mivot/index
    utils/index
    utils/prototypes

docs/mivot/index.rst

@@ -0,0 +1,235 @@
+********************
+MIVOT (`pyvo.mivot`)
+********************
+
+This module contains the new feature of annotations in VOTable.
+Astropy version >= 6.0 is required.
+
+Introduction
+============
+.. pull-quote::
+
+    Model Instances in VOTables (MIVOT) defines a syntax to map VOTable
+    data to any model serialized in VO-DML. The annotation operates as a
+    bridge between the data and the model. It associates the column/param
+    metadata from the VOTable to the data model elements (class, attributes,
+    types, etc.) [...].
+    The data model elements are grouped in an independent annotation block
+    complying with the MIVOT XML syntax. This annotation block is added
+    as an extra resource element at the top of the VOTable result resource. The
+    MIVOT syntax allows to describe a data structure as a hierarchy of classes.
+    It is also able to represent relations and composition between them. It can
+    also build up data model objects by aggregating instances from different
+    tables of the VOTable.
+
+- Model Instances in VOTables is a VO `standard <https://ivoa.net/documents/MIVOT/20230620/REC-mivot-1.0.pdf>`_
+- Requires Astropy>=6.0
+- ``pyvo.mivot`` is a prototype feature which must be activated with ``activate_features("MIVOT")``
+
+
+Implementation Scope
+--------------------
+This implementation is totally model-agnostic.
+
+- It does not operate any validation against specific data models.
+- It just requires the annotation syntax being compliant with the standards.
+
+However, many data samples used for the test suite and provided as examples
+are based on the ``EpochPropagation`` class of the ``Mango`` data model
+that is still a draft.
+This class collects all the parameters we need to compute the epoch propagation of moving sky objects.
+Some of the examples have been provided by a special end-point of the Vizier cone-search service
+(https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch) that maps query results to this model.
+
+.. image:: _images/mangoEpochPosition.png
+  :width: 500
+  :alt: EpochPropagation class used to validate this api.
+
+It is to be noted that the Vizier service does not annotate errors at the time of writing (Q1 2024)
+
+The implementation uses the Astropy read/write annotation module (6.0+),
+which allows to get (and set) Mivot blocks from/into VOTables as an XML element serialized as a string.
+
+.. pull-quote::
+
+    Not all MIVOT features are supported by this implementation, which mainly focuses on the
+    epoch propagation use case:
+
+    - ``JOIN`` features are not supported.
+    - ``TEMPLATES`` with more than one ``INSANCE`` not supported.
+
+Integrated Readout
+------------------
+The ``ModelViewer`` module manages access to data mapped to a model through dynamically
+generated objects (``MivotInstance``class).
+The example below shows how a VOTable, resulting from a cone-search query which data are mapped
+to the ``EpochPosition`` class, can be consumed.
+
+.. doctest-remote-data::
+    >>> import astropy.units as u
+    >>> from astropy.coordinates import SkyCoord
+    >>> from pyvo.dal.scs import SCSService
+    >>> from pyvo.utils.prototype import activate_features
+    >>> from pyvo.mivot.version_checker import check_astropy_version
+    >>> from pyvo.mivot.viewer.mivot_viewer import MivotViewer
+    >>> activate_features("MIVOT")
+    >>> if check_astropy_version() is False:
+    ...    pytest.skip("MIVOT test skipped because of the astropy version.")
+    >>> scs_srv = SCSService("https://cdsarc.cds.unistra.fr/beta/viz-bin/mivotconesearch/I/239/hip_main")
+    >>> m_viewer = MivotViewer(
+    ...     scs_srv.search(
+    ...         pos=SkyCoord(ra=52.26708 * u.degree, dec=59.94027 * u.degree, frame='icrs'),
+    ...         radius=0.05
+    ...     )
+    ... )
+    >>> mivot_instance = m_viewer.dm_instance
+    >>> print(mivot_instance.dmtype)
+    EpochPosition
+    >>> print(mivot_instance.Coordinate_coordSys.spaceRefFrame.value)
+    ICRS
+    >>> while m_viewer.next():
+    ...     print(f"position: {mivot_instance.latitude.value} {mivot_instance.longitude.value}")
+    position: 59.94033461 52.26722684
+
+
+In this example, the data readout is totally managed by the ``MivotViewer`` instance.
+The ``astropy.io.votable`` API is encapsulated in this module.
+
+Model leaves (class attributes) are complex types that provide additional information:
+
+- ``value``: attribute value
+- ``dmtype``: attribute type such as defined in the Mivot annotations
+- ``unit``: attribute unit such as defined in the Mivot annotations
+- ``ref``: identifier of the table column mapped on the attribute
+
+The model view on a data row can also be passed as a Python dictionary
+using the ``dict`` property of ``MivotInstance``.
+
+.. code-block:: python
+    :caption: Working with a model view as a dictionary
+              (the JSON layout has been squashed for display purpose)
+
+    from pyvo.mivot.utils.dict_utils import DictUtils
+
+    mivot_instance = m_viewer.dm_instance
+    mivot_object_dict = mivot_object.dict
+
+    DictUtils.print_pretty_json(mivot_object_dict)
+	{
+        "dmtype": "EpochPosition",
+        "longitude": {"value": 359.94372764, "unit": "deg"},
+        "latitude": {"value": -0.28005255, "unit": "deg"},
+        "pmLongitude": {"value": -5.14, "unit": "mas/yr"},
+        "pmLatitude": {"value": -25.43, "unit": "mas/yr"},
+        "epoch": {"value": 1991.25, "unit": "year"},
+        "Coordinate_coordSys": {
+            "dmtype": "SpaceSys",
+            "dmid": "SpaceFrame_ICRS",
+            "dmrole": "coordSys",
+            "spaceRefFrame": {"value": "ICRS"},
+        },
+    }
+
+- It is recommended to use a copy of the
+  dictionary as it will be rebuilt each time the ``dict`` property is invoked.
+- The default representation of ``MivotInstance`` instances is made with a pretty
+  string serialization of this dictionary.
+
+Per-Row Readout
+---------------
+
+The annotation schema can also be applied to table rows read outside of the ``MivotViewer``
+with the `astropy.io.votable` API:
+
+.. code-block:: python
+    :caption: Accessing the model view of Astropy table rows
+
+    votable = parse(path_to_votable)
+    table = votable.resources[0].tables[0]
+    # init the viewer
+    mivot_viewer = MivotViewer(votable, resource_number=0)
+    mivot_object = mivot_viewer.dm_instance
+    # and feed it with the table row
+    read = []
+    for rec in table.array:
+        mivot_object.update(rec)
+        read.append(mivot_object.longitude.value)
+        # show that the model retrieve the correct data values
+        assert rec["RAICRS"] == mivot_object.longitude.value
+        assert rec["DEICRS"] == mivot_object.latitude.value
+
+In this case, it is up to the user to ensure that the read data rows are those mapped by the Mivot annotations.
+
+For XML Hackers
+---------------
+
+The model instances can also be serialized as XML elements that can be parsed with XPath queries.
+
+.. code-block:: python
+    :caption: Accessing the XML view of the mapped model instances
+
+    with MivotViewer(path_to_votable) as mivot_viewer:
+        while mivot_viewer.next():
+        	xml_view = mivot_viewer.xml_view
+        	# do whatever you want with this XML element
+
+It to be noted that ``mivot_viewer.xml_view`` is a shortcut
+for ``mivot_viewer.xml_view.view`` where ``mivot_viewer.xml_view``
+is is an instance of ``pyvo.mivot.viewer.XmlViewer``.
+This object provides many functions facilitating the XML parsing.
+
+Class Generation in a Nutshell
+------------------------------
+
+MIVOT reconstructs model structures with 3 elements:
+
+- ``INSTANCE`` for the objects
+- ``ATTRIBUTE`` for the attributes
+- ``COLLECTION`` for the elements with a cardinality greater than 1
+
+The role played by each of these elements in the model hierarchy is defined
+by its ``@dmrole`` XML attribute. Types of both ``INSTANCE`` and ``ATTRIBUTE`` are defined by
+their ``@dmtype`` XML attributes.
+
+``MivotInstance`` classes are built by following MIVOT annotation structure:
+
+- ``INSTANCE`` are represented by Python classes
+- ``ATTRIBUTE`` are represented by Python class fields
+- ``COLLECTION`` are represented by Python lists ([])
+
+``@dmrole`` and ``@dmtype`` cannot be used as Python keywords as such, because they are built from VO-DML
+identifiers, which have the following structure: ``model:a.b``.
+
+- Only the last part of the path is kept for attribute names.
+- For class names, forbidden characters (``:`` or ``.``) are replaced with ``_``.
+- Original ``@dmtype`` are kept as attributes of generated Python objects.
+- The structure of the ``MivotInstance`` objects can be inferred from the mapped model in 2 different ways:
+
+  - 1.  From the MIVOT instance property ``MivotInstance.dict`` a shown above.
+        This is a pure Python dictionary but its access can be slow because it is generated
+        on the fly each time the property is invoked.
+  - 2.  From the internal  class dictionary ``MivotInstance.__dict__``
+        (see the Python `data model <https://docs.python.org/3/reference/datamodel.html>`_).
+
+ .. code-block:: python
+    :caption: Exploring the MivotInstance structure with the internal dictionaries
+
+    mivot_instance = mivot_viewer.dm_instance
+
+    print(mivot_instance.__dict__.keys())
+    dict_keys(['dmtype', 'longitude', 'latitude', 'pmLongitude', 'pmLatitude', 'epoch', 'Coordinate_coordSys'])
+
+    print(mivot_instance.Coordinate_coordSys.__dict__.keys())
+    dict_keys(['dmtype', 'dmid', 'dmrole', 'spaceRefFrame'])
+
+    print(mivot_instance.Coordinate_coordSys.spaceRefFrame.__dict__.keys())
+    dict_keys(['dmtype', 'value', 'unit', 'ref'])
+
+Reference/API
+=============
+
+.. automodapi:: pyvo.mivot
+.. automodapi:: pyvo.mivot.viewer
+.. automodapi:: pyvo.mivot.seekers
+.. automodapi:: pyvo.mivot.features
+.. automodapi:: pyvo.mivot.utils

pyvo/conftest.py

@@ -42,6 +42,7 @@ def pytest_configure(config):
         PYTEST_HEADER_MODULES.pop('h5py', None)
         PYTEST_HEADER_MODULES.pop('Scipy', None)
         PYTEST_HEADER_MODULES.pop('Matplotlib', None)
+        PYTEST_HEADER_MODULES['defusedxml'] = 'defusedxml'
 
         from . import __version__
         TESTED_VERSIONS['pyvo'] = __version__

pyvo/mivot/__init__.py

@@ -0,0 +1,2 @@
+# package entry point
+from .viewer.mivot_viewer import MivotViewer

pyvo/mivot/features/static_reference_resolver.py

@@ -0,0 +1,71 @@
+"""
+Class used to resolve each static REFERENCE found in mivot_block.
+"""
+from copy import deepcopy
+from pyvo.mivot.utils.exceptions import MivotException
+from pyvo.mivot.utils.xpath_utils import XPath
+from pyvo.utils.prototype import prototype_feature
+
+
+@prototype_feature('MIVOT')
+class StaticReferenceResolver:
+    """
+    Namespace for the function processing the static REFERENCEs
+    """
+    @staticmethod
+    def resolve(annotation_seeker, templates_ref, mivot_block):
+        """
+        Resolve all static REFERENCEs found in the mivot_block.
+        The referenced objects are first searched in GLOBALS and then in the templates_ref table.
+        REFERENCE elements are replaced with the referenced objects set with the roles of the REFERENCEs.
+        Works even if REFERENCE tags are numbered by the former processing.
+        Parameters
+        ----------
+        annotation_seeker : AnnotationSeeker
+            Utility to extract desired elements from the mapping block.
+        templates_ref : str
+            Identifier of the table where the mivot_block comes from.
+        mivot_block : xml.etree.ElementTree
+            The XML element object.
+        Returns
+        -------
+        int
+            The number of references resolved.
+        Raises
+        ------
+        MappingException
+            If the reference cannot be resolved.
+        NotImplementedError
+            If the reference is dynamic.
+        """
+        resolved_refs = 0
+        for ele in XPath.x_path_startwith(mivot_block, './/REFERENCE_'):
+            dmref = ele.get("dmref")
+            # If we have no @dmref in REFERENCE, we consider this is a ref based on a keys
+            if dmref is None:
+                raise NotImplementedError("Dynamic reference not implemented")
+            target = annotation_seeker.get_globals_instance_by_dmid(dmref)
+            found_in_global = True
+            if target is None and templates_ref is not None:
+                target = annotation_seeker.get_templates_instance_by_dmid(templates_ref, dmref)
+                found_in_global = False
+            if target is None:
+                raise MivotException(f"Cannot resolve reference={dmref}")
+            # Resolve static references recursively
+            if not found_in_global:
+                StaticReferenceResolver.resolve(annotation_seeker, templates_ref, ele)
+            else:
+                StaticReferenceResolver.resolve(annotation_seeker, None, ele)
+            # Set the reference role to the copied instance
+            target_copy = deepcopy(target)
+            # If the reference is within a collection: no role
+            if ele.get('dmrole'):
+                target_copy.attrib["dmrole"] = ele.get('dmrole')
+            parent_map = {c: p for p in mivot_block.iter() for c in p}
+            parent = parent_map[ele]
+            # Insert the referenced object
+            parent.append(target_copy)
+            # Drop the reference
+            parent.remove(ele)
+            resolved_refs += 1
+        return resolved_refs

pyvo/mivot/seekers/__init__.py

@@ -0,0 +1,4 @@
+"""
+``seekers`` package contains utilities for retrieving 
+components of VOTales or of Mivot blocks
+"""