Added the ability for PySHACL to use baked in graphs instead of fetch…

…ing them from a HTTP endpoint when a known graph is imported using owl:imports

- This allows for time savings on graph-load and saves a HTTP request
- Also allows us to embed fixed errata versions of files in place of release-time ones online

With new features, comes new bugs
With the ability to now load SPARQLFunctions, this removes the barrier for loading Schema.org SHACL in advanced mode
But when doing so revealed more issues. They are now fixed:
Fixed SPARQLConstraintComponent getting confused when `shacl.ttl` was loaded into your Shapes file using owl:imports
Fixed #61

Refactored `SPARQLConstraintComponent` code, to allow for other custom constraint components in the future
- This prevented SPARQLConstraintComponent getting confused when `shacl.ttl` was loaded into the Shapes file
  using owl:imports

CHANGELOG.md

@@ -4,6 +4,27 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Python PEP 440 Versioning](https://www.python.org/dev/peps/pep-0440/).
 
+## [0.13.2] - 2020-09-07
+
+## Added
+- Added the ability for PySHACL to use baked in graphs instead of fetching them from a HTTP endpoint when a known graph
+  is imported using owl:imports
+  - This allows for time savings on graph-load and saves a HTTP request
+  - Also allows us to embed fixed errata versions of files in place of release-time ones online
+
+## Fixed
+- With new features, comes new bugs
+- With the ability to now load SPARQLFunctions, this removes the barrier for loading Schema.org SHACL in advanced mode
+- But when doing so revealed more issues. They are now fixed:
+- Fixed SPARQLConstraintComponent getting confused when `shacl.ttl` was loaded into your Shapes file using owl:imports
+- Fixed https://github.com/RDFLib/pySHACL/issues/61
+
+## Changed
+- Refactored `SPARQLConstraintComponent` code, to allow for other custom constraint components in the future
+  - This prevented SPARQLConstraintComponent getting confused when `shacl.ttl` was loaded into the Shapes file
+  using owl:imports
+
+
 ## [0.13.1] - 2020-09-07
 
 ## Added
@@ -645,7 +666,8 @@ just leaves the files open. Now it is up to the command-line client to close the
 
 - Initial version, limited functionality
 
-[Unreleased]: https://github.com/RDFLib/pySHACL/compare/v0.13.1...HEAD
+[Unreleased]: https://github.com/RDFLib/pySHACL/compare/v0.13.2...HEAD
+[0.13.2]: https://github.com/RDFLib/pySHACL/compare/v0.13.1...v0.13.2
 [0.13.1]: https://github.com/RDFLib/pySHACL/compare/v0.13.0...v0.13.1
 [0.13.0]: https://github.com/RDFLib/pySHACL/compare/v0.12.2...v0.13.0
 [0.12.2]: https://github.com/RDFLib/pySHACL/compare/v0.12.1.post2...v0.12.2

MANIFEST.in

@@ -2,7 +2,7 @@ include *.txt *.md
 include pyproject.toml
 include poetry.lock
 include MANIFEST.in
-include pyshacl/*.pickle
+include pyshacl/assets/*.pickle
 include pyshacl/*.spec
 include test/*.py
 include test/issues/*.py

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.masonry.api"
 
 [tool.poetry]
 name = "pyshacl"
-version = "0.13.1"
+version = "0.13.2"
 # Don't forget to change the version number in __init__.py along with this one
 description = "Python SHACL Validator"
 license = "Apache-2.0"
@@ -37,7 +37,7 @@ include = [
 	"*.txt",
 	"pyproject.toml",
     "MANIFEST.in",
-    "pyshacl/*.pickle",
+    "pyshacl/assets/*.pickle",
     "pyshacl/*.spec"
 ]
 

pyshacl/__init__.py

@@ -6,7 +6,7 @@
 
 
 # version compliant with https://www.python.org/dev/peps/pep-0440/
-__version__ = '0.13.1'
+__version__ = '0.13.2'
 # Don't forget to change the version number in pyproject.toml along with this one
 
 __all__ = ['validate', 'Validator', '__version__', 'Shape', 'ShapesGraph']

pyshacl/constraints/sparql/sparql_based_constraint_components.py

@@ -2,6 +2,8 @@
 """
 https://www.w3.org/TR/shacl/#sparql-constraint-components
 """
+import typing
+
 from typing import Dict, List, Tuple, Union
 
 import rdflib
@@ -14,6 +16,11 @@
 from pyshacl.pytypes import GraphLike
 
 
+if typing.TYPE_CHECKING:
+    from pyshacl.shape import Shape
+    from pyshacl.shapes_graph import ShapesGraph
+
+
 SH_nodeValidator = SH.term('nodeValidator')
 SH_propertyValidator = SH.term('propertyValidator')
 SH_validator = SH.term('validator')
@@ -25,13 +32,13 @@
 
 
 class BoundShapeValidatorComponent(ConstraintComponent):
-    def __init__(self, constraint, shape, validator):
+    def __init__(self, constraint, shape: 'Shape', validator):
         """
         Create a new custom constraint, by applying a ConstraintComponent and a Validator to a Shape
         :param constraint: The source ConstraintComponent, this is needed to bind the parameters in the query_helper
         :type constraint: SPARQLConstraintComponent
         :param shape:
-        :type shape: pyshacl.shape.Shape
+        :type shape: Shape
         :param validator:
         :type validator: AskConstraintValidator | SelectConstraintValidator
         """
@@ -103,7 +110,7 @@ def evaluate(self, target_graph: GraphLike, focus_value_nodes: Dict, _evaluation
 class SPARQLConstraintComponentValidator(object):
     validator_cache: Dict[Tuple[int, str], Union['SelectConstraintValidator', 'AskConstraintValidator']] = {}
 
-    def __new__(cls, shacl_graph, node, *args, **kwargs):
+    def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs):
         cache_key = (id(shacl_graph.graph), str(node))
         found_in_cache = cls.validator_cache.get(cache_key, False)
         if found_in_cache:
@@ -163,7 +170,7 @@ def apply_to_shape_via_constraint(self, constraint, shape, **kwargs) -> BoundSha
 
         return BoundShapeValidatorComponent(constraint, shape, self)
 
-    def __init__(self, shacl_graph, node, **kwargs):
+    def __init__(self, shacl_graph: 'ShapesGraph', node, **kwargs):
         initialised = getattr(self, 'initialised', False)
         if initialised:
             return
@@ -183,10 +190,10 @@ def __init__(self, shacl_graph, node, **kwargs):
 
 
 class AskConstraintValidator(SPARQLConstraintComponentValidator):
-    def __new__(cls, shacl_graph, node, *args, **kwargs):
+    def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs):
         return object.__new__(cls)
 
-    def __init__(self, shacl_graph, node, *args, **kwargs):
+    def __init__(self, shacl_graph: 'ShapesGraph', node, *args, **kwargs):
         super(AskConstraintValidator, self).__init__(shacl_graph, node, **kwargs)
         g = shacl_graph.graph
         ask_vals = set(g.objects(node, SH_ask))
@@ -244,10 +251,10 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind
 
 
 class SelectConstraintValidator(SPARQLConstraintComponentValidator):
-    def __new__(cls, shacl_graph, node, *args, **kwargs):
+    def __new__(cls, shacl_graph: 'ShapesGraph', node, *args, **kwargs):
         return object.__new__(cls)
 
-    def __init__(self, shacl_graph, node, *args, **kwargs):
+    def __init__(self, shacl_graph: 'ShapesGraph', node, *args, **kwargs):
         super(SelectConstraintValidator, self).__init__(shacl_graph, node, **kwargs)
         g = shacl_graph.graph
         select_vals = set(g.objects(node, SH_select))
@@ -324,27 +331,24 @@ def validate(self, focus, value_nodes, target_graph, query_helper=None, new_bind
             return violations
 
 
-class SPARQLConstraintComponent(object):
-    """
-    SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused.
-    Link:
-    https://www.w3.org/TR/shacl/#sparql-constraint-components
-    """
+class CustomConstraintComponentFactory(object):
+    __slots__ = set()
 
-    def __init__(self, shacl_graph, node):
-        self.sg = shacl_graph
-        self.node = node
+    def __new__(cls, shacl_graph: 'ShapesGraph', node):
+        self = list()
+        self.append(shacl_graph)
+        self.append(node)
         optional_params = []
         mandatory_params = []
-        param_nodes = set(self.sg.objects(self.node, SH_parameter))
+        param_nodes = set(shacl_graph.objects(node, SH_parameter))
         if len(param_nodes) < 1:
             # TODO:coverage: we don't have any tests for invalid constraints
             raise ConstraintLoadError(
                 "A sh:ConstraintComponent must have at least one value for sh:parameter",
                 "https://www.w3.org/TR/shacl/#constraint-components-parameters",
             )
         for param_node in iter(param_nodes):
-            path_nodes = set(self.sg.objects(param_node, SH_path))
+            path_nodes = set(shacl_graph.objects(param_node, SH_path))
             if len(path_nodes) < 1:
                 # TODO:coverage: we don't have any tests for invalid constraints
                 raise ConstraintLoadError(
@@ -358,7 +362,7 @@ def __init__(self, shacl_graph, node):
                     "https://www.w3.org/TR/shacl/#constraint-components-parameters",
                 )
             path = next(iter(path_nodes))
-            parameter = SHACLParameter(self.sg, param_node, path=path, logger=None)  # pass in logger?
+            parameter = SHACLParameter(shacl_graph, param_node, path=path, logger=None)  # pass in logger?
             if parameter.optional:
                 optional_params.append(parameter)
             else:
@@ -369,43 +373,88 @@ def __init__(self, shacl_graph, node):
                 "A sh:ConstraintComponent must have at least one non-optional parameter.",
                 "https://www.w3.org/TR/shacl/#constraint-components-parameters",
             )
-        self.parameters = mandatory_params + optional_params
-        validator_node_list = set(self.sg.graph.objects(node, SH_validator))
-        node_val_node_list = set(self.sg.graph.objects(node, SH_nodeValidator))
-        prop_val_node_list = set(self.sg.graph.objects(node, SH_propertyValidator))
-        self.validator_nodes = validator_node_list
-        self.node_validator_nodes = node_val_node_list
-        self.prop_validator_nodes = prop_val_node_list
-        val_count = len(self.validator_nodes)
-        node_val_count = len(self.node_validator_nodes)
-        prop_val_count = len(self.prop_validator_nodes)
-        if (val_count + node_val_count + prop_val_count) < 1:
-            # TODO:coverage: No test for this case, do we need to test this?
-            raise ConstraintLoadError(
-                "ConstraintComponent must have at least one sh:validator, "
-                "sh:nodeValidator, or sh:propertyValidator predicates.",
-                "https://www.w3.org/TR/shacl/#ConstraintComponent",
-            )
+        self.append(mandatory_params + optional_params)
+
+        validator_node_set = set(shacl_graph.graph.objects(node, SH_validator))
+        node_val_node_set = set(shacl_graph.graph.objects(node, SH_nodeValidator))
+        prop_val_node_set = set(shacl_graph.graph.objects(node, SH_propertyValidator))
+        validator_node_set = validator_node_set.difference(node_val_node_set)
+        validator_node_set = validator_node_set.difference(prop_val_node_set)
+        self.append(validator_node_set)
+        self.append(node_val_node_set)
+        self.append(prop_val_node_set)
+        is_sparql_constraint_component = False
+        for s in (validator_node_set, node_val_node_set, prop_val_node_set):
+            for v in s:
+                v_types = set(shacl_graph.graph.objects(v, RDF_type))
+                if SH_SPARQLAskValidator in v_types or SH_SPARQLSelectValidator in v_types:
+                    is_sparql_constraint_component = True
+                    break
+                v_props = set(p[0] for p in shacl_graph.graph.predicate_objects(v))
+                if SH_ask in v_props or SH_select in v_props:
+                    is_sparql_constraint_component = True
+                    break
+                if is_sparql_constraint_component:
+                    raise ConstraintLoadError(
+                        "Found a mix of SPARQL-based validators and non-SPARQL validators on a SPARQLConstraintComponent.",
+                        'https://www.w3.org/TR/shacl/#constraint-components-validators',
+                    )
+        if is_sparql_constraint_component:
+            return SPARQLConstraintComponent(*self)
+        else:
+            return CustomConstraintComponent(*self)
+
 
-    def make_validator_for_shape(self, shape):
+class CustomConstraintComponent(object):
+    __slots__ = ('sg', 'node', 'parameters', 'validators', 'node_validators', 'property_validators')
+
+    def __new__(cls, shacl_graph: 'ShapesGraph', node, parameters, validators, node_validators, property_validators):
+        self = super(CustomConstraintComponent, cls).__new__(cls)
+        self.sg = shacl_graph
+        self.node = node
+        self.parameters = parameters
+        self.validators = validators
+        self.node_validators = node_validators
+        self.property_validators = property_validators
+        return self
+
+    def make_validator_for_shape(self, shape: 'Shape'):
+        raise NotImplementedError()
+
+
+class SPARQLConstraintComponent(CustomConstraintComponent):
+    """
+    SPARQL-based constraints provide a lot of flexibility but may be hard to understand for some people or lead to repetition. This section introduces SPARQL-based constraint components as a way to abstract the complexity of SPARQL and to declare high-level reusable components similar to the Core constraint components. Such constraint components can be declared using the SHACL RDF vocabulary and thus shared and reused.
+    Link:
+    https://www.w3.org/TR/shacl/#sparql-constraint-components
+    """
+
+    __slots__ = set()
+
+    def __new__(cls, shacl_graph, node, parameters, validators, node_validators, property_validators):
+        return super(SPARQLConstraintComponent, cls).__new__(
+            cls, shacl_graph, node, parameters, validators, node_validators, property_validators
+        )
+
+    def make_validator_for_shape(self, shape: 'Shape'):
         """
         :param shape:
-        :type shape: pyshacl.shape.Shape
+        :type shape: Shape
         :return:
         """
-        val_count = len(self.validator_nodes)
-        node_val_count = len(self.node_validator_nodes)
-        prop_val_count = len(self.prop_validator_nodes)
+        val_count = len(self.validators)
+        node_val_count = len(self.node_validators)
+        prop_val_count = len(self.property_validators)
         must_be_select_val = False
         must_be_ask_val = False
         if shape.is_property_shape and prop_val_count > 0:
-            validator_node = next(iter(self.prop_validator_nodes))
+            validator_node = next(iter(self.property_validators))
             must_be_select_val = True
         elif (not shape.is_property_shape) and node_val_count > 0:
-            validator_node = next(iter(self.node_validator_nodes))
+            validator_node = next(iter(self.node_validators))
             must_be_select_val = True
         elif val_count > 0:
-            validator_node = next(iter(self.validator_nodes))
+            validator_node = next(iter(self.validators))
             must_be_ask_val = True
         else:
             raise ConstraintLoadError(