Merge pull request #1956 from strictdoc-project/stanislaw/functions

Feature: Parsing source code functions into requirements graph

.coveragerc

@@ -5,7 +5,7 @@ omit =
     */.venv/*
 
 [report]
-fail_under = 65.0
+fail_under = 60.0
 precision = 2
 skip_covered = true
 show_missing = true

.python-version

@@ -1,2 +1,2 @@
-3.8
+3.9
 

pyproject.toml

@@ -56,6 +56,11 @@ dependencies = [
     # Parsing and rendering RST.
     "docutils >= 0.16, == 0.*",
 
+    # Tree Sitter is used for language/AST-aware parsing of Python, C and other files.
+    "tree-sitter",
+    "tree-sitter-c",
+    "tree-sitter-python",
+
     # Requirements-to-source traceability. Colored syntax for source files.
     "pygments >= 2.10.0, == 2.*",
 

strictdoc/backend/sdoc/error_handling.py

@@ -1,4 +1,6 @@
 # mypy: disable-error-code="attr-defined,no-untyped-call,no-untyped-def,union-attr"
+from typing import Optional
+
 from textx import TextXSyntaxError
 
 from strictdoc.backend.sdoc.models.document import SDocDocument
@@ -25,7 +27,13 @@ def get_textx_syntax_error_message(exception: TextXSyntaxError):
 
 class StrictDocSemanticError(Exception):
     def __init__(
-        self, title, hint, example, line=None, col=None, filename=None
+        self,
+        title: str,
+        hint: Optional[str],
+        example: Optional[str],
+        line: Optional[int] = None,
+        col: Optional[int] = None,
+        filename: Optional[str] = None,
     ):
         super().__init__(title, hint, line, col, filename)
         self.title = title

strictdoc/backend/sdoc/grammar/type_system.py

@@ -47,6 +47,7 @@
   ('  FORMAT: ' g_file_format = FileEntryFormat '\n')?
    '  VALUE: ' g_file_path = /.*$/ '\n'
   ('  LINE_RANGE: ' g_line_range = /.*$/ '\n')?
+  ('  FUNCTION: ' function = /.*$/ '\n')?
 ;
 
 FileEntryFormat[noskipws]:

strictdoc/backend/sdoc/models/type_system.py

@@ -56,6 +56,7 @@ def __init__(
         g_file_format: Optional[str],
         g_file_path: str,
         g_line_range: Optional[str],
+        function: Optional[str],
     ):
         self.parent = parent
 
@@ -85,6 +86,11 @@ def __init__(
                 int(range_components_str[1]),
             )
 
+        # textX parses an optional element as an empty string. We make it to None ourselves.
+        self.function: Optional[str] = (
+            function if function is not None and len(function) > 0 else None
+        )
+
 
 class FileEntryFormat:
     SOURCECODE = "Sourcecode"

strictdoc/backend/sdoc_source_code/marker_parser.py

@@ -0,0 +1,104 @@
+import re
+from typing import List, Union
+
+from strictdoc.backend.sdoc_source_code.models.function_range_marker import (
+    FunctionRangeMarker,
+)
+from strictdoc.backend.sdoc_source_code.models.range_marker import (
+    LineMarker,
+    RangeMarker,
+)
+from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
+
+REGEX_REQ = r"[A-Za-z][A-Za-z0-9\\-]+"
+# @relation(REQ-1, scope=function)
+REGEX_FUNCTION = (
+    rf"@relation\((/?)({REGEX_REQ}(?:, {REGEX_REQ})*)\, scope=function\)"
+)
+REGEX_RANGE = rf"@sdoc\[(/?)({REGEX_REQ}(?:, {REGEX_REQ})*)\]"
+REGEX_LINE = rf"@sdoc\((/?)({REGEX_REQ}(?:, {REGEX_REQ})*)\)"
+
+
+class MarkerParser:
+    @staticmethod
+    def parse(
+        input_string: str,
+        line_start: int,
+        line_end: int,
+        comment_line_start: int,
+        comment_column_start: int,
+    ) -> List[Union[FunctionRangeMarker, RangeMarker, LineMarker]]:
+        markers: List[Union[FunctionRangeMarker, RangeMarker, LineMarker]] = []
+        for input_line_idx_, input_line_ in enumerate(
+            input_string.splitlines()
+        ):
+            match_function = None
+            match_line = None
+            match_range = None
+
+            match_function = re.search(REGEX_FUNCTION, input_line_)
+            if match_function is None:
+                match_range = re.search(REGEX_RANGE, input_line_)
+                if match_range is None:
+                    match_line = re.search(REGEX_LINE, input_line_)
+
+            match = (
+                match_function
+                if match_function is not None
+                else match_range
+                if match_range is not None
+                else match_line
+            )
+            if match is None:
+                continue
+
+            start_or_end = match.group(1) != "/"
+            req_list = match.group(2)
+
+            first_requirement_index = match.start(2)
+
+            current_line = comment_line_start + input_line_idx_
+            first_requirement_column = first_requirement_index + 1
+            if input_line_idx_ == 0:
+                first_requirement_column += comment_column_start - 1
+            requirements = []
+            for req_match in re.finditer(REGEX_REQ, req_list):
+                req_item = req_match.group(0)  # Matched REQ-XXX item
+                # Calculate actual position relative to the original string
+                start_index = (
+                    req_match.start()
+                )  # Offset by where group 1 starts
+                requirement = Req(None, req_item)
+                requirement.ng_source_line = current_line
+                requirement.ng_source_column = (
+                    first_requirement_column + start_index
+                )
+                requirements.append(requirement)
+
+            if match_function is not None:
+                function_marker = FunctionRangeMarker(None, requirements)
+                function_marker.ng_source_line_begin = line_start
+                function_marker.ng_range_line_begin = line_start
+                function_marker.ng_range_line_end = line_end
+                function_marker.ng_marker_line = current_line
+                function_marker.ng_marker_column = first_requirement_column
+                markers.append(function_marker)
+            elif match_range is not None:
+                range_marker = RangeMarker(
+                    None, "[" if start_or_end else "[/", requirements
+                )
+                range_marker.ng_source_line_begin = line_start
+                range_marker.ng_source_column_begin = first_requirement_column
+                range_marker.ng_range_line_begin = line_start
+                range_marker.ng_range_line_end = line_end
+                markers.append(range_marker)
+            elif match_line is not None:
+                line_marker = LineMarker(None, requirements)
+                line_marker.ng_source_line_begin = line_start
+                line_marker.ng_range_line_begin = line_start
+                line_marker.ng_range_line_end = line_end
+                markers.append(line_marker)
+            else:
+                continue
+
+        return markers

strictdoc/backend/sdoc_source_code/models/function.py

@@ -0,0 +1,20 @@
+from typing import Any, List
+
+from strictdoc.helpers.auto_described import auto_described
+
+
+@auto_described
+class Function:
+    def __init__(
+        self,
+        parent: Any,
+        name: str,
+        line_begin: int,
+        line_end: int,
+        parts: List[Any],
+    ):
+        self.parent = parent
+        self.name = name
+        self.parts: List[Any] = parts
+        self.line_begin = line_begin
+        self.line_end = line_end

strictdoc/backend/sdoc_source_code/models/function_range_marker.py

@@ -0,0 +1,52 @@
+# mypy: disable-error-code="no-untyped-def,type-arg"
+from typing import List, Optional
+
+from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
+from strictdoc.helpers.auto_described import auto_described
+
+
+@auto_described
+class FunctionRangeMarker:
+    def __init__(self, parent, reqs_objs: List[Req]):
+        assert isinstance(reqs_objs, list)
+        self.parent = parent
+        self.reqs_objs: List[Req] = reqs_objs
+        self.reqs: List[str] = list(map(lambda req: req.uid, reqs_objs))
+
+        # Line number of the marker in the source code.
+        self.ng_source_line_begin: Optional[int] = None
+        self.ng_source_column_begin: Optional[int] = None
+
+        # Line number of the marker range in the source code:
+        # TODO: Improve description.
+        # For Begin ranges:
+        #   ng_range_line_begin == ng_source_line_begin  # noqa: ERA001
+        #   ng_range_line_end == ng_source_line_begin of the End marker  # noqa: ERA001, E501
+        # For End ranges:
+        #   ng_range_line_begin == ng_range_line_begin of the Begin marker  # noqa: ERA001, E501
+        #   ng_range_line_end == ng_source_line_begin  # noqa: ERA001
+        self.ng_range_line_begin: Optional[int] = None
+        self.ng_range_line_end: Optional[int] = None
+
+        self.ng_marker_line: Optional[int] = None
+        self.ng_marker_column: Optional[int] = None
+
+        self.ng_is_nodoc = "nosdoc" in self.reqs
+
+    def is_range_marker(self) -> bool:
+        return True
+
+    def is_line_marker(self) -> bool:
+        return False
+
+    def is_begin(self) -> bool:
+        return True
+
+    def is_end(self) -> bool:
+        return False
+
+
+@auto_described
+class ForwardFunctionRangeMarker(FunctionRangeMarker):
+    def __init__(self, parent, reqs_objs: List[Req]):
+        super().__init__(parent, reqs_objs)

strictdoc/backend/sdoc_source_code/models/range_marker.py

@@ -1,21 +1,22 @@
 # mypy: disable-error-code="no-untyped-def,type-arg"
-from typing import List
+from typing import Any, List, Optional
 
 from strictdoc.backend.sdoc_source_code.models.requirement_marker import Req
 from strictdoc.helpers.auto_described import auto_described
 
 
 @auto_described
 class RangeMarker:
-    def __init__(self, parent, begin_or_end, reqs_objs: List[Req]):
+    def __init__(self, parent: Any, begin_or_end: str, reqs_objs: List[Req]):
         assert isinstance(reqs_objs, list)
-        self.parent = parent
-        self.begin_or_end = begin_or_end
+        self.parent: Any = parent
+        self.begin_or_end: str = begin_or_end
         self.reqs_objs: List[Req] = reqs_objs
         self.reqs: List[str] = list(map(lambda req: req.uid, reqs_objs))
 
         # Line number of the marker in the source code.
-        self.ng_source_line_begin = None
+        self.ng_source_line_begin: Optional[int] = None
+        self.ng_source_column_begin: Optional[int] = None
 
         # Line number of the marker range in the source code:
         # TODO: Improve description.
@@ -25,50 +26,51 @@ def __init__(self, parent, begin_or_end, reqs_objs: List[Req]):
         # For End ranges:
         #   ng_range_line_begin == ng_range_line_begin of the Begin marker  # noqa: ERA001, E501
         #   ng_range_line_end == ng_source_line_begin  # noqa: ERA001
-        self.ng_range_line_begin = None
-        self.ng_range_line_end = None
+        self.ng_range_line_begin: Optional[int] = None
+        self.ng_range_line_end: Optional[int] = None
 
         self.ng_is_nodoc = "nosdoc" in self.reqs
 
-    def is_begin(self):
+    def is_begin(self) -> bool:
         return self.begin_or_end == "["
 
-    def is_end(self):
+    def is_end(self) -> bool:
         return self.begin_or_end == "[/"
 
-    def is_range_marker(self):
+    def is_range_marker(self) -> bool:
         return True
 
-    def is_line_marker(self):
+    def is_line_marker(self) -> bool:
         return False
 
 
 @auto_described
 class LineMarker:
-    def __init__(self, parent, reqs_objs):
+    def __init__(self, parent: Any, reqs_objs: List[Req]) -> None:
         assert isinstance(reqs_objs, list)
         self.parent = parent
         self.reqs_objs = reqs_objs
         self.reqs = list(map(lambda req: req.uid, reqs_objs))
 
         # Line number of the marker in the source code.
-        self.ng_source_line_begin = None
+        self.ng_source_line_begin: Optional[int] = None
+        self.ng_source_column_begin: Optional[int] = None
 
-        self.ng_range_line_begin = None
-        self.ng_range_line_end = None
+        self.ng_range_line_begin: Optional[int] = None
+        self.ng_range_line_end: Optional[int] = None
 
         self.ng_is_nodoc = "nosdoc" in self.reqs
 
-    def is_begin(self):
+    def is_begin(self) -> bool:
         return True
 
-    def is_end(self):
+    def is_end(self) -> bool:
         return False
 
-    def is_range_marker(self):
+    def is_range_marker(self) -> bool:
         return False
 
-    def is_line_marker(self):
+    def is_line_marker(self) -> bool:
         return True
 
 
@@ -81,19 +83,19 @@ def __init__(self, start_or_end: bool, reqs_objs: List):
         self.reqs_objs = reqs_objs
 
         # Line number of the marker in the source code.
-        self.ng_source_line_begin = None
+        self.ng_source_line_begin: Optional[int] = None
 
-        self.ng_range_line_begin = None
-        self.ng_range_line_end = None
+        self.ng_range_line_begin: Optional[int] = None
+        self.ng_range_line_end: Optional[int] = None
 
-    def is_begin(self):
+    def is_begin(self) -> bool:
         return self.start_or_end
 
-    def is_end(self):
+    def is_end(self) -> bool:
         return not self.start_or_end
 
-    def is_range_marker(self):
+    def is_range_marker(self) -> bool:
         return True
 
-    def is_line_marker(self):
+    def is_line_marker(self) -> bool:
         return False

strictdoc/backend/sdoc_source_code/models/requirement_marker.py

@@ -1,15 +1,16 @@
-# mypy: disable-error-code="no-untyped-def"
+from typing import Any, Optional
+
 from strictdoc.helpers.auto_described import auto_described
 
 
 @auto_described
 class Req:
-    def __init__(self, parent, uid: str):
+    def __init__(self, parent: Any, uid: str):
         assert isinstance(uid, str)
         assert len(uid) > 0
 
         self.parent = parent
         self.uid: str = uid
 
-        self.ng_source_line = None
-        self.ng_source_column = None
+        self.ng_source_line: Optional[int] = None
+        self.ng_source_column: Optional[int] = None