Merge pull request #8 from dnv-opensource/feature/assertions

Assertions

CHANGELOG.md

@@ -3,10 +3,13 @@
 All notable changes to the [sim-explorer] project will be documented in this file.<br>
 The changelog format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
-## [Unreleased]
-
 -/-
 
+## [0.2.0] - 2024-12-18
+New Assertions release:
+
+* Added support for assertions in each of the cases to have some kind of evaluation being run after every simulation.
+* Display features to show the results of the assertions in a developer friendly format.
 
 ## [0.1.0] - 2024-11-08
 

docs/source/conf.py

@@ -26,7 +26,7 @@
 author = "Siegfried Eisinger, DNV Simulation Technology Team, SEACo project team"
 
 # The full version, including alpha/beta/rc tags
-release = "0.1.0"
+release = "0.2.0"
 
 # -- General configuration ---------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

pyproject.toml

@@ -22,7 +22,7 @@ packages = [
 
 [project]
 name = "sim-explorer"
-version = "0.1.2"
+version = "0.2.0"
 description = "Experimentation tools on top of OSP simulation models."
 readme = "README.rst"
 requires-python = ">= 3.10"
@@ -61,6 +61,8 @@ dependencies = [
     "fmpy>=0.3.21",
     "component-model>=0.1.0",
     "plotly>=5.24.1",
+    "pydantic>=2.10.3",
+    "rich>=13.9.4",
 ]
 
 [project.optional-dependencies]

src/sim_explorer/cli/display_results.py

@@ -0,0 +1,85 @@
+from rich.console import Console
+from rich.panel import Panel
+
+from sim_explorer.models import AssertionResult
+
+console = Console()
+
+
+def reconstruct_assertion_name(result: AssertionResult) -> str:
+    """
+    Reconstruct the assertion name from the key and expression.
+
+    :param result: Assertion result.
+    :return: Reconstructed assertion name.
+    """
+    time = result.time if result.time is not None else ""
+    return f"{result.key}@{result.temporal.name}{time}({result.expression})"
+
+
+def log_assertion_results(results: dict[str, list[AssertionResult]]):
+    """
+    Log test scenarios and results in a visually appealing bullet-point list format.
+
+    :param scenarios: Dictionary where keys are scenario names and values are lists of test results.
+                      Each test result is a tuple (test_name, status, details).
+                      Status is True for pass, False for fail.
+    """
+    total_passed = 0
+    total_failed = 0
+
+    console.print()
+
+    # Print results for each assertion executed in each of the cases ran
+    for case_name, assertions in results.items():
+        # Show case name first
+        console.print(f"[bold magenta]• {case_name}[/bold magenta]")
+        for assertion in assertions:
+            if assertion.result:
+                total_passed += 1
+            else:
+                total_failed += 1
+
+            # Print assertion status, details and error message if failed
+            status_icon = "✅" if assertion.result else "❌"
+            status_color = "green" if assertion.result else "red"
+            assertion_name = reconstruct_assertion_name(assertion)
+
+            # Need to add some padding to show that the assertion belongs to a case
+            console.print(f"   [{status_color}]{status_icon}[/] [cyan]{assertion_name}[/cyan]: {assertion.description}")
+
+            if not assertion.result:
+                console.print("      [red]⚠️ Error:[/] [dim]Assertion has failed[/dim]")
+
+        console.print()  # Add spacing between scenarios
+
+    if total_failed == 0 and total_passed == 0:
+        return
+
+    # Summary at the end
+    passed_tests = f"[green]✅ {total_passed} tests passed[/green] 😎" if total_passed > 0 else ""
+    failed_tests = f"[red]❌ {total_failed} tests failed[/red] 😭" if total_failed > 0 else ""
+    padding = "   " if total_passed > 0 and total_failed > 0 else ""
+    console.print(
+        Panel.fit(
+            f"{passed_tests}{padding}{failed_tests}", title="[bold blue]Test Summary[/bold blue]", border_style="blue"
+        )
+    )
+
+
+def group_assertion_results(results: list[AssertionResult]) -> dict[str, list[AssertionResult]]:
+    """
+    Group test results by case name.
+
+    :param results: list of assertion results.
+    :return: Dictionary where keys are case names and values are lists of assertion results.
+    """
+    grouped_results: dict[str, list[AssertionResult]] = {}
+    for result in results:
+        case_name = result.case
+        if case_name and case_name not in grouped_results:
+            grouped_results[case_name] = []
+
+        if case_name:
+            grouped_results[case_name].append(result)
+    return grouped_results

src/sim_explorer/cli/sim_explorer.py

@@ -7,15 +7,17 @@
 import sys
 from pathlib import Path
 
+from sim_explorer.case import Case, Cases
+from sim_explorer.cli.display_results import group_assertion_results, log_assertion_results
+from sim_explorer.utils.logging import configure_logging
+
 # Remove current directory from Python search path.
 # Only through this trick it is possible that the current CLI file 'sim_explorer.py'
 # carries the same name as the package 'sim_explorer' we import from in the next lines.
 # If we did NOT remove the current directory from the Python search path,
 # Python would start searching for the imported names within the current file (sim_explorer.py)
 # instead of the package 'sim_explorer' (and the import statements fail).
 sys.path = [path for path in sys.path if Path(path) != Path(__file__).parent]
-from sim_explorer.case import Case, Cases
-from sim_explorer.utils.logging import configure_logging
 
 logger = logging.getLogger(__name__)
 
@@ -153,12 +155,19 @@ def main() -> None:
 
     elif args.run is not None:
         case = cases.case_by_name(args.run)
+
         if case is None:
             logger.error(f"Case {args.run} not found in {args.cases}")
             return
+
         logger.info(f"{log_msg_stub}\t option: run \t\t\t{args.run}\n")
         # Invoke API
-        case.run()
+        cases.run_case(case, run_subs=False, run_assertions=True)
+
+        # Display assertion results
+        assertion_results = [assertion for assertion in cases.assertion.report()]
+        grouped_results = group_assertion_results(assertion_results)
+        log_assertion_results(grouped_results)
 
     elif args.Run is not None:
         case = cases.case_by_name(args.Run)
@@ -167,7 +176,12 @@ def main() -> None:
             return
         logger.info(f"{log_msg_stub}\t --Run \t\t\t{args.Run}\n")
         # Invoke API
-        cases.run_case(case, run_subs=True)
+        cases.run_case(case, run_subs=True, run_assertions=True)
+
+        # Display assertion results
+        assertion_results = [assertion for assertion in cases.assertion.report()]
+        grouped_results = group_assertion_results(assertion_results)
+        log_assertion_results(grouped_results)
 
 
 if __name__ == "__main__":

src/sim_explorer/models.py

@@ -0,0 +1,24 @@
+from enum import IntEnum
+
+from pydantic import BaseModel
+
+
+class Temporal(IntEnum):
+    UNDEFINED = 0
+    A = 1
+    ALWAYS = 1
+    F = 2
+    FINALLY = 2
+    T = 3
+    TIME = 3
+
+
+class AssertionResult(BaseModel):
+    key: str
+    expression: str
+    result: bool
+    temporal: Temporal
+    time: float | int | None
+    description: str
+    case: str | None
+    details: str

src/sim_explorer/simulator_interface.py

@@ -6,7 +6,7 @@
 
 from libcosimpy.CosimEnums import CosimVariableCausality, CosimVariableType, CosimVariableVariability  # type: ignore
 from libcosimpy.CosimExecution import CosimExecution  # type: ignore
-from libcosimpy.CosimLogging import CosimLogLevel, log_output_level
+from libcosimpy.CosimLogging import CosimLogLevel, log_output_level  # type: ignore
 from libcosimpy.CosimManipulator import CosimManipulator  # type: ignore
 from libcosimpy.CosimObserver import CosimObserver  # type: ignore