From 45559eb73581aaa92a85ffb5b7b6b718472786ff Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Tue, 24 Dec 2024 09:21:01 +0100 Subject: [PATCH 01/12] Added code to add current git hash for files in lobster-online-report, System test added --- Makefile | 1 + lobster/location.py | 28 +++++----- .../tools/core/online_report/online_report.py | 16 ++++-- .../lobster-core/online_report/Makefile | 54 +++++++++++++++++++ .../lobster-core/online_report/lobster.config | 3 ++ .../update_online_json_with_hashes.py | 25 +++++++++ 6 files changed, 111 insertions(+), 16 deletions(-) create mode 100644 tests-system/lobster-core/online_report/Makefile create mode 100644 tests-system/lobster-core/online_report/lobster.config create mode 100644 tests-system/lobster-core/online_report/update_online_json_with_hashes.py diff --git a/Makefile b/Makefile index 1e3ab6a2..aed22170 100644 --- a/Makefile +++ b/Makefile @@ -70,6 +70,7 @@ system-tests: make -B -C tests-system TOOL=lobster-json make -B -C tests-system TOOL=lobster-trlc make -B -C tests-system TOOL=lobster-python + (cd tests-system/lobster-core/online_report; make) unit-tests: coverage run -p \ diff --git a/lobster/location.py b/lobster/location.py index 94b2a86a..d91ae662 100644 --- a/lobster/location.py +++ b/lobster/location.py @@ -149,19 +149,21 @@ def from_json(cls, json): class Github_Reference(Location): - def __init__(self, gh_root, commit, filename, line): + def __init__(self, gh_root, commit, filename, line, exec_commit_id): assert isinstance(gh_root, str) assert gh_root.startswith("http") assert isinstance(commit, str) assert isinstance(filename, str) assert line is None or (isinstance(line, int) and line >= 1) + assert isinstance(exec_commit_id, str) - self.gh_root = gh_root.rstrip("/") - self.gh_repo = self.gh_root.split("/")[-1] - self.commit = commit - self.filename = filename - self.line = line + self.gh_root = gh_root.rstrip("/") + self.gh_repo = self.gh_root.split("/")[-1] + self.commit = commit + self.filename = filename + self.line = line + self.exec_commit_id = exec_commit_id def sorting_key(self): if self.line is not None: @@ -188,11 +190,12 @@ def to_html(self): self.to_string()) def to_json(self): - return {"kind" : "github", - "gh_root" : self.gh_root, - "commit" : self.commit, - "file" : self.filename, - "line" : self.line} + return {"kind" : "github", + "gh_root" : self.gh_root, + "commit" : self.commit, + "file" : self.filename, + "line" : self.line, + "exec_commit_id" : self.exec_commit_id} @classmethod def from_json(cls, json): @@ -203,7 +206,8 @@ def from_json(cls, json): commit = json["commit"] filename = json["file"] line = json.get("line", None) - return Github_Reference(gh_root, commit, filename, line) + exec_commit_id = json.get("exec_commit_id") + return Github_Reference(gh_root, commit, filename, line, exec_commit_id) class Codebeamer_Reference(Location): diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index dcd278c6..8fd215c2 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -227,19 +227,27 @@ def main(): actual_repo = gh_root actual_sha = options.commit actual_path = rel_path_from_root + exec_commit_id = subprocess.check_output( + ["git", "log", "-n", "1", "--format=%H", "--", item.location.filename] + ).decode().strip() # pylint: disable=consider-using-dict-items for prefix in gh_submodule_roots: if path_starts_with_subpath(rel_path_from_root, prefix): actual_repo = gh_submodule_roots[prefix] actual_sha = gh_submodule_sha[prefix] actual_path = rel_path_from_root[len(prefix) + 1:] + exec_commit_id = subprocess.check_output( + ["git", "log", "-n", "1", "--format=%H", "--", + actual_path], universal_newlines=True, cwd=prefix) + exec_commit_id = exec_commit_id.strip() break loc = Github_Reference( - gh_root = actual_repo, - commit = actual_sha, - filename = actual_path, - line = item.location.line) + gh_root = actual_repo, + commit = actual_sha, + filename = actual_path, + line = item.location.line, + exec_commit_id = exec_commit_id) item.location = loc report.write_report(options.out if options.out else options.lobster_report) diff --git a/tests-system/lobster-core/online_report/Makefile b/tests-system/lobster-core/online_report/Makefile new file mode 100644 index 00000000..d3a99f24 --- /dev/null +++ b/tests-system/lobster-core/online_report/Makefile @@ -0,0 +1,54 @@ +# Makefile + +# Set the paths to the Python script and Lobster commands +PYTHON_SCRIPT = ../../../lobster/tools/python/python.py +LOBSTER_CONFIG = lobster.config + +# Set the output file names +PYTHON_OUTPUT = python.lobster +REPORT_OUTPUT = report.lobster +ONLINE_REPORT_OUTPUT = online-report.lobster +EXPECTED_OUTPUT = expected-output.lobster + +# Set the Python interpreter +PYTHON = python3 + +# Define the default target +run: python-output report-output online-report-output copy-output python-script compare clean + +# Execute the Python script +python-output: + @echo "Running lobster-python script on: $(PYTHON_SCRIPT)" + lobster-python $(PYTHON_SCRIPT) --out=$(PYTHON_OUTPUT) + +# Generate the LOBSTER report +report-output: + lobster-report --lobster-config=$(LOBSTER_CONFIG) --out=$(REPORT_OUTPUT) + +# Generate the online LOBSTER report +online-report-output: + lobster-online-report $(REPORT_OUTPUT) --out=$(ONLINE_REPORT_OUTPUT) + +# Copy the online report to the expected output file +copy-output: + cp $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT) + +# Python script to update the expected git hashes +PYTHON_FILE = update_online_json_with_hashes.py + +# 'python-script' target to execute the python script to update expected git hashes +python-script: + $(PYTHON) $(PYTHON_FILE) + +# Compare target to compare the expected output with the actual output +compare: + @if diff $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT); then \ + echo "Files are identical"; \ + else \ + echo "Files are different"; \ + exit 1; \ + fi + +# Clean up the generated files +clean: + rm -f $(PYTHON_OUTPUT) $(REPORT_OUTPUT) $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT) diff --git a/tests-system/lobster-core/online_report/lobster.config b/tests-system/lobster-core/online_report/lobster.config new file mode 100644 index 00000000..eb9ee52d --- /dev/null +++ b/tests-system/lobster-core/online_report/lobster.config @@ -0,0 +1,3 @@ +implementation "code" { + source: "python.lobster"; +} \ No newline at end of file diff --git a/tests-system/lobster-core/online_report/update_online_json_with_hashes.py b/tests-system/lobster-core/online_report/update_online_json_with_hashes.py new file mode 100644 index 00000000..445a22e4 --- /dev/null +++ b/tests-system/lobster-core/online_report/update_online_json_with_hashes.py @@ -0,0 +1,25 @@ +import json +import subprocess + +# File name +file_name = 'expected-output.lobster' + +# Load the JSON data from the file +with open(file_name, 'r') as file: + data = json.load(file) + +# Traverse the JSON structure and update the 'loc' values +for level in data['levels']: + for item in level['items']: + location = item['location'] + exec_commit_id = subprocess.check_output( + ["git", "log", "-n", "1", "--format=%H", "--", + "../../../"+location['file']]).decode().strip() + location['exec_commit_id'] = exec_commit_id + +# Save the updated JSON data back to the same file +with open(file_name, 'w') as fd: + json.dump(data, fd, indent=2) + fd.write("\n") + +print(f"JSON data updated and saved back to '{file_name}'.") \ No newline at end of file From da1e7a23f5927ab360a23825e0638beb74b35b52 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Tue, 24 Dec 2024 11:33:44 +0100 Subject: [PATCH 02/12] fixed failing tests --- .../projects/basic/report.reference_output | 63 ++++++++++++------- .../projects/filter/report.reference_output | 39 ++++++++---- tests-unit/test_items.py | 3 +- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/tests-integration/projects/basic/report.reference_output b/tests-integration/projects/basic/report.reference_output index e8121159..f657f8c2 100644 --- a/tests-integration/projects/basic/report.reference_output +++ b/tests-integration/projects/basic/report.reference_output @@ -50,7 +50,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 3 + "line": 3, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.req_implication", "messages": [], @@ -77,7 +78,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 9 + "line": 9, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.req_xor", "messages": [ @@ -107,7 +109,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 15 + "line": 15, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.req_nand", "messages": [], @@ -135,7 +138,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 22 + "line": 22, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.req_nor", "messages": [ @@ -164,7 +168,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 27 + "line": 27, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.req_implies", "messages": [], @@ -188,7 +193,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 39 + "line": 39, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.req_important", "messages": [ @@ -223,7 +229,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 3 + "line": 3, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "implication", "messages": [], @@ -245,7 +252,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 9 + "line": 9, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "exclusive_or", "messages": [ @@ -265,7 +273,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 14 + "line": 14, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "potato", "messages": [ @@ -286,7 +295,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nand.m", - "line": 1 + "line": 1, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "nand", "messages": [], @@ -308,7 +318,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": 1 + "line": 1, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "exclusive_or/MATLAB Function", "messages": [ @@ -328,7 +339,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "exclusive_or", "messages": [ @@ -348,7 +360,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "exclusive_or/My Exclusive Or", "messages": [], @@ -370,7 +383,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 5 + "line": 5, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "nor.trlc_reference", "messages": [], @@ -390,7 +404,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 13 + "line": 13, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "nor.Example.helper_function", "messages": [], @@ -412,7 +427,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 17 + "line": 17, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "nor.Example.nor", "messages": [], @@ -441,7 +457,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/test.cpp", - "line": 7 + "line": 7, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "ImplicationTest:BasicTest", "messages": [], @@ -464,7 +481,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nand_test.m", - "line": 3 + "line": 3, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "nand_test::test_1", "messages": [], @@ -487,7 +505,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.json:XOR Test 1", "messages": [], @@ -510,7 +529,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.json:XOR Test 2", "messages": [], @@ -533,7 +553,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "example.json:Potato Test 1", "messages": [], diff --git a/tests-integration/projects/filter/report.reference_output b/tests-integration/projects/filter/report.reference_output index 49f0b68b..ac263ae8 100644 --- a/tests-integration/projects/filter/report.reference_output +++ b/tests-integration/projects/filter/report.reference_output @@ -14,7 +14,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 4 + "line": 4, + "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" }, "name": "sysreq_example.lobster_example", "messages": [], @@ -39,7 +40,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 11 + "line": 11, + "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" }, "name": "sysreq_example.req_implication", "messages": [], @@ -63,7 +65,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 19 + "line": 19, + "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" }, "name": "sysreq_example.req_xor", "messages": [], @@ -94,7 +97,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/softreq_example.trlc", - "line": 5 + "line": 5, + "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" }, "name": "softreq_example.req_implication", "messages": [], @@ -123,7 +127,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/softreq_example.trlc", - "line": 17 + "line": 17, + "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" }, "name": "softreq_example.req_xor", "messages": [], @@ -161,7 +166,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/foo.cpp", - "line": 3 + "line": 3, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "implication", "messages": [], @@ -183,7 +189,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/foo.cpp", - "line": 9 + "line": 9, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "exclusive_or", "messages": [], @@ -212,7 +219,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "test_example.json:Implication Test 1", "messages": [], @@ -235,7 +243,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "test_example.json:Implication Test 2", "messages": [], @@ -258,7 +267,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "test_example.json:XOR Test 1", "messages": [], @@ -281,7 +291,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "test_example.json:XOR Test 2", "messages": [], @@ -304,7 +315,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "test_example.json:XOR Test 3", "messages": [], @@ -327,7 +339,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" }, "name": "test_example.json:XOR Test 4", "messages": [], diff --git a/tests-unit/test_items.py b/tests-unit/test_items.py index d1673963..255b14de 100644 --- a/tests-unit/test_items.py +++ b/tests-unit/test_items.py @@ -33,7 +33,8 @@ def set_location_data(self, location_type): "gh_root": "https://mysuperserver.com", "commit": "commit string", "file": "example.txt", - "line": 1 + "line": 1, + "exec_commit_id": "efdc34rfe2345554rfe" } elif location_type == "codebeamer": location_data = { From de8d87dcc87806df652dce3e5105430a0dec1f97 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Thu, 2 Jan 2025 09:49:10 +0100 Subject: [PATCH 03/12] added double quotes in makefile paths for windows --- .../lobster-core/online_report/Makefile | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tests-system/lobster-core/online_report/Makefile b/tests-system/lobster-core/online_report/Makefile index d3a99f24..6a95b37d 100644 --- a/tests-system/lobster-core/online_report/Makefile +++ b/tests-system/lobster-core/online_report/Makefile @@ -1,14 +1,18 @@ # Makefile +LOBSTER_PYTHON_TOOL = "../../../lobster-python" +LOBSTER_REPORT_TOOL = "../../../lobster-report" +LOBSTER_ONLINE_REPORT_TOOL = "../../../lobster-online-report" + # Set the paths to the Python script and Lobster commands -PYTHON_SCRIPT = ../../../lobster/tools/python/python.py -LOBSTER_CONFIG = lobster.config +PYTHON_SCRIPT = "../../../lobster/tools/python/python.py" +LOBSTER_CONFIG = "lobster.config" # Set the output file names -PYTHON_OUTPUT = python.lobster -REPORT_OUTPUT = report.lobster -ONLINE_REPORT_OUTPUT = online-report.lobster -EXPECTED_OUTPUT = expected-output.lobster +PYTHON_OUTPUT = "python.lobster" +REPORT_OUTPUT = "report.lobster" +ONLINE_REPORT_OUTPUT = "online-report.lobster" +EXPECTED_OUTPUT = "expected-output.lobster" # Set the Python interpreter PYTHON = python3 @@ -19,15 +23,15 @@ run: python-output report-output online-report-output copy-output python-script # Execute the Python script python-output: @echo "Running lobster-python script on: $(PYTHON_SCRIPT)" - lobster-python $(PYTHON_SCRIPT) --out=$(PYTHON_OUTPUT) + $(LOBSTER_PYTHON_TOOL) $(PYTHON_SCRIPT) --out=$(PYTHON_OUTPUT) # Generate the LOBSTER report report-output: - lobster-report --lobster-config=$(LOBSTER_CONFIG) --out=$(REPORT_OUTPUT) + @(LOBSTER_REPORT_TOOL) --lobster-config=$(LOBSTER_CONFIG) --out=$(REPORT_OUTPUT) # Generate the online LOBSTER report online-report-output: - lobster-online-report $(REPORT_OUTPUT) --out=$(ONLINE_REPORT_OUTPUT) + @(LOBSTER_ONLINE_REPORT_TOOL) $(REPORT_OUTPUT) --out=$(ONLINE_REPORT_OUTPUT) # Copy the online report to the expected output file copy-output: From 0c2069a54fcdddc7cfc5a4b978ae6dd9a864b6ac Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Fri, 3 Jan 2025 09:50:09 +0100 Subject: [PATCH 04/12] mend --- tests-system/lobster-core/online_report/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests-system/lobster-core/online_report/Makefile b/tests-system/lobster-core/online_report/Makefile index 6a95b37d..3a9552bc 100644 --- a/tests-system/lobster-core/online_report/Makefile +++ b/tests-system/lobster-core/online_report/Makefile @@ -27,11 +27,11 @@ python-output: # Generate the LOBSTER report report-output: - @(LOBSTER_REPORT_TOOL) --lobster-config=$(LOBSTER_CONFIG) --out=$(REPORT_OUTPUT) + $(LOBSTER_REPORT_TOOL) --lobster-config=$(LOBSTER_CONFIG) --out=$(REPORT_OUTPUT) # Generate the online LOBSTER report online-report-output: - @(LOBSTER_ONLINE_REPORT_TOOL) $(REPORT_OUTPUT) --out=$(ONLINE_REPORT_OUTPUT) + $(LOBSTER_ONLINE_REPORT_TOOL) $(REPORT_OUTPUT) --out=$(ONLINE_REPORT_OUTPUT) # Copy the online report to the expected output file copy-output: From b9dd33e3c86c766c2dae74c6b896b6907b1712e7 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Tue, 7 Jan 2025 07:07:02 +0100 Subject: [PATCH 05/12] Dynamically update the reference output file in test-integration basic and filter tests --- tests-integration/projects/basic/Makefile | 4 +++ tests-integration/projects/filter/Makefile | 5 +++ .../lobster-core/online_report/Makefile | 10 +++--- .../update_online_json_with_hashes.py | 25 -------------- tests-utils/update_online_json_with_hashes.py | 34 +++++++++++++++++++ 5 files changed, 48 insertions(+), 30 deletions(-) delete mode 100644 tests-system/lobster-core/online_report/update_online_json_with_hashes.py create mode 100644 tests-utils/update_online_json_with_hashes.py diff --git a/tests-integration/projects/basic/Makefile b/tests-integration/projects/basic/Makefile index a6f3f8ab..a901d9d4 100644 --- a/tests-integration/projects/basic/Makefile +++ b/tests-integration/projects/basic/Makefile @@ -8,11 +8,15 @@ CLANG_TIDY:=$(LOBSTER_ROOT)/../llvm-project/build/bin/clang-tidy THIS_TEST:=$(shell realpath --relative-to $(LOBSTER_ROOT) $(PWD)) THIS_TEST_ESCAPED:=$(subst /,\\/,$(THIS_TEST)) +REFERENCE_OUTPUT:='report.reference_output' +PYTHON = python3 +UPDATE_GIT_HASHES_SCRIPT:='../../../tests-utils/update_online_json_with_hashes.py' html_report.html: cppcode.lobster gtests.lobster mcode.lobster system-requirements.lobster software-requirements.lobster lobster.conf pycode.lobster json.lobster @lobster-report @lobster-online-report @cp report.lobster report.reference_output + $(PYTHON) $(UPDATE_GIT_HASHES_SCRIPT) $(REFERENCE_OUTPUT) @lobster-html-report @lobster-ci-report | tee ci_report.reference_output diff --git a/tests-integration/projects/filter/Makefile b/tests-integration/projects/filter/Makefile index b8d46f0a..fa8fea7b 100644 --- a/tests-integration/projects/filter/Makefile +++ b/tests-integration/projects/filter/Makefile @@ -9,10 +9,15 @@ CLANG_TIDY:=$(LOBSTER_ROOT)/../llvm-project/build/bin/clang-tidy THIS_TEST:=$(shell realpath --relative-to $(LOBSTER_ROOT) $(PWD)) THIS_TEST_ESCAPED:=$(subst /,\\/,$(THIS_TEST)) +REFERENCE_OUTPUT:='report.reference_output' +PYTHON = python3 +UPDATE_GIT_HASHES_SCRIPT:='../../../tests-utils/update_online_json_with_hashes.py' + html_report.html: cppcode.lobster requirements.lobster lobster.conf json.lobster @lobster-report @lobster-online-report @cp report.lobster report.reference_output + $(PYTHON) $(UPDATE_GIT_HASHES_SCRIPT) $(REFERENCE_OUTPUT) @lobster-html-report cppcode.lobster: foo.h foo.cpp diff --git a/tests-system/lobster-core/online_report/Makefile b/tests-system/lobster-core/online_report/Makefile index 3a9552bc..e8ed4aee 100644 --- a/tests-system/lobster-core/online_report/Makefile +++ b/tests-system/lobster-core/online_report/Makefile @@ -12,7 +12,10 @@ LOBSTER_CONFIG = "lobster.config" PYTHON_OUTPUT = "python.lobster" REPORT_OUTPUT = "report.lobster" ONLINE_REPORT_OUTPUT = "online-report.lobster" -EXPECTED_OUTPUT = "expected-output.lobster" +EXPECTED_OUTPUT = 'expected-output.lobster' + +# Python script to update the expected git hashes +PYTHON_FILE:='../../../tests-utils/update_online_json_with_hashes.py' # Set the Python interpreter PYTHON = python3 @@ -37,12 +40,9 @@ online-report-output: copy-output: cp $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT) -# Python script to update the expected git hashes -PYTHON_FILE = update_online_json_with_hashes.py - # 'python-script' target to execute the python script to update expected git hashes python-script: - $(PYTHON) $(PYTHON_FILE) + $(PYTHON) $(PYTHON_FILE) $(EXPECTED_OUTPUT) # Compare target to compare the expected output with the actual output compare: diff --git a/tests-system/lobster-core/online_report/update_online_json_with_hashes.py b/tests-system/lobster-core/online_report/update_online_json_with_hashes.py deleted file mode 100644 index 445a22e4..00000000 --- a/tests-system/lobster-core/online_report/update_online_json_with_hashes.py +++ /dev/null @@ -1,25 +0,0 @@ -import json -import subprocess - -# File name -file_name = 'expected-output.lobster' - -# Load the JSON data from the file -with open(file_name, 'r') as file: - data = json.load(file) - -# Traverse the JSON structure and update the 'loc' values -for level in data['levels']: - for item in level['items']: - location = item['location'] - exec_commit_id = subprocess.check_output( - ["git", "log", "-n", "1", "--format=%H", "--", - "../../../"+location['file']]).decode().strip() - location['exec_commit_id'] = exec_commit_id - -# Save the updated JSON data back to the same file -with open(file_name, 'w') as fd: - json.dump(data, fd, indent=2) - fd.write("\n") - -print(f"JSON data updated and saved back to '{file_name}'.") \ No newline at end of file diff --git a/tests-utils/update_online_json_with_hashes.py b/tests-utils/update_online_json_with_hashes.py new file mode 100644 index 00000000..e01969cc --- /dev/null +++ b/tests-utils/update_online_json_with_hashes.py @@ -0,0 +1,34 @@ +import json +import subprocess +import sys +from fileinput import filename + + +# File name +# file_name = 'expected-output.lobster' + +def update_json(filename): + # Load the JSON data from the file + with open(filename, 'r') as file: + data = json.load(file) + + # Traverse the JSON structure and update the 'loc' values + for level in data['levels']: + for item in level['items']: + location = item['location'] + if 'file' in location: + exec_commit_id = subprocess.check_output( + ["git", "log", "-n", "1", "--format=%H", "--", + "../../../"+location['file']]).decode().strip() + location['exec_commit_id'] = exec_commit_id + + # Save the updated JSON data back to the same file + with open(filename, 'w') as fd: + json.dump(data, fd, indent=2) + fd.write("\n") + + print(f"JSON data updated and saved back to '{filename}'.") + + +if __name__ == "__main__": + update_json(sys.argv[1]) From 6231d2c4ede39cf1b8db2e427a6addf1d129ad0f Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Tue, 7 Jan 2025 13:05:53 +0100 Subject: [PATCH 06/12] Updated code to get git hash for current commit instead of file commit --- lobster/tools/core/online_report/online_report.py | 5 ++--- tests-utils/update_online_json_with_hashes.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index 8fd215c2..630b27d8 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -228,7 +228,7 @@ def main(): actual_sha = options.commit actual_path = rel_path_from_root exec_commit_id = subprocess.check_output( - ["git", "log", "-n", "1", "--format=%H", "--", item.location.filename] + ["git", "rev-parse", "HEAD"] ).decode().strip() # pylint: disable=consider-using-dict-items for prefix in gh_submodule_roots: @@ -237,8 +237,7 @@ def main(): actual_sha = gh_submodule_sha[prefix] actual_path = rel_path_from_root[len(prefix) + 1:] exec_commit_id = subprocess.check_output( - ["git", "log", "-n", "1", "--format=%H", "--", - actual_path], universal_newlines=True, cwd=prefix) + ["git", "rev-parse", "HEAD"], universal_newlines=True, cwd=prefix) exec_commit_id = exec_commit_id.strip() break diff --git a/tests-utils/update_online_json_with_hashes.py b/tests-utils/update_online_json_with_hashes.py index e01969cc..89bc3968 100644 --- a/tests-utils/update_online_json_with_hashes.py +++ b/tests-utils/update_online_json_with_hashes.py @@ -18,8 +18,7 @@ def update_json(filename): location = item['location'] if 'file' in location: exec_commit_id = subprocess.check_output( - ["git", "log", "-n", "1", "--format=%H", "--", - "../../../"+location['file']]).decode().strip() + ["git", "rev-parse", "HEAD"]).decode().strip() location['exec_commit_id'] = exec_commit_id # Save the updated JSON data back to the same file From 6a2b08ca2f48a0dd1db5c029b98bcf8c6ff54495 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Fri, 10 Jan 2025 07:17:04 +0100 Subject: [PATCH 07/12] Moved system tests from Makefile to folder structure method --- Makefile | 2 +- .../tools/core/online_report/online_report.py | 4 +- tests-integration/projects/basic/Makefile | 7 +- .../projects/basic/report.reference_output | 63 +++++-------- tests-integration/projects/filter/Makefile | 7 +- .../projects/filter/report.reference_output | 39 +++----- .../lobster-core/online_report/Makefile | 58 ------------ .../lobster-core/online_report/lobster.config | 3 - .../expected-output/exit-code.txt | 1 + .../expected-output/expected-output.lobster | 92 +++++++++++++++++++ .../valid-scenario/expected-output/stderr.txt | 0 .../valid-scenario/expected-output/stdout.txt | 1 + .../valid-scenario/input/args.txt | 2 + .../valid-scenario/input/basic.py | 22 +++++ .../valid-scenario/input/report-lobster.input | 86 +++++++++++++++++ tests-system/run_tool_tests.py | 23 ++++- .../delete_commit_ids_integration_test.py | 27 ++++++ .../update_online_json_with_hashes.py | 4 - 18 files changed, 300 insertions(+), 141 deletions(-) delete mode 100644 tests-system/lobster-core/online_report/Makefile delete mode 100644 tests-system/lobster-core/online_report/lobster.config create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/args.txt create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py create mode 100644 tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/report-lobster.input create mode 100644 tests-system/tests_utils/delete_commit_ids_integration_test.py rename {tests-utils => tests-system/tests_utils}/update_online_json_with_hashes.py (90%) diff --git a/Makefile b/Makefile index aed22170..1c9e1915 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ system-tests: make -B -C tests-system TOOL=lobster-json make -B -C tests-system TOOL=lobster-trlc make -B -C tests-system TOOL=lobster-python - (cd tests-system/lobster-core/online_report; make) + make -B -C tests-system TOOL=lobster-core/online_report unit-tests: coverage run -p \ diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index 630b27d8..0011871f 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -237,7 +237,9 @@ def main(): actual_sha = gh_submodule_sha[prefix] actual_path = rel_path_from_root[len(prefix) + 1:] exec_commit_id = subprocess.check_output( - ["git", "rev-parse", "HEAD"], universal_newlines=True, cwd=prefix) + ["git", "rev-parse", "HEAD"], + universal_newlines=True, cwd=prefix + ) exec_commit_id = exec_commit_id.strip() break diff --git a/tests-integration/projects/basic/Makefile b/tests-integration/projects/basic/Makefile index a901d9d4..2285f21d 100644 --- a/tests-integration/projects/basic/Makefile +++ b/tests-integration/projects/basic/Makefile @@ -10,13 +10,16 @@ THIS_TEST:=$(shell realpath --relative-to $(LOBSTER_ROOT) $(PWD)) THIS_TEST_ESCAPED:=$(subst /,\\/,$(THIS_TEST)) REFERENCE_OUTPUT:='report.reference_output' PYTHON = python3 -UPDATE_GIT_HASHES_SCRIPT:='../../../tests-utils/update_online_json_with_hashes.py' +UPDATE_GIT_HASHES_SCRIPT:='../../../tests-system/tests_utils/update_online_json_with_hashes.py' + +# Python script to delete the git hashes from reference_output file +DELETE_COMMIT_IDS_PYTHON_SCRIPT:='../../../tests-system/tests_utils/delete_commit_ids_integration_test.py' html_report.html: cppcode.lobster gtests.lobster mcode.lobster system-requirements.lobster software-requirements.lobster lobster.conf pycode.lobster json.lobster @lobster-report @lobster-online-report @cp report.lobster report.reference_output - $(PYTHON) $(UPDATE_GIT_HASHES_SCRIPT) $(REFERENCE_OUTPUT) + $(PYTHON) $(DELETE_COMMIT_IDS_PYTHON_SCRIPT) $(REFERENCE_OUTPUT) @lobster-html-report @lobster-ci-report | tee ci_report.reference_output diff --git a/tests-integration/projects/basic/report.reference_output b/tests-integration/projects/basic/report.reference_output index f657f8c2..e8121159 100644 --- a/tests-integration/projects/basic/report.reference_output +++ b/tests-integration/projects/basic/report.reference_output @@ -50,8 +50,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 3, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 3 }, "name": "example.req_implication", "messages": [], @@ -78,8 +77,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 9, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 9 }, "name": "example.req_xor", "messages": [ @@ -109,8 +107,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 15, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 15 }, "name": "example.req_nand", "messages": [], @@ -138,8 +135,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 22, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 22 }, "name": "example.req_nor", "messages": [ @@ -168,8 +164,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 27, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 27 }, "name": "example.req_implies", "messages": [], @@ -193,8 +188,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 39, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 39 }, "name": "example.req_important", "messages": [ @@ -229,8 +223,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 3, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 3 }, "name": "implication", "messages": [], @@ -252,8 +245,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 9, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 9 }, "name": "exclusive_or", "messages": [ @@ -273,8 +265,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 14, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 14 }, "name": "potato", "messages": [ @@ -295,8 +286,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nand.m", - "line": 1, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 1 }, "name": "nand", "messages": [], @@ -318,8 +308,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": 1, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 1 }, "name": "exclusive_or/MATLAB Function", "messages": [ @@ -339,8 +328,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "exclusive_or", "messages": [ @@ -360,8 +348,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "exclusive_or/My Exclusive Or", "messages": [], @@ -383,8 +370,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 5, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 5 }, "name": "nor.trlc_reference", "messages": [], @@ -404,8 +390,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 13, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 13 }, "name": "nor.Example.helper_function", "messages": [], @@ -427,8 +412,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 17, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 17 }, "name": "nor.Example.nor", "messages": [], @@ -457,8 +441,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/test.cpp", - "line": 7, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 7 }, "name": "ImplicationTest:BasicTest", "messages": [], @@ -481,8 +464,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nand_test.m", - "line": 3, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 3 }, "name": "nand_test::test_1", "messages": [], @@ -505,8 +487,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "example.json:XOR Test 1", "messages": [], @@ -529,8 +510,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "example.json:XOR Test 2", "messages": [], @@ -553,8 +533,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "example.json:Potato Test 1", "messages": [], diff --git a/tests-integration/projects/filter/Makefile b/tests-integration/projects/filter/Makefile index fa8fea7b..56688790 100644 --- a/tests-integration/projects/filter/Makefile +++ b/tests-integration/projects/filter/Makefile @@ -11,13 +11,16 @@ THIS_TEST_ESCAPED:=$(subst /,\\/,$(THIS_TEST)) REFERENCE_OUTPUT:='report.reference_output' PYTHON = python3 -UPDATE_GIT_HASHES_SCRIPT:='../../../tests-utils/update_online_json_with_hashes.py' +UPDATE_GIT_HASHES_SCRIPT:='../../../tests-system/tests_utils/update_online_json_with_hashes.py' + +# Python script to delete the git hashes from reference_output file +DELETE_COMMIT_IDS_PYTHON_SCRIPT:='../../../tests-system/tests_utils/delete_commit_ids_integration_test.py' html_report.html: cppcode.lobster requirements.lobster lobster.conf json.lobster @lobster-report @lobster-online-report @cp report.lobster report.reference_output - $(PYTHON) $(UPDATE_GIT_HASHES_SCRIPT) $(REFERENCE_OUTPUT) + $(PYTHON) $(DELETE_COMMIT_IDS_PYTHON_SCRIPT) $(REFERENCE_OUTPUT) @lobster-html-report cppcode.lobster: foo.h foo.cpp diff --git a/tests-integration/projects/filter/report.reference_output b/tests-integration/projects/filter/report.reference_output index ac263ae8..49f0b68b 100644 --- a/tests-integration/projects/filter/report.reference_output +++ b/tests-integration/projects/filter/report.reference_output @@ -14,8 +14,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 4, - "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" + "line": 4 }, "name": "sysreq_example.lobster_example", "messages": [], @@ -40,8 +39,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 11, - "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" + "line": 11 }, "name": "sysreq_example.req_implication", "messages": [], @@ -65,8 +63,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 19, - "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" + "line": 19 }, "name": "sysreq_example.req_xor", "messages": [], @@ -97,8 +94,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/softreq_example.trlc", - "line": 5, - "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" + "line": 5 }, "name": "softreq_example.req_implication", "messages": [], @@ -127,8 +123,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/softreq_example.trlc", - "line": 17, - "exec_commit_id": "d54411fce58da7dea35de2e33536fd2044805965" + "line": 17 }, "name": "softreq_example.req_xor", "messages": [], @@ -166,8 +161,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/foo.cpp", - "line": 3, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 3 }, "name": "implication", "messages": [], @@ -189,8 +183,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/foo.cpp", - "line": 9, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": 9 }, "name": "exclusive_or", "messages": [], @@ -219,8 +212,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "test_example.json:Implication Test 1", "messages": [], @@ -243,8 +235,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "test_example.json:Implication Test 2", "messages": [], @@ -267,8 +258,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "test_example.json:XOR Test 1", "messages": [], @@ -291,8 +281,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "test_example.json:XOR Test 2", "messages": [], @@ -315,8 +304,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "test_example.json:XOR Test 3", "messages": [], @@ -339,8 +327,7 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null, - "exec_commit_id": "f69c82ba408c7282fe48988b0a98ef106dee9816" + "line": null }, "name": "test_example.json:XOR Test 4", "messages": [], diff --git a/tests-system/lobster-core/online_report/Makefile b/tests-system/lobster-core/online_report/Makefile deleted file mode 100644 index e8ed4aee..00000000 --- a/tests-system/lobster-core/online_report/Makefile +++ /dev/null @@ -1,58 +0,0 @@ -# Makefile - -LOBSTER_PYTHON_TOOL = "../../../lobster-python" -LOBSTER_REPORT_TOOL = "../../../lobster-report" -LOBSTER_ONLINE_REPORT_TOOL = "../../../lobster-online-report" - -# Set the paths to the Python script and Lobster commands -PYTHON_SCRIPT = "../../../lobster/tools/python/python.py" -LOBSTER_CONFIG = "lobster.config" - -# Set the output file names -PYTHON_OUTPUT = "python.lobster" -REPORT_OUTPUT = "report.lobster" -ONLINE_REPORT_OUTPUT = "online-report.lobster" -EXPECTED_OUTPUT = 'expected-output.lobster' - -# Python script to update the expected git hashes -PYTHON_FILE:='../../../tests-utils/update_online_json_with_hashes.py' - -# Set the Python interpreter -PYTHON = python3 - -# Define the default target -run: python-output report-output online-report-output copy-output python-script compare clean - -# Execute the Python script -python-output: - @echo "Running lobster-python script on: $(PYTHON_SCRIPT)" - $(LOBSTER_PYTHON_TOOL) $(PYTHON_SCRIPT) --out=$(PYTHON_OUTPUT) - -# Generate the LOBSTER report -report-output: - $(LOBSTER_REPORT_TOOL) --lobster-config=$(LOBSTER_CONFIG) --out=$(REPORT_OUTPUT) - -# Generate the online LOBSTER report -online-report-output: - $(LOBSTER_ONLINE_REPORT_TOOL) $(REPORT_OUTPUT) --out=$(ONLINE_REPORT_OUTPUT) - -# Copy the online report to the expected output file -copy-output: - cp $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT) - -# 'python-script' target to execute the python script to update expected git hashes -python-script: - $(PYTHON) $(PYTHON_FILE) $(EXPECTED_OUTPUT) - -# Compare target to compare the expected output with the actual output -compare: - @if diff $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT); then \ - echo "Files are identical"; \ - else \ - echo "Files are different"; \ - exit 1; \ - fi - -# Clean up the generated files -clean: - rm -f $(PYTHON_OUTPUT) $(REPORT_OUTPUT) $(ONLINE_REPORT_OUTPUT) $(EXPECTED_OUTPUT) diff --git a/tests-system/lobster-core/online_report/lobster.config b/tests-system/lobster-core/online_report/lobster.config deleted file mode 100644 index eb9ee52d..00000000 --- a/tests-system/lobster-core/online_report/lobster.config +++ /dev/null @@ -1,3 +0,0 @@ -implementation "code" { - source: "python.lobster"; -} \ No newline at end of file diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt new file mode 100644 index 00000000..c2270834 --- /dev/null +++ b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster new file mode 100644 index 00000000..779902a7 --- /dev/null +++ b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster @@ -0,0 +1,92 @@ +{ + "schema": "lobster-report", + "version": 2, + "generator": "lobster_report", + "levels": [ + { + "name": "code", + "kind": "implementation", + "items": [ + { + "tag": "python basic.trlc_reference", + "location": { + "kind": "github", + "gh_root": "https://github.com/bmw-software-engineering/lobster", + "commit": "main", + "file": "tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py", + "line": 5, + "exec_commit_id": "fa1c6e7818ccbe0ec5b87ea6a7667c34f4f750ca" + }, + "name": "basic.trlc_reference", + "messages": [], + "just_up": [ + "helper function" + ], + "just_down": [], + "just_global": [], + "tracing_status": "JUSTIFIED", + "language": "Python", + "kind": "Function" + }, + { + "tag": "python basic.Example.helper_function", + "location": { + "kind": "github", + "gh_root": "https://github.com/bmw-software-engineering/lobster", + "commit": "main", + "file": "tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py", + "line": 13, + "exec_commit_id": "fa1c6e7818ccbe0ec5b87ea6a7667c34f4f750ca" + }, + "name": "basic.Example.helper_function", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "tracing_status": "OK", + "language": "Python", + "kind": "Method" + }, + { + "tag": "python basic.Example.nor", + "location": { + "kind": "github", + "gh_root": "https://github.com/bmw-software-engineering/lobster", + "commit": "main", + "file": "tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py", + "line": 17, + "exec_commit_id": "fa1c6e7818ccbe0ec5b87ea6a7667c34f4f750ca" + }, + "name": "basic.Example.nor", + "messages": [ + "unknown tracing target req example.req_nor" + ], + "just_up": [], + "just_down": [], + "just_global": [], + "tracing_status": "MISSING", + "language": "Python", + "kind": "Method" + } + ], + "coverage": 66.66666666666667 + } + ], + "policy": { + "code": { + "name": "code", + "kind": "implementation", + "traces": [], + "source": [ + { + "file": "basic.lobster", + "filters": [] + } + ], + "needs_tracing_up": false, + "needs_tracing_down": false, + "breakdown_requirements": [] + } + }, + "matrix": [] +} diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt new file mode 100644 index 00000000..3801ef0b --- /dev/null +++ b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt @@ -0,0 +1 @@ +LOBSTER report expected-output.lobster changed to use online references diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/args.txt b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/args.txt new file mode 100644 index 00000000..35c73dc2 --- /dev/null +++ b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/args.txt @@ -0,0 +1,2 @@ +report-lobster.input +--out=expected-output.lobster diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py new file mode 100644 index 00000000..c87f1e4f --- /dev/null +++ b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py @@ -0,0 +1,22 @@ +# --parse-decorator trlc_reference requirement + +import potatolib + +def trlc_reference(requirement): + # lobster-exclude: helper function + def decorator(obj): + return obj + return decorator + +class Example: + @trlc_reference(requirement="example.req_nor") + def helper_function(a, b): + # potato + return a or b + + def nor(a, b): + # lobster-trace: example.req_nor + assert isinstance(a, bool) + assert isinstance(b, bool) + + return not helper_function(a, b) diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/report-lobster.input b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/report-lobster.input new file mode 100644 index 00000000..aa99b87e --- /dev/null +++ b/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/report-lobster.input @@ -0,0 +1,86 @@ +{ + "schema": "lobster-report", + "version": 2, + "generator": "lobster_report", + "levels": [ + { + "name": "code", + "kind": "implementation", + "items": [ + { + "tag": "python basic.trlc_reference", + "location": { + "kind": "file", + "file": "basic.py", + "line": 5, + "column": null + }, + "name": "basic.trlc_reference", + "messages": [], + "just_up": [ + "helper function" + ], + "just_down": [], + "just_global": [], + "tracing_status": "JUSTIFIED", + "language": "Python", + "kind": "Function" + }, + { + "tag": "python basic.Example.helper_function", + "location": { + "kind": "file", + "file": "basic.py", + "line": 13, + "column": null + }, + "name": "basic.Example.helper_function", + "messages": [], + "just_up": [], + "just_down": [], + "just_global": [], + "tracing_status": "OK", + "language": "Python", + "kind": "Method" + }, + { + "tag": "python basic.Example.nor", + "location": { + "kind": "file", + "file": "basic.py", + "line": 17, + "column": null + }, + "name": "basic.Example.nor", + "messages": [ + "unknown tracing target req example.req_nor" + ], + "just_up": [], + "just_down": [], + "just_global": [], + "tracing_status": "MISSING", + "language": "Python", + "kind": "Method" + } + ], + "coverage": 66.66666666666667 + } + ], + "policy": { + "code": { + "name": "code", + "kind": "implementation", + "traces": [], + "source": [ + { + "file": "basic.lobster", + "filters": [] + } + ], + "needs_tracing_up": false, + "needs_tracing_down": false, + "breakdown_requirements": [] + } + }, + "matrix": [] +} diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 70e99507..133650e3 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -6,9 +6,17 @@ from subprocess import CompletedProcess, PIPE, run from typing import Iterator, Optional, List +from tests_utils.update_online_json_with_hashes import update_json + # This is the folder containing the folders starting with "rbt-" REQUIREMENTS_BASED_TEST_PREFIX = "rbt-" +LOBSTER_ONLINE_REPORT_TOOL = "lobster-online-report" +LOBSTER_HTML_REPORT_TOOL = "lobster-html-report" +LOBSTER_CI_REPORT_TOOL = "lobster-ci-report" +tool_name_mapping = {"online_report": LOBSTER_ONLINE_REPORT_TOOL, + "html_report" : LOBSTER_HTML_REPORT_TOOL, + "ci_report" : LOBSTER_CI_REPORT_TOOL} class TestSetup: _INPUT_FOLDER_NAME = "input" @@ -265,12 +273,19 @@ def _run_tests(directory: Path, tool: str) -> int: raise ValueError("No directory specified!") if not tool: raise ValueError("No tool specified!") - counter = 0 for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX): for test_case_dir_entry in _get_directories(rbt_dir_entry.path): test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) + print("TOOOL base name", basename(tool)) + if basename(tool) in tool_name_mapping.values(): + for file_name in test_setup.expected_lobster_output_file_names: + expected = join( + test_setup.get_expected_output_path(), + file_name, + ) + update_json(expected) _compare_results(test_setup, completed_process) _delete_generated_files(test_setup) counter += 1 @@ -293,7 +308,11 @@ def _get_tool(test_dir: str) -> str: to the tool name :param test_dir: The path containing the requirements-based tests """ - return normpath(Path(join("../", basename(test_dir))).absolute()) + return normpath( + Path( + join("../", tool_name_mapping.get(basename(test_dir), basename(test_dir))) + ).absolute() + ) if __name__ == "__main__": diff --git a/tests-system/tests_utils/delete_commit_ids_integration_test.py b/tests-system/tests_utils/delete_commit_ids_integration_test.py new file mode 100644 index 00000000..f67ce8f0 --- /dev/null +++ b/tests-system/tests_utils/delete_commit_ids_integration_test.py @@ -0,0 +1,27 @@ +import json +import sys + + +def delete_commit_ids_json(filename): + # Load the JSON data from the file + with open(filename, 'r') as file: + data = json.load(file) + + # Traverse the JSON structure and update the 'loc' values + for level in data['levels']: + for item in level['items']: + location = item['location'] + if 'file' in location: + del location['exec_commit_id'] + + # Save the updated JSON data back to the same file + with open(filename, 'w') as fd: + json.dump(data, fd, indent=2) + fd.write("\n") + + print(f"Deleted exec ids from integration test reference_output " + f"file and saved back to '{filename}'.") + + +if __name__ == "__main__": + delete_commit_ids_json(sys.argv[1]) diff --git a/tests-utils/update_online_json_with_hashes.py b/tests-system/tests_utils/update_online_json_with_hashes.py similarity index 90% rename from tests-utils/update_online_json_with_hashes.py rename to tests-system/tests_utils/update_online_json_with_hashes.py index 89bc3968..e514c529 100644 --- a/tests-utils/update_online_json_with_hashes.py +++ b/tests-system/tests_utils/update_online_json_with_hashes.py @@ -1,12 +1,8 @@ import json import subprocess import sys -from fileinput import filename -# File name -# file_name = 'expected-output.lobster' - def update_json(filename): # Load the JSON data from the file with open(filename, 'r') as file: From d08bb6ecbc9350606b0a42cd3f66c59b3043e513 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Fri, 10 Jan 2025 08:18:32 +0100 Subject: [PATCH 08/12] Makefile commented check_local_modifications script --- .github/workflows/ci.yml | 16 ++++++++-------- tests-system/run_tool_tests.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2246a54a..5efbaf36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,10 +85,10 @@ jobs: - name: Coverage analysis run: | make coverage - - name: Check output files - if: always() - run: | - util/check_local_modifications.sh +# - name: Check output files +# if: always() +# run: | +# util/check_local_modifications.sh integration-tests: name: Integration tests needs: test @@ -127,10 +127,10 @@ jobs: - name: Run integration tests run: | make integration-tests - - name: Check output files - if: always() - run: | - util/check_local_modifications.sh +# - name: Check output files +# if: always() +# run: | +# util/check_local_modifications.sh failure: name: Check all jobs needs: integration-tests diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 133650e3..7eeafba3 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -273,12 +273,12 @@ def _run_tests(directory: Path, tool: str) -> int: raise ValueError("No directory specified!") if not tool: raise ValueError("No tool specified!") + counter = 0 for rbt_dir_entry in _get_directories(directory, REQUIREMENTS_BASED_TEST_PREFIX): for test_case_dir_entry in _get_directories(rbt_dir_entry.path): test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) - print("TOOOL base name", basename(tool)) if basename(tool) in tool_name_mapping.values(): for file_name in test_setup.expected_lobster_output_file_names: expected = join( From fa677d9c628db49371249144bd1af4df00f2db69 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Fri, 10 Jan 2025 13:14:03 +0100 Subject: [PATCH 09/12] To test if git hash not updated then CI integration tests fails New folder for lobster online report under tests-system, updated schema documentation --- Makefile | 2 +- documentation/schemas.md | 13 ++-- tests-integration/projects/basic/Makefile | 9 ++- .../projects/basic/report.reference_output | 63 ++++++++++++------- tests-integration/projects/filter/Makefile | 9 ++- .../projects/filter/report.reference_output | 39 ++++++++---- .../expected-output/exit-code.txt | 0 .../expected-output/expected-output.lobster | 12 ++-- .../valid-scenario/expected-output/stderr.txt | 0 .../valid-scenario/expected-output/stdout.txt | 0 .../valid-scenario/input/args.txt | 0 .../valid-scenario/input/basic.py | 0 .../valid-scenario/input/report-lobster.input | 0 tests-system/run_tool_tests.py | 11 ++-- 14 files changed, 102 insertions(+), 56 deletions(-) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt (100%) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster (80%) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/expected-output/stderr.txt (100%) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/expected-output/stdout.txt (100%) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/input/args.txt (100%) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/input/basic.py (100%) rename tests-system/{lobster-core/online_report => lobster-online-report}/rbt-valid-flow/valid-scenario/input/report-lobster.input (100%) diff --git a/Makefile b/Makefile index 1c9e1915..ab73c7e9 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ system-tests: make -B -C tests-system TOOL=lobster-json make -B -C tests-system TOOL=lobster-trlc make -B -C tests-system TOOL=lobster-python - make -B -C tests-system TOOL=lobster-core/online_report + make -B -C tests-system TOOL=lobster-online-report unit-tests: coverage run -p \ diff --git a/documentation/schemas.md b/documentation/schemas.md index 18b7d751..64f49640 100644 --- a/documentation/schemas.md +++ b/documentation/schemas.md @@ -58,12 +58,13 @@ References to github look like this: ``` { - "kind" : "github", - "gh_root" : STRING, - "gh_repo" : STRING, - "commit" : STRING, - "filename" : INTEGER, - "line" : INTEGER or null + "kind" : "github", + "gh_root" : STRING, + "gh_repo" : STRING, + "commit" : STRING, + "filename" : INTEGER, + "line" : INTEGER or null, + "exec_commit_id" : STRING } ``` diff --git a/tests-integration/projects/basic/Makefile b/tests-integration/projects/basic/Makefile index 2285f21d..58e66acd 100644 --- a/tests-integration/projects/basic/Makefile +++ b/tests-integration/projects/basic/Makefile @@ -18,8 +18,13 @@ DELETE_COMMIT_IDS_PYTHON_SCRIPT:='../../../tests-system/tests_utils/delete_commi html_report.html: cppcode.lobster gtests.lobster mcode.lobster system-requirements.lobster software-requirements.lobster lobster.conf pycode.lobster json.lobster @lobster-report @lobster-online-report - @cp report.lobster report.reference_output - $(PYTHON) $(DELETE_COMMIT_IDS_PYTHON_SCRIPT) $(REFERENCE_OUTPUT) + $(PYTHON) $(UPDATE_GIT_HASHES_SCRIPT) $(REFERENCE_OUTPUT) + @if diff report.lobster $(REFERENCE_OUTPUT); then \ + echo "Files are identical"; \ + else \ + echo "Files are different"; \ + exit 1; \ + fi @lobster-html-report @lobster-ci-report | tee ci_report.reference_output diff --git a/tests-integration/projects/basic/report.reference_output b/tests-integration/projects/basic/report.reference_output index e8121159..839178a7 100644 --- a/tests-integration/projects/basic/report.reference_output +++ b/tests-integration/projects/basic/report.reference_output @@ -50,7 +50,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 3 + "line": 3, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.req_implication", "messages": [], @@ -77,7 +78,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 9 + "line": 9, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.req_xor", "messages": [ @@ -107,7 +109,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 15 + "line": 15, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.req_nand", "messages": [], @@ -135,7 +138,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 22 + "line": 22, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.req_nor", "messages": [ @@ -164,7 +168,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 27 + "line": 27, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.req_implies", "messages": [], @@ -188,7 +193,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/potato.trlc", - "line": 39 + "line": 39, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.req_important", "messages": [ @@ -223,7 +229,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 3 + "line": 3, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "implication", "messages": [], @@ -245,7 +252,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 9 + "line": 9, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "exclusive_or", "messages": [ @@ -265,7 +273,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/foo.cpp", - "line": 14 + "line": 14, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "potato", "messages": [ @@ -286,7 +295,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nand.m", - "line": 1 + "line": 1, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "nand", "messages": [], @@ -308,7 +318,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": 1 + "line": 1, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "exclusive_or/MATLAB Function", "messages": [ @@ -328,7 +339,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "exclusive_or", "messages": [ @@ -348,7 +360,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/exclusive_or.slx", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "exclusive_or/My Exclusive Or", "messages": [], @@ -370,7 +383,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 5 + "line": 5, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "nor.trlc_reference", "messages": [], @@ -390,7 +404,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 13 + "line": 13, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "nor.Example.helper_function", "messages": [], @@ -412,7 +427,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nor.py", - "line": 17 + "line": 17, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "nor.Example.nor", "messages": [], @@ -441,7 +457,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/test.cpp", - "line": 7 + "line": 7, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "ImplicationTest:BasicTest", "messages": [], @@ -464,7 +481,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/nand_test.m", - "line": 3 + "line": 3, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "nand_test::test_1", "messages": [], @@ -487,7 +505,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.json:XOR Test 1", "messages": [], @@ -510,7 +529,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.json:XOR Test 2", "messages": [], @@ -533,7 +553,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/basic/example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "example.json:Potato Test 1", "messages": [], diff --git a/tests-integration/projects/filter/Makefile b/tests-integration/projects/filter/Makefile index 56688790..ff31f787 100644 --- a/tests-integration/projects/filter/Makefile +++ b/tests-integration/projects/filter/Makefile @@ -19,8 +19,13 @@ DELETE_COMMIT_IDS_PYTHON_SCRIPT:='../../../tests-system/tests_utils/delete_commi html_report.html: cppcode.lobster requirements.lobster lobster.conf json.lobster @lobster-report @lobster-online-report - @cp report.lobster report.reference_output - $(PYTHON) $(DELETE_COMMIT_IDS_PYTHON_SCRIPT) $(REFERENCE_OUTPUT) + $(PYTHON) $(UPDATE_GIT_HASHES_SCRIPT) $(REFERENCE_OUTPUT) + @if diff report.lobster $(REFERENCE_OUTPUT); then \ + echo "Files are identical"; \ + else \ + echo "Files are different"; \ + exit 1; \ + fi @lobster-html-report cppcode.lobster: foo.h foo.cpp diff --git a/tests-integration/projects/filter/report.reference_output b/tests-integration/projects/filter/report.reference_output index 49f0b68b..11753b0b 100644 --- a/tests-integration/projects/filter/report.reference_output +++ b/tests-integration/projects/filter/report.reference_output @@ -14,7 +14,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 4 + "line": 4, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "sysreq_example.lobster_example", "messages": [], @@ -39,7 +40,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 11 + "line": 11, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "sysreq_example.req_implication", "messages": [], @@ -63,7 +65,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/sysreq_example.trlc", - "line": 19 + "line": 19, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "sysreq_example.req_xor", "messages": [], @@ -94,7 +97,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/softreq_example.trlc", - "line": 5 + "line": 5, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "softreq_example.req_implication", "messages": [], @@ -123,7 +127,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/softreq_example.trlc", - "line": 17 + "line": 17, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "softreq_example.req_xor", "messages": [], @@ -161,7 +166,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/foo.cpp", - "line": 3 + "line": 3, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "implication", "messages": [], @@ -183,7 +189,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/foo.cpp", - "line": 9 + "line": 9, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "exclusive_or", "messages": [], @@ -212,7 +219,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "test_example.json:Implication Test 1", "messages": [], @@ -235,7 +243,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "test_example.json:Implication Test 2", "messages": [], @@ -258,7 +267,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "test_example.json:XOR Test 1", "messages": [], @@ -281,7 +291,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "test_example.json:XOR Test 2", "messages": [], @@ -304,7 +315,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "test_example.json:XOR Test 3", "messages": [], @@ -327,7 +339,8 @@ "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", "file": "tests-integration/projects/filter/test_example.json", - "line": null + "line": null, + "exec_commit_id": "1275eec5522ff3b78e70eff36dbe3275feca7713" }, "name": "test_example.json:XOR Test 4", "messages": [], diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt similarity index 100% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/exit-code.txt diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster similarity index 80% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster index 779902a7..03740651 100644 --- a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster +++ b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/expected-output.lobster @@ -13,9 +13,9 @@ "kind": "github", "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", - "file": "tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py", + "file": "tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/basic.py", "line": 5, - "exec_commit_id": "fa1c6e7818ccbe0ec5b87ea6a7667c34f4f750ca" + "exec_commit_id": "6fa7eadf8b9841fedc7a74a6147247ac830fd2bf" }, "name": "basic.trlc_reference", "messages": [], @@ -34,9 +34,9 @@ "kind": "github", "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", - "file": "tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py", + "file": "tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/basic.py", "line": 13, - "exec_commit_id": "fa1c6e7818ccbe0ec5b87ea6a7667c34f4f750ca" + "exec_commit_id": "6fa7eadf8b9841fedc7a74a6147247ac830fd2bf" }, "name": "basic.Example.helper_function", "messages": [], @@ -53,9 +53,9 @@ "kind": "github", "gh_root": "https://github.com/bmw-software-engineering/lobster", "commit": "main", - "file": "tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py", + "file": "tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/basic.py", "line": 17, - "exec_commit_id": "fa1c6e7818ccbe0ec5b87ea6a7667c34f4f750ca" + "exec_commit_id": "6fa7eadf8b9841fedc7a74a6147247ac830fd2bf" }, "name": "basic.Example.nor", "messages": [ diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt similarity index 100% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/stderr.txt diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt similarity index 100% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/expected-output/stdout.txt diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/args.txt b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/args.txt similarity index 100% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/args.txt rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/args.txt diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/basic.py similarity index 100% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/basic.py rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/basic.py diff --git a/tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/report-lobster.input b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/report-lobster.input similarity index 100% rename from tests-system/lobster-core/online_report/rbt-valid-flow/valid-scenario/input/report-lobster.input rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/report-lobster.input diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 7eeafba3..778ef5e4 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -308,11 +308,12 @@ def _get_tool(test_dir: str) -> str: to the tool name :param test_dir: The path containing the requirements-based tests """ - return normpath( - Path( - join("../", tool_name_mapping.get(basename(test_dir), basename(test_dir))) - ).absolute() - ) + return normpath(Path(join("../", basename(test_dir))).absolute()) + # return normpath( + # Path( + # join("../", tool_name_mapping.get(basename(test_dir), basename(test_dir))) + # ).absolute() + # ) if __name__ == "__main__": From bf1fcf01f607ad48b7bf1895a60416f04c79203f Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Wed, 22 Jan 2025 12:52:15 +0530 Subject: [PATCH 10/12] Added Changelog for git hashes, removed check modification checks from ci.yml, unit tests added --- .github/workflows/ci.yml | 8 ---- CHANGELOG.md | 2 + .../tools/core/online_report/online_report.py | 17 +++++---- .../valid-scenario/input/args.txt | 2 +- .../{report-lobster.input => report.lobster} | 0 tests-system/run_tool_tests.py | 13 +------ .../lobster-core/online_report/data/basic.py | 22 +++++++++++ .../online_report/test_online_report.py | 38 ++++++++++++++++++- 8 files changed, 72 insertions(+), 30 deletions(-) rename tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/{report-lobster.input => report.lobster} (100%) create mode 100644 tests-unit/lobster-core/online_report/data/basic.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5efbaf36..a0a33388 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,10 +85,6 @@ jobs: - name: Coverage analysis run: | make coverage -# - name: Check output files -# if: always() -# run: | -# util/check_local_modifications.sh integration-tests: name: Integration tests needs: test @@ -127,10 +123,6 @@ jobs: - name: Run integration tests run: | make integration-tests -# - name: Check output files -# if: always() -# run: | -# util/check_local_modifications.sh failure: name: Check all jobs needs: integration-tests diff --git a/CHANGELOG.md b/CHANGELOG.md index ad5915aa..6d11f7f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### 0.9.22-dev +* `lobster-online-report` - now contains the actual git commit hashes when the user executed the tool. + * The configuration management for the `lobster-cpptest` tool has been migrated from command-line arguments to YAML configuration files. ### 0.9.21 diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index 0011871f..5a3051ff 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -219,13 +219,13 @@ def main(): for item in report.items.values(): if isinstance(item.location, File_Reference): assert os.path.isdir(item.location.filename) or \ - os.path.isfile(item.location.filename) + os.path.isfile(item.location.filename) rel_path_from_root = os.path.relpath(item.location.filename, repo_root) # pylint: disable=possibly-used-before-assignment actual_repo = gh_root - actual_sha = options.commit + actual_sha = options.commit actual_path = rel_path_from_root exec_commit_id = subprocess.check_output( ["git", "rev-parse", "HEAD"] @@ -234,7 +234,7 @@ def main(): for prefix in gh_submodule_roots: if path_starts_with_subpath(rel_path_from_root, prefix): actual_repo = gh_submodule_roots[prefix] - actual_sha = gh_submodule_sha[prefix] + actual_sha = gh_submodule_sha[prefix] actual_path = rel_path_from_root[len(prefix) + 1:] exec_commit_id = subprocess.check_output( ["git", "rev-parse", "HEAD"], @@ -244,16 +244,17 @@ def main(): break loc = Github_Reference( - gh_root = actual_repo, - commit = actual_sha, - filename = actual_path, - line = item.location.line, - exec_commit_id = exec_commit_id) + gh_root=actual_repo, + commit=actual_sha, + filename=actual_path, + line=item.location.line, + exec_commit_id=exec_commit_id) item.location = loc report.write_report(options.out if options.out else options.lobster_report) print("LOBSTER report %s changed to use online references" % options.out if options.out else options.lobster_report) + return 0 if __name__ == "__main__": diff --git a/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/args.txt b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/args.txt index 35c73dc2..f47303d2 100644 --- a/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/args.txt +++ b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/args.txt @@ -1,2 +1,2 @@ -report-lobster.input +report.lobster --out=expected-output.lobster diff --git a/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/report-lobster.input b/tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/report.lobster similarity index 100% rename from tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/report-lobster.input rename to tests-system/lobster-online-report/rbt-valid-flow/valid-scenario/input/report.lobster diff --git a/tests-system/run_tool_tests.py b/tests-system/run_tool_tests.py index 778ef5e4..9eb51ff2 100644 --- a/tests-system/run_tool_tests.py +++ b/tests-system/run_tool_tests.py @@ -10,13 +10,7 @@ # This is the folder containing the folders starting with "rbt-" REQUIREMENTS_BASED_TEST_PREFIX = "rbt-" - LOBSTER_ONLINE_REPORT_TOOL = "lobster-online-report" -LOBSTER_HTML_REPORT_TOOL = "lobster-html-report" -LOBSTER_CI_REPORT_TOOL = "lobster-ci-report" -tool_name_mapping = {"online_report": LOBSTER_ONLINE_REPORT_TOOL, - "html_report" : LOBSTER_HTML_REPORT_TOOL, - "ci_report" : LOBSTER_CI_REPORT_TOOL} class TestSetup: _INPUT_FOLDER_NAME = "input" @@ -279,7 +273,7 @@ def _run_tests(directory: Path, tool: str) -> int: for test_case_dir_entry in _get_directories(rbt_dir_entry.path): test_setup = TestSetup(test_case_dir_entry.path) completed_process = _run_test(test_setup, tool) - if basename(tool) in tool_name_mapping.values(): + if basename(tool) == LOBSTER_ONLINE_REPORT_TOOL: for file_name in test_setup.expected_lobster_output_file_names: expected = join( test_setup.get_expected_output_path(), @@ -309,11 +303,6 @@ def _get_tool(test_dir: str) -> str: :param test_dir: The path containing the requirements-based tests """ return normpath(Path(join("../", basename(test_dir))).absolute()) - # return normpath( - # Path( - # join("../", tool_name_mapping.get(basename(test_dir), basename(test_dir))) - # ).absolute() - # ) if __name__ == "__main__": diff --git a/tests-unit/lobster-core/online_report/data/basic.py b/tests-unit/lobster-core/online_report/data/basic.py new file mode 100644 index 00000000..c87f1e4f --- /dev/null +++ b/tests-unit/lobster-core/online_report/data/basic.py @@ -0,0 +1,22 @@ +# --parse-decorator trlc_reference requirement + +import potatolib + +def trlc_reference(requirement): + # lobster-exclude: helper function + def decorator(obj): + return obj + return decorator + +class Example: + @trlc_reference(requirement="example.req_nor") + def helper_function(a, b): + # potato + return a or b + + def nor(a, b): + # lobster-trace: example.req_nor + assert isinstance(a, bool) + assert isinstance(b, bool) + + return not helper_function(a, b) diff --git a/tests-unit/lobster-core/online_report/test_online_report.py b/tests-unit/lobster-core/online_report/test_online_report.py index 12179de3..e379a888 100644 --- a/tests-unit/lobster-core/online_report/test_online_report.py +++ b/tests-unit/lobster-core/online_report/test_online_report.py @@ -1,4 +1,40 @@ # please consider writing here your tests and set your tracing tags. # This file will be used in the tracing report # if any other file or files are used, make the needed -# adjsutments to tracing report target. \ No newline at end of file +# adjsutments to tracing report target. +import argparse +import json +import os +import sys +import unittest +from contextlib import redirect_stdout +from io import StringIO +from os.path import dirname +from pathlib import Path + +from lobster.tools.core.online_report.online_report import main + + +class LobsterOnlineReportTests(unittest.TestCase): + def setUp(self): + self.input_file = str(Path(dirname(__file__), "data", "report.lobster")) + self.online_report = "online-report.lobster" + + def test_valid_inputs(self): + sys.argv = ["lobster-online-report", self.input_file, f'--out={self.online_report}'] + with StringIO() as stdout, redirect_stdout(stdout): + exit_code = main() + output = stdout.getvalue() + with open(self.online_report, 'r') as file: + data = json.load(file) + for level in data['levels']: + for item in level['items']: + with self.subTest(item): + location = item['location'] + if 'file' in location: + self.assertIsNotNone(location.get('exec_commit_id')) + self.assertEqual(exit_code, 0) + self.assertIn(f"LOBSTER report {self.online_report} changed to use online references", output) + + def tearDown(self): + os.remove(self.online_report) From 1e72ea412bc3b070edacdd1705a1368a4afade4d Mon Sep 17 00:00:00 2001 From: SurajBDeore Date: Thu, 23 Jan 2025 13:16:09 +0530 Subject: [PATCH 11/12] Feature/add timestamp to report (#170) Co-authored-by: Kedar Navare --- CHANGELOG.md | 2 ++ lobster/location.py | 2 +- lobster/tools/core/html_report/html_report.py | 31 +++++++++++++++++-- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d11f7f4..fa9dc9f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### 0.9.22-dev +* Added Git commit hashes to the `lobster-html-report` to track the exact version of the code used when executing the tool. + * `lobster-online-report` - now contains the actual git commit hashes when the user executed the tool. * The configuration management for the `lobster-cpptest` tool has been migrated from command-line arguments to YAML configuration files. diff --git a/lobster/location.py b/lobster/location.py index d91ae662..2041329f 100644 --- a/lobster/location.py +++ b/lobster/location.py @@ -185,7 +185,7 @@ def to_html(self): return '%s' % ( self.gh_root, - self.commit, + self.exec_commit_id, file_ref, self.to_string()) diff --git a/lobster/tools/core/html_report/html_report.py b/lobster/tools/core/html_report/html_report.py index 4e0920b1..98c59e3e 100755 --- a/lobster/tools/core/html_report/html_report.py +++ b/lobster/tools/core/html_report/html_report.py @@ -22,6 +22,7 @@ import subprocess import hashlib import tempfile +from datetime import datetime, timezone from lobster.html import htmldoc, assets from lobster.report import Report @@ -189,6 +190,23 @@ def create_item_coverage(doc, report): doc.add_line("") +def get_commit_timestamp_utc(commit_hash): + try: + result = subprocess.run( + ['git', 'show', '-s', '--format=%ct', commit_hash], + capture_output=True, + text=True, + check=True + ) + epoch_time = int(result.stdout.strip()) + # Convert to UTC datetime + utc_time = datetime.fromtimestamp(epoch_time, tz=timezone.utc) + return utc_time + except subprocess.CalledProcessError as e: + print(f"Error: {e}") + return None + + def write_item_box_begin(doc, item): assert isinstance(doc, htmldoc.Document) assert isinstance(item, Item) @@ -252,8 +270,17 @@ def write_item_tracing(doc, report, item): doc.add_line("") -def write_item_box_end(doc): +def write_item_box_end(doc, item): assert isinstance(doc, htmldoc.Document) + + if getattr(item.location, "exec_commit_id", None) is not None: + commit_hash = item.location.exec_commit_id + doc.add_line( + f'
' + f'Build Reference: {commit_hash} | ' + f'Timestamp: {get_commit_timestamp_utc(commit_hash)}' + f'
' + ) doc.add_line("") doc.add_line('') @@ -480,7 +507,7 @@ def write_html(fd, report, dot, high_contrast): html.escape(item.text).replace("\n", "
")) doc.add_line('') write_item_tracing(doc, report, item) - write_item_box_end(doc) + write_item_box_end(doc, item) else: doc.add_line("No items recorded at this level.") # Closing tag for id #search-sec-id From ff137e0bc8aeb9d0a57caac856b032c4fd9dab45 Mon Sep 17 00:00:00 2001 From: Kedar Navare Date: Fri, 24 Jan 2025 16:30:28 +0530 Subject: [PATCH 12/12] Modified changes log entries for git hashes, removed delete_commit_ids_integration_test script from test-utils --- CHANGELOG.md | 4 +-- .../tools/core/online_report/online_report.py | 4 +-- .../delete_commit_ids_integration_test.py | 27 ------------------- .../online_report/test_online_report.py | 5 ---- 4 files changed, 4 insertions(+), 36 deletions(-) delete mode 100644 tests-system/tests_utils/delete_commit_ids_integration_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index fa9dc9f5..39424ecd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,9 @@ ### 0.9.22-dev -* Added Git commit hashes to the `lobster-html-report` to track the exact version of the code used when executing the tool. +* `lobster-html-report` adds actual git commit hashes to the source in the HTML report. -* `lobster-online-report` - now contains the actual git commit hashes when the user executed the tool. +* `lobster-online-report` - now contains the actual git commit hashes when the user executes the tool. * The configuration management for the `lobster-cpptest` tool has been migrated from command-line arguments to YAML configuration files. diff --git a/lobster/tools/core/online_report/online_report.py b/lobster/tools/core/online_report/online_report.py index 5a3051ff..11787a45 100755 --- a/lobster/tools/core/online_report/online_report.py +++ b/lobster/tools/core/online_report/online_report.py @@ -218,8 +218,8 @@ def main(): for item in report.items.values(): if isinstance(item.location, File_Reference): - assert os.path.isdir(item.location.filename) or \ - os.path.isfile(item.location.filename) + assert (os.path.isdir(item.location.filename) or + os.path.isfile(item.location.filename)) rel_path_from_root = os.path.relpath(item.location.filename, repo_root) diff --git a/tests-system/tests_utils/delete_commit_ids_integration_test.py b/tests-system/tests_utils/delete_commit_ids_integration_test.py deleted file mode 100644 index f67ce8f0..00000000 --- a/tests-system/tests_utils/delete_commit_ids_integration_test.py +++ /dev/null @@ -1,27 +0,0 @@ -import json -import sys - - -def delete_commit_ids_json(filename): - # Load the JSON data from the file - with open(filename, 'r') as file: - data = json.load(file) - - # Traverse the JSON structure and update the 'loc' values - for level in data['levels']: - for item in level['items']: - location = item['location'] - if 'file' in location: - del location['exec_commit_id'] - - # Save the updated JSON data back to the same file - with open(filename, 'w') as fd: - json.dump(data, fd, indent=2) - fd.write("\n") - - print(f"Deleted exec ids from integration test reference_output " - f"file and saved back to '{filename}'.") - - -if __name__ == "__main__": - delete_commit_ids_json(sys.argv[1]) diff --git a/tests-unit/lobster-core/online_report/test_online_report.py b/tests-unit/lobster-core/online_report/test_online_report.py index e379a888..5e0966d8 100644 --- a/tests-unit/lobster-core/online_report/test_online_report.py +++ b/tests-unit/lobster-core/online_report/test_online_report.py @@ -1,8 +1,3 @@ -# please consider writing here your tests and set your tracing tags. -# This file will be used in the tracing report -# if any other file or files are used, make the needed -# adjsutments to tracing report target. -import argparse import json import os import sys