Merge pull request #7

1.4 version

README.md

@@ -78,42 +78,57 @@ to configure test environment, you can use additional option:
 pytest --analyzer sync --testRunEnv windows11,chrome,1920x1080
 ```
 
-### Submitting Test Artifacts (NEW)
+### Submitting Test Artifacts
 
-According documentation, Testomat.io does not store any screenshots,
-logs or other artifacts. In order to manage them it is advised to use S3 Buckets.
+Testomat.io does not store any screenshots,logs or other artifacts.
+
+In order to manage them it is advised to use S3 Buckets (GCP Storage).
 https://docs.testomat.io/usage/test-artifacts/
 
-In order to save artifacts, enable **Share credentials with Testomat.io Reporter** option in testomat.io Settings ->
+In order for analyser to have access to your cloud bucket - enable **Share credentials with Testomat.io Reporter** option in testomat.io Settings ->
 Artifacts.
 
-To send artifact to s3 bucket, next code should be added to test:
+You would need to decide when and where you want to upload your test artifacts to cloud storage
+
+Using pytest fixtures might be a good choice, ex.:
 
 ```python
-# file_path - path to file to be uploaded
-# file_bytes - bytes of the file to be uploaded
-# key - file name in the s3 bucket
-# bucket_name - name of the bucket to upload file to. If not set, bucket name from pytest.ini will be used, if set, overrides bucket name from pytest.ini
-artifact_url = pytest.s3_connector.upload_file(file_path, key, bucket_name)
-# or
-artifact_url = pytest.s3_connector.upload_file_object(file_bytes, key, bucket_name)
+@pytest.fixture(scope="function")
+def page(context, request):
+    page = context.new_page()
+    yield
+    if request.node.rep_call.failed:
+        random_string = ''.join(random.choices(string.ascii_letters + string.digits, k=8))
+
+        filename = f"{random_string}.png"
+        screenshot_path = os.path.join(artifacts_dir, filename)
+        page.screenshot(path=screenshot_path)
+        # file_path - required, path to file to be uploaded
+        # file_bytes - required, bytes of the file to be uploaded
+        # key - required, file name in the s3 bucket
+        # bucket_name - optional,name of the bucket to upload file to. Default value is taken from Testomatio.io
+        artifact_url = pytest.s3_connector.upload_file(screenshot_path, filename)
+        # or
+        # artifact_url = pytest.s3_connector.upload_file_object(file_bytes, key, bucket_name)
+        request.node.testomatio = {"artifacts": [artifact_url]}
+    page.close()
 ```
 
 ⚠️ Please take into account s3_connector available only after **pytest_collection_modifyitems()** hook is executed.
 
-In conftest.py file next hook can be added. set attribute testomatio_artifacts. This list will be sent to testomat.io
+If you prefer to use pytest hooks - add `pytest_runtest_makereport` hook in your `conftest.py` file.
 
 ```python
 def pytest_runtest_makereport(item, call):
-    artifact_urls = ['url1', 'url2']
-    setattr(item, 'testomatio_artifacts', artifact_urls)
+    artifact_url = pytest.s3_connector.upload_file(screenshot_path, filename)
+    item.testomatio = {"artifacts": [artifact_url]}
 ```
 
 Eny environments used in test run. Should be placed in comma separated list, NO SPACES ALLOWED.
 
 ### Clarifications
 
-- tests can be synced even without `@mark.testomatio('@T96c700e6')` decorator.
+- tests can be synced even without `@patest.mark.testomatio('@T96c700e6')` decorator.
 - test title in testomat.io == test name in pytest
 - test suit title in testomat.io == test file name in pytest
 
@@ -123,16 +138,43 @@ To make analyzer experience more consistent, it uses standard pytest markers.
 Testomat.io test id is a string value that starts with `@T` and has 8 symbols after.
 
 ```python
-from pytest import mark
+import pytest
 
 
-@mark.testomatio('@T96c700e6')
+@pytest.mark.testomatio('@T96c700e6')
 def test_example():
     assert 2 + 2 == 4
 ```
 
+### Compatibility table with [Testomatio check-tests](https://github.com/testomatio/check-tests)
+
+| Action |  Compatibility | Method |
+|--------|--------|-------|
+| Importing test into Testomatio | complete | `pytest --analyzer add` |
+| Exclude hook code of a test | N/A | N/A |
+| Include line number code of a test | N/A | N/A |
+| Import Parametrized Tests | complete | default behaviour |
+| Disable Detached Tests | complete | `pytest --analyzer add --no-detached` |
+| Synchronous Import | complete | default behaviour |
+| Auto-assign Test IDs in Source Code | complete | default behaviour |
+| Keep Test IDs Between Projects | complete | `pytest --analyzer add --create` |
+| Clean Test IDs | complete | `pytest --analyzer remove` |
+| Import Into a Branch | N/A | N/A |
+| Keep Structure of Source Code | complete | `pytest --analyzer add --keep-structure` |
+| Delete Empty Suites | complete | `pytest --analyzer add --no-empty` |
+| Import Into a Specific Suite | N/A | N/A |
+| Debugging | parity | `pytest --analyzer debug` |
+
+
 ## Change log
 
+### 1.4.0 - Fixes artifacts and test sync with Testomatio
+- Fixes artifacts uploads
+- Fixes test id resolution when syncing local test with Testomatio
+- Fixes test id when sending test into test run
+- Adds `--create`, `--no-detached`, `--keep-structure`, `--no-empty`,  for compatibility with original Testomatio check-tests
+- Improves file update so it doesn't cause code style changes
+
 ### 1.3.0 - added artifacts support connector
 - [issue 5](https://github.com/Ypurek/pytest-analyzer/issues/5) - connection issues not blocking test execution anymore
 

analyzer/analyzer.py

@@ -32,7 +32,26 @@ def pytest_addoption(parser: Parser) -> None:
     parser.addoption(f'--testRunEnv',
                      action='store',
                      help='specify test run environment for testomat.io. Works only with --analyzer sync')
-
+    parser.addoption(f'--create',
+                    action='store_true',
+                    default=False,
+                    dest="create",
+                    help='To import tests with Test IDs set in source code into a project use --create option. In this case, a new project will be populated with the same Test IDs.. Use --add together with --create option to enable this behavior.')
+    parser.addoption(f'--no-empty',
+                     action='store_true',
+                     default=False,
+                     dest="no_empty",
+                     help='Delete empty suites. If tests were marked with IDs and imported to already created suites in Testomat.io newly imported suites may become empty. Use --add together with --no-empty option to clean them up after import.')
+    parser.addoption(f'--no-detach',
+                     action='store_true',
+                     default=False,
+                     dest="no_detach",
+                     help='Disable detaching tests. If a test from a previous import was not found on next import it is marked as "detached". This is done to ensure that deleted tests are not staying in Testomatio while deleted in codebase. To disable this behavior and don\'t mark anything on detached on import use --add together with --no-detached option.')
+    parser.addoption(f'--keep-structure',
+                     action='store_true',
+                     default=False,
+                     dest="keep_structure",
+                     help='Keep structure of source code. If suites are not created in Testomat.io they will be created based on the file structure. Use --add together with --structure option to enable this behavior.')
     parser.addini('testomatio_url', 'testomat.io base url', default='https://app.testomat.io')
 
 
@@ -61,7 +80,13 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
         match config.getoption(analyzer_option):
             case 'add':
                 connector: Connector = pytest.connector
-                connector.load_tests(meta)
+                connector.load_tests(
+                    meta,
+                    no_empty=config.getoption('no_empty'),
+                    no_detach=config.getoption('no_detach'),
+                    structure=config.getoption('keep_structure'),
+                    create=config.getoption('create')
+                )
                 testomatio_tests = connector.get_tests(meta)
                 add_and_enrich_tests(meta, test_files, test_names, testomatio_tests, decorator_name)
                 pytest.exit(
@@ -90,7 +115,8 @@ def pytest_collection_modifyitems(session: Session, config: Config, items: list[
                     if all((s3_access_key, s3_secret_key, s3_endpoint, s3_bucket)):
                         pytest.s3_connector = S3Connector(s3_access_key, s3_secret_key, s3_endpoint, s3_bucket)
                         pytest.s3_connector.login()
-                pytest.s3_connector = S3Connector('', '', '', '')
+                    else:
+                        pytest.s3_connector = S3Connector('', '', '', '')
             case 'debug':
                 with open(metadata_file, 'w') as file:
                     data = json.dumps([i.to_dict() for i in meta], indent=4)
@@ -109,13 +135,14 @@ def pytest_runtest_makereport(item: Item, call: CallInfo):
         return
 
     test_item = TestItem(item)
+    test_id = test_item.id if not test_item.id.startswith("@T") else test_item.id[2:]
     request = {
         'status': None,
         'title': test_item.title,
         'run_time': call.duration,
         'suite_title': test_item.file_name,
         'suite_id': None,
-        'test_id': test_item.id[2:] if test_item.id else None,  # remove @T if exists
+        'test_id': test_id,
         'message': None,
         'stack': None,
         'example': None,

analyzer/connector.py

@@ -15,19 +15,27 @@ def __init__(self, base_url: str = 'https://app.testomat.io', api_key: str = Non
         self.jwt: str = ''
         self.api_key = api_key
 
-    def load_tests(self, tests: list[TestItem], no_empty: bool = True, no_detach: bool = True):
+    def load_tests(
+            self,
+            tests: list[TestItem],
+            no_empty: bool = False,
+            no_detach: bool = False,
+            structure: bool = False,
+            create: bool = False
+        ):
         request = {
-            "framework": "pytest",
-            "language": "python",
+            "framework": "",
+            "language": "",
             "noempty": no_empty,
             "no-detach": no_detach,
-            "structure": True,
+            "structure": structure if not no_empty else False,
+            "create": create,
             "sync": True,
             "tests": []
         }
         for test in tests:
             request['tests'].append({
-                "name": test.user_title,
+                "name": test.sync_title,
                 "suites": [
                     test.file_name,
                     test.class_name
@@ -48,7 +56,7 @@ def load_tests(self, tests: list[TestItem], no_empty: bool = True, no_detach: bo
             log.error(f'Generic exception happened. Please report an issue. {e}')
             return
 
-        if response.status_code == 200:
+        if response.status_code < 400:
             log.info(f'Tests loaded to {self.base_url}')
         else:
             log.error(f'Failed to load tests to {self.base_url}. Status code: {response.status_code}')

analyzer/decorator_updater.py

@@ -1,11 +1,8 @@
-import ast
-import autopep8
+import libcst as cst
+from typing import List, Tuple, Union
 
-pytest_mark = 'pytest', 'mark'
-
-
-class DecoratorUpdater(ast.NodeTransformer):
-    def __init__(self, mapped_tests: list[tuple[str, int]], all_tests: list[str], decorator_name: str):
+class DecoratorUpdater(cst.CSTTransformer):
+    def __init__(self, mapped_tests: List[Tuple[str, int]], all_tests: List[str], decorator_name: str):
         self.mapped_tests = mapped_tests
         self.all_tests = all_tests
         self.decorator_name = decorator_name
@@ -15,65 +12,67 @@ def _get_id_by_title(self, title: str):
             if pair[0] == title:
                 return pair[1]
 
-    def _remove_decorator(self, node: ast.FunctionDef) -> ast.FunctionDef:
+    def _remove_decorator(self, node: cst.FunctionDef) -> cst.FunctionDef:
         node.decorator_list = [decorator for decorator in node.decorator_list if
-                               not (isinstance(decorator, ast.Call) and decorator.func.attr == self.decorator_name)]
+                               not (isinstance(decorator, cst.Call) and decorator.func.attr == self.decorator_name)]
         return node
 
-    def remove_decorators(self, tree: ast.Module) -> ast.Module:
-        for node in ast.walk(tree):
-            if isinstance(node, ast.FunctionDef):
+    def remove_decorators(self, tree: cst.Module) -> cst.Module:
+        for node in cst.walk(tree):
+            if isinstance(node, cst.FunctionDef):
                 self.visit_FunctionDef(node, remove=True)
         return tree
 
-    def visit_FunctionDef(self, node: ast.FunctionDef, remove=False) -> ast.FunctionDef:
-        if remove:
-            return self._remove_decorator(node)
-        else:
-            if node.name in self.all_tests:
-                if not any(isinstance(decorator, ast.Call) and
-                           decorator.func.attr == self.decorator_name
-                           for decorator in node.decorator_list):
-                    test_id = self._get_id_by_title(node.name)
-                    deco_name = f'mark.{self.decorator_name}(\'{test_id}\')'
-                    decorator = ast.Name(id=deco_name, ctx=ast.Load())
-                    node.decorator_list = [decorator] + node.decorator_list
-        return node
+    def leave_FunctionDef(self, original_node: cst.FunctionDef, updated_node: cst.FunctionDef) -> cst.FunctionDef:
+        if original_node.name.value in self.all_tests:
+            test_id = self._get_id_by_title(original_node.name.value)
+            deco_name = f'pytest.mark.{self.decorator_name}("{test_id}")'
+            decorator = cst.Decorator(decorator=cst.parse_expression(deco_name))
 
-    def insert_pytest_mark_import(self, tree: ast.Module, module_name: str, decorator_name: str) -> None:
-        # Check if the import statement already exists
-        if not any(
-                isinstance(node, ast.ImportFrom) and
-                node.module == module_name and
-                any(alias.name == decorator_name for alias in node.names)
-                for node in tree.body
-        ):
-            import_node = ast.ImportFrom(
-                module=module_name,
-                names=[ast.alias(name=decorator_name, asname=None)],
-                level=0
-            )
-            tree.body.insert(0, import_node)
+            # Check if the decorator already exists
+            for existing_decorator in original_node.decorators:
+                if isinstance(existing_decorator.decorator, cst.Call) and \
+                isinstance(existing_decorator.decorator.func, cst.Attribute) and \
+                existing_decorator.decorator.func.attr.value == self.decorator_name:
+                    # The decorator already exists, so we don't add it
+                    return updated_node
 
+            # The decorator doesn't exist, so we add it
+            return updated_node.with_changes(decorators=[decorator] + list(updated_node.decorators))
+        return updated_node
+
+class DecoratorRemover(cst.CSTTransformer):
+    def __init__(self, decorator_name: str):
+        self.decorator_name = decorator_name
+
+    def leave_Decorator(self, original_node: cst.Decorator, updated_node: cst.Decorator) -> Union[cst.Decorator, cst.RemovalSentinel]:
+        if isinstance(original_node.decorator, cst.Call) and \
+           isinstance(original_node.decorator.func, cst.Attribute) and \
+           original_node.decorator.func.attr.value == self.decorator_name and \
+           isinstance(original_node.decorator.func.value, cst.Attribute) and \
+           original_node.decorator.func.value.attr.value == 'mark' and \
+           isinstance(original_node.decorator.func.value.value, cst.Name) and \
+           original_node.decorator.func.value.value.value == 'pytest':
+            return cst.RemovalSentinel.REMOVE
+        return updated_node
 
 def update_tests(file: str,
-                 mapped_tests: list[tuple[str, int]],
-                 all_tests: list[str],
+                 mapped_tests: List[Tuple[str, int]],
+                 all_tests: List[str],
                  decorator_name: str,
                  remove=False):
     with open(file, 'r') as f:
         source_code = f.read()
 
-    tree = ast.parse(source_code)
+    tree = cst.parse_module(source_code)
     transform = DecoratorUpdater(mapped_tests, all_tests, decorator_name)
     if remove:
-        transform.remove_decorators(tree)
+        transform = DecoratorRemover(decorator_name)
+        tree = tree.visit(transform)
     else:
-        tree = transform.visit(tree)
-        transform.insert_pytest_mark_import(tree, *pytest_mark)
-    updated_source_code = ast.unparse(tree)
-
-    pep8_source_code = autopep8.fix_code(updated_source_code)
+        transform = DecoratorUpdater(mapped_tests, all_tests, decorator_name)
+        tree = tree.visit(transform)
+    updated_source_code = tree.code
 
     with open(file, "w") as file:
-        file.write(pep8_source_code)
+        file.write(updated_source_code)

analyzer/helper.py

@@ -63,7 +63,7 @@ def add_and_enrich_tests(meta: list[TestItem], test_files: set,
     tcm_test_data = parse_test_list(testomatio_tests)
     for test in meta:
         for tcm_test in tcm_test_data:
-            if test.user_title == tcm_test.title and test.file_name == tcm_test.file_name:
+            if test.title == tcm_test.title and test.file_name == tcm_test.file_name:
                 test.id = tcm_test.id
                 tcm_test_data.remove(tcm_test)
                 break