Updating the testimony to allow dynamic analysis

testimony/__init__.py

@@ -94,37 +94,24 @@ class TestFunction(object):
     """
 
     def __init__(self, function_def, parent_class=None, testmodule=None):
-        """Wrap a ``ast.FunctionDef`` instance used to extract information."""
         self.docstring = ast.get_docstring(function_def)
         self.function_def = function_def
         self.name = function_def.name
-        if parent_class:
-            self.parent_class = parent_class.name
-            self.parent_class_def = parent_class
-            self.class_docstring = ast.get_docstring(self.parent_class_def)
-        else:
-            self.parent_class = None
-            self.parent_class_def = None
-            self.class_docstring = None
-        self.testmodule = testmodule.path
+        self.parent_class = parent_class.name if parent_class else None
+        self.parent_class_def = parent_class
+        self.class_docstring = ast.get_docstring(parent_class) if parent_class else None
+        self.testmodule = testmodule.path if testmodule else None
         self.module_def = testmodule
-        self.module_docstring = ast.get_docstring(self.module_def)
+        self.module_docstring = ast.get_docstring(self.module_def) if self.module_def else None
         self.pkginit = os.path.join(
-            os.path.dirname(self.testmodule), '__init__.py')
-        if os.path.exists(self.pkginit):
-            self.pkginit_def = ast.parse(''.join(open(self.pkginit)))
-            self.pkginit_docstring = ast.get_docstring(self.pkginit_def)
-        else:
-            self.pkginit_def = None
-            self.pkginit_docstring = None
-        self.tokens = {}
-        self.invalid_tokens = {}
+            os.path.dirname(self.testmodule), '__init__.py') if self.testmodule else None
+        self.tokens, self.invalid_tokens = {}, {}
         self._rst_parser_messages = []
+
         tokens = SETTINGS.get('tokens').keys() or None
-        minimum_tokens = [key for key, value
-                          in SETTINGS.get('tokens').items()
-                          if value.required] or None
+        minimum_tokens = [key for key, value in SETTINGS.get('tokens').items() if value.required] or None
         self.parser = DocstringParser(tokens, minimum_tokens)
+
         self._parse_docstring()
         self._parse_decorators()
 
@@ -163,26 +150,15 @@ def _parse_docstring(self):
             self.tokens['test'] = docstring.strip().split('\n')[0]
 
     def _parse_decorators(self):
-        """Get decorators from class and function definition.
-
-        Modules and packages can't be decorated, so they are skipped.
-        Decorator can be pytest marker or function call.
-        ``tokens`` attribute will be updated with new value ``decorators``.
-        """
+        """Extract decorators from class and function definitions."""
         token_decorators = []
         for level in (self.parent_class_def, self.function_def):
             decorators = getattr(level, 'decorator_list', None)
-            if not decorators:
-                continue
-
-            for decorator in decorators:
-                try:
+            if decorators:
+                for decorator in decorators:
                     token_decorators.append(
                         getattr(decorator, 'func', decorator).id
                     )
-                except AttributeError:
-                    continue
-
         if token_decorators:
             self.tokens['decorators'] = token_decorators
 
@@ -270,14 +246,12 @@ def __str__(self):
 
 
 def main(report, paths, json_output, markdown_output, nocolor):
-    """Entry point for the testimony project.
-
-    Expects a valid report type and valid directory paths, hopefully argparse
-    is taking care of validation
-    """
-    SETTINGS['json'] = json_output
-    SETTINGS['markdown'] = markdown_output
-    SETTINGS['nocolor'] = nocolor
+    """Entry point for testimony report generation."""
+    SETTINGS.update({
+        'json': json_output,
+        'markdown': markdown_output,
+        'nocolor': nocolor
+    })
 
     if report == SUMMARY_REPORT:
         report_function = summary_report

testimony/cli.py

@@ -4,6 +4,7 @@
 
 from testimony import SETTINGS, config, constants, main
 
+from testimony.parser import DocstringParser  # Import the parser with dynamic loading
 
 @click.command()
 @click.option('-j', '--json', help='JSON output', is_flag=True)
@@ -21,12 +22,51 @@
 def testimony(
         json, markdown, nocolor, tokens, minimum_tokens, config_file,
         report, path):
-    """Inspect and report on the Python test cases."""
+    # load config if possible
     if config_file:
         SETTINGS['tokens'] = config.parse_config(config_file)
     if tokens:
         config.update_tokens_dict(SETTINGS['tokens'], tokens)
     if minimum_tokens:
-        config.update_tokens_dict(
-            SETTINGS['tokens'], minimum_tokens, {'required': True})
-    main(report, path, json, markdown, nocolor)
+        config.update_tokens_dict(SETTINGS['tokens'], minimum_tokens, {'required': True})
+
+    # initialize the parser
+    parser = DocstringParser(tokens=SETTINGS['tokens'], minimum_tokens=SETTINGS['tokens'].get('required', []))
+
+    # loop through each provided path and parse
+    results = []
+    for module_path in path:
+        try:
+            # load and parse module dynamically
+            valid_tokens, invalid_tokens, parse_messages = parser.load_and_parse(module_path)
+            results.append({
+                'module': module_path,
+                'valid_tokens': valid_tokens,
+                'invalid_tokens': invalid_tokens,
+                'parse_messages': parse_messages
+            })
+        except Exception as e:
+            print(f"Error processing module {module_path}: {e}")
+
+    # Generate report
+    if json:
+        import json as json_lib
+        print(json_lib.dumps(results, indent=2))
+    elif markdown:
+        for result in results:
+            print(f"## Report for {result['module']}")
+            print("### Valid Tokens")
+            for token, value in result['valid_tokens'].items():
+                print(f"- **{token}**: {value}")
+            print("### Invalid Tokens")
+            for token, value in result['invalid_tokens'].items():
+                print(f"- **{token}**: {value}")
+            print("### Parse Messages")
+            for message in result['parse_messages']:
+                print(f"- {message}")
+    else:
+        for result in results:
+            print(f"Report for {result['module']}")
+            print("Valid Tokens:", result['valid_tokens'])
+            print("Invalid Tokens:", result['invalid_tokens'])
+            print("Parse Messages:", result['parse_messages'])

testimony/config.py

@@ -52,7 +52,7 @@ class TokenConfig(object):
     """
     Represent config for one token.
 
-    Currently only checks for value.
+    Includes dynamic decorator handling.
     """
 
     def __init__(self, name, config):
@@ -70,16 +70,23 @@ def __init__(self, name, config):
         self.required = config.get('required', False)
         self.token_type = None
 
+        # set token type if available in TOKENS_TYPES
         if config.get('type') in TOKEN_TYPES:
             self.token_type = config['type']
 
+        # additional handling for choice, string, and decorator types
         if self.token_type == 'choice':
             assert 'choices' in config
             assert isinstance(config['choices'], list)
             self.casesensitive = config.get('casesensitive', True)
             self.choices = [i if self.casesensitive else i.lower()
                             for i in config['choices']]
 
+        elif self.token_type == 'decorator':
+            # set specific defaults or validation parameters if needed
+            self.decorator_name = config.get('decorator_name')
+            self.default_value = config.get('default_value', None)
+
         elif self.token_type == 'string':
             pass
 
@@ -96,4 +103,7 @@ def validate(self, what):
             return what in self.choices
         elif self.token_type == 'string':
             return isinstance(what, str)  # validate it's a string
+        elif self.token_type == 'decorator':
+            # Additional decorator-related validation if needed
+            return what == self.default_value or what is not None
         return True  # assume valid for unknown types

testimony/constants.py

@@ -18,7 +18,8 @@
 
 TOKEN_TYPES = [
     'choice',
-    'string'
+    'string',
+    'decorator' # new token type for dynamic decorator values
 ]
 
 DEFAULT_TOKENS = (

testimony/parser.py

@@ -4,6 +4,9 @@
 from io import StringIO
 from xml.etree import ElementTree
 
+import importlib
+import traceback
+
 from docutils.core import publish_string
 from docutils.parsers.rst import nodes, roles
 from docutils.readers import standalone
@@ -54,6 +57,32 @@ def __init__(self, tokens=None, minimum_tokens=None):
             roles.register_generic_role(role, nodes.raw)
             roles.register_generic_role('py:' + role, nodes.raw)
 
+    def load_and_parse(self, module_name):
+        """Dynamically import a module, run decorators, and parse docstrings"""
+        try:
+            # dynamically import the module by name
+            module = importlib.import_module(module_name)
+        except ImportError as e:
+            print(f"Error importing module {module_name}: {e}")
+            return {}, {}, [f"ImportError: {str(e)}"]
+
+        valid_tokens, invalid_tokens, parse_message = {}, {}, []
+
+        # go through each member in the module to extract docstrings
+        for attr_name in dir(module):
+            attr = getattr(module, attr_name)
+            if callable(attr) and attr.__doc__:
+                try:
+                    # parse the docstrings after decorators have been applied
+                    v_tokens, iv_tokens, message = self.parse(attr.__doc__)
+                    valid_tokens.update(v_tokens)
+                    invalid_tokens.update(iv_tokens)
+                    parse_message.extend(message)
+                except Exception as e:
+                    parse_message.append(f"Parsing error for {attr_name}: {e}")
+                    traceback.print_exc()
+        return valid_tokens, invalid_tokens, parse_message
+
     def parse(self, docstring=None):
         """Parse docstring and report parsing issues, valid and invalid tokens.