Use 'mypy' to verify typing annotations of Python functions. (#51)

.github/scripts/run_pylint_check.py

@@ -3,13 +3,15 @@
 import argparse
 import glob
 import io
+import logging
 import os
 import subprocess
 import sys
 
 from github import Github
 from pylint import lint as pl_run
 from pylint.__pkginfo__ import __version__ as pl_version
+from mypy.version import __version__ as mp_version
 from cython_lint import cython_lint as cl_run
 from cython_lint import __version__ as cl_version
 from cpplint import __VERSION__ as cp_version
@@ -22,20 +24,27 @@ def main():
     parser.add_argument("--token", help="The GitHub API token to use")
     parser.add_argument("--repo", help="The owner and repository we are operating on")
     args = parser.parse_args()
+
+    # Set up logging
+    logging.basicConfig(format="%(asctime)s - %(message)s", level=logging.INFO)
+
     # Connect to GitHub REST API
     gh = Github(args.token)
+
     # Run the linters
     pylint(gh, args.repo)
+    mypy(gh, args.repo)
     cython_lint(gh, args.repo)
     cpplint(gh, args.repo)
 
 
 def _check_issues(gh, repo, tool):
     open_issues = gh.search_issues(f"repo:{repo} label:automated/{tool} is:issue is:open")
     if open_issues.totalCount != 0:
-        print(f"Skipping '{tool}' run due to existing issue {open_issues[0].html_url}.")
+        logging.info(f"Found existing 'automated/{tool}' issue: '{open_issues[0].html_url}'.  Skipping '{tool}' run.")
         return True
     else:
+        logging.info(f"Found no existing 'automated/{tool}' issues.")
         return False
 
 
@@ -47,8 +56,8 @@ def _file_issue(gh, repo, tool, tool_args, tool_version, output):
                   f"comment), this is indicative of a new check in this new version of `{tool}`.\n\n"
                   f"Please investigate these issues, and either fix the source or disable the check with a "
                   f"comment.  Further checks by this automation will be held until this issue is closed.  Make "
-                  f"sure that the fix updates the `{tool}` requirement in `pyproject.toml` to the version "
-                  f"identified here ({tool_version}).\n\n"
+                  f"sure that the fix updates the `{tool}` requirement in `pyproject.toml` (the `lint` key of the "
+                  f"`project.optional-dependencies` section) to the version identified here ({tool_version}).\n\n"
                   f"For reference, here is the output of this version of `{tool}`:\n\n"
                   f"```\n"
                   f"$ {tool} {tool_args}\n"
@@ -58,7 +67,7 @@ def _file_issue(gh, repo, tool, tool_args, tool_version, output):
     repo = gh.get_repo(repo)
     repo_label = repo.get_label(f"automated/{tool}")
     new_issue = repo.create_issue(title=issue_title, body=issue_body, labels=[repo_label])
-    print(f"Opened issue {new_issue.html_url}")
+    logging.info(f"Opened issue '{new_issue.html_url}'.")
 
 
 class _StdoutCapture:
@@ -85,22 +94,43 @@ def pylint(gh, repo):
 
     # Now run pylint
     with _StdoutCapture() as capture:
+        logging.info("Running 'pylint spotfire'.")
         result = pl_run.Run(["spotfire"], exit=False)
+        logging.info(f"Return code '{result.linter.msg_status}'.")
     if result.linter.msg_status == 0:
         return
 
     # File an issue
     _file_issue(gh, repo, "pylint", "spotfire", pl_version, capture.output())
 
 
+def mypy(gh, repo):
+    # Determine if we should run pylint
+    if _check_issues(gh, repo, "mypy"):
+        return
+
+    # Now run mypy
+    command = [sys.executable, "-m", "mypy", "spotfire"]
+    logging.info("Running 'mypy spotfire'.")
+    result = subprocess.run(command, capture_output=True, check=False)
+    logging.info(f"Return code '{result.returncode}'.")
+    if result.returncode == 0:
+        return
+
+    # File an issue
+    _file_issue(gh, repo, "mypy", "spotfire", mp_version, result.stdout.decode("utf-8"))
+
+
 def cython_lint(gh, repo):
     # Determine if we should run cython-lint
     if _check_issues(gh, repo, "cython-lint"):
         return
 
     # Now run cython-lint
     with _StdoutCapture() as capture:
+        logging.info("Running 'cython-lint spotfire vendor'.")
         result = cl_run.main(["spotfire", "vendor"])
+        logging.info(f"Return code '{result}'.")
     if result == 0:
         return
 
@@ -116,8 +146,10 @@ def cpplint(gh, repo):
     # Now run cpplint
     command = [sys.executable, "-m", "cpplint"]
     command.extend(glob.glob("spotfire/*_helpers.[ch]"))
+    logging.info("Running 'cpplint spotfire/*_helpers.[ch]'.")
     result = subprocess.run(command, capture_output=True, check=False)
-    if not result.returncode:
+    logging.info(f"Return code '{result.returncode}'.")
+    if result.returncode == 0:
         return
 
     # File an issue

.github/workflows/build.yaml

@@ -29,16 +29,6 @@ jobs:
             spotfire/test/files/**
             spotfire/requirements.txt
             test_requirements_*.txt
-      - uses: actions/upload-artifact@v3
-        with:
-          name: analysis-files
-          path: |
-            pyproject.toml
-            CPPLINT.cfg
-            **/*.pyx
-            **/*.pxd
-            **/*.pxi
-            spotfire/*_helpers.*
       - name: Dynamic Elements
         id: dynamic
         run: |
@@ -153,20 +143,23 @@ jobs:
           python-version: ${{ steps.version.outputs.python-version }}
       - uses: actions/download-artifact@v3
         with:
-          name: wheel-${{ steps.version.outputs.python-version }}-ubuntu-latest
+          name: sdist
           path: dist
       - uses: actions/download-artifact@v3
         with:
-          name: analysis-files
-          path: analysis
+          name: wheel-${{ steps.version.outputs.python-version }}-ubuntu-latest
+          path: dist
       - name: Install Tools
         run: |
           pip install `ls dist/*.whl`[lint]
       - name: Run Analysis Tools
         run: |
-          cd analysis
-          mkdir .save && mv spotfire vendor .save
+          tar zxf dist/spotfire-*.tar.gz
+          # Analyses that work on the installed package
+          mv spotfire-*/pyproject.toml .
           pylint spotfire
-          mv .save/* . && rmdir .save
+          # Analyses that work on the sources of the package
+          mv spotfire-*/{spotfire,vendor} .
+          mypy spotfire
           cython-lint spotfire vendor
           find spotfire -name '*_helpers.[ch]' | xargs cpplint --repository=.

.github/workflows/pylint.yaml

@@ -14,7 +14,10 @@ jobs:
           python-version: '3.x'
       - name: Install Tools
         run: |
-          pip install PyGithub pylint cython-lint cpplint
+          pip install PyGithub pylint mypy cython-lint cpplint pip-tools
+          pip-compile -o types.txt --extra=dev,types setup.py
+          pip install -r types.txt
+          rm types.txt
       - name: Run Analysis Tools
         run: |
           python .github/scripts/run_pylint_check.py --token ${{ secrets.GITHUB_TOKEN }} --repo ${{ github.repository }}

MANIFEST.in

@@ -1,9 +1,9 @@
 include LICENSE
 include *requirements_*.txt
-include spotfire/requirements.txt
 include spotfire/*.pyx
 include spotfire/*_helpers.*
 recursive-exclude spotfire/test/files *
+include CPPLINT.cfg
 exclude setup.cfg
 recursive-include vendor *.pxd *.h *.c
 recursive-exclude vendor/sbdf-c/tests *.h *.c

pyproject.toml

@@ -60,8 +60,14 @@ dev = [
     "html-testRunner",
 ]
 # Static analysis requirements
+types = [
+    "pandas-stubs",
+    "types-Pillow",
+    "types-seaborn",
+]
 lint = [
     "pylint == 3.0.3",
+    "mypy == 1.8.0", "spotfire[types]",
     "cython-lint == 0.16.0",
     "cpplint == 1.6.1",
 ]
@@ -267,3 +273,14 @@ redefining-builtins-modules = ["six.moves", "future.builtins"]
 
 [tool.cython-lint]
 max-line-length = 120
+
+[tool.mypy]
+check_untyped_defs = true
+plugins = ["numpy.typing.mypy_plugin"]
+
+[[tool.mypy.overrides]]
+module = [
+    "geopandas",
+    "HtmlTestRunner",
+]
+ignore_missing_imports = true

spotfire/_utils.py

@@ -5,14 +5,17 @@
 """Utilities used by multiple submodules."""
 import os
 import tempfile
+import typing
 
 
-def type_name(type_: type) -> str:
+def type_name(type_: typing.Optional[type]) -> str:
     """Convert a type object to a string in a consistent manner.
 
     :param type_: the type object to convert
     :return: a string with the type name
     """
+    if not type_:
+        return "None"
     type_qualname = type_.__qualname__
     type_module = type_.__module__
     if type_module not in ("__main__", "builtins"):
@@ -22,10 +25,12 @@ def type_name(type_: type) -> str:
 
 class TempFiles:
     """Utility class that manages the lifecycle of multiple temporary files."""
+    _files: typing.List[typing.IO]
+
     def __init__(self) -> None:
         self._files = []
 
-    def new_file(self, **kwargs) -> tempfile.NamedTemporaryFile:
+    def new_file(self, **kwargs) -> typing.IO:
         """Create a temporary file object that will be tracked by this manager.
 
         :param kwargs: NamedTemporaryFile arguments

spotfire/cabfile.pyi

@@ -0,0 +1,15 @@
+# Copyright © 2024. Cloud Software Group, Inc.
+# This file is subject to the license terms contained
+# in the license file that is distributed with this file.
+
+import typing
+
+
+class CabFile:
+    def __init__(self, file: typing.Any): ...
+    def __enter__(self) -> CabFile: ...
+    def __exit__(self, exc_type, exc_val, exc_tb): ...
+    def __repr__(self) -> str: ...
+    def write(self, filename: str, arcname: typing.Optional[str] = None) -> None: ...
+    def writestr(self, arcname: str, data: bytes) -> None: ...
+    def close(self) -> None: ...

spotfire/codesign.pyi

@@ -0,0 +1,17 @@
+# Copyright © 2024. Cloud Software Group, Inc.
+# This file is subject to the license terms contained
+# in the license file that is distributed with this file.
+
+import enum
+import typing
+
+
+class CertificateStoreLocation(enum.Enum):
+    CURRENT_USER = 1
+    LOCAL_MACHINE = 2
+
+def codesign_file(filename: typing.Any, certificate: str, password: typing.Any, timestamp: typing.Optional[str] = None,
+                  use_rfc3161: bool = False, use_sha256: bool = False) -> None: ...
+def codesign_file_from_store(filename: typing.Any, store_location: CertificateStoreLocation, store_name: typing.Any,
+                             store_cn: typing.Any, timestamp: typing.Optional[str] = None,
+                             use_rfc3161: bool = False, use_sha256: bool = False) -> None: ...