pyproject.toml

[build-system]
build-backend = "setuptools.build_meta"
requires = ["setuptools>=66", "setuptools_scm[toml]>=3.5.0", "wheel"]

[project]
name = "catalystcoop.pudl"
description = "An open data processing pipeline for US energy data"
readme = { file = "README.rst", content-type = "text/x-rst" }
authors = [{ name = "Catalyst Cooperative", email = "pudl@catalyst.coop" }]
requires-python = ">=3.12,<3.13"
dynamic = ["version"]
license = { file = "LICENSE.txt" }
dependencies = [
    "addfips>=0.4",
    "alembic>=1.13",
    "boto3>=1.35",
    "bottleneck>=1.4.0", # pandas[performance]; 1.3.7 required for Python 3.12
    "build>=1.2",
    "catalystcoop.dbfread>=3.0,<3.1",
    "catalystcoop.ferc-xbrl-extractor>=1.5.1,<2",
    "click>=8",
    "coloredlogs>=14.0", # Dagster requires 14.0
    "conda-lock>=2.5.7",
    "coverage>=7.6",
    "dagster>=1.9.9",
    "dagster-postgres>=0.25.9,<1", # Update when dagster-postgres graduates to 1.x
    "dask>=2025",
    "dask-expr>=2", # Required for dask[dataframe]
    "datasette>=0.65",
    "deltalake>=0.24,<1",
    "doc8>=1.1",
    "duckdb>=1.1.3",
    "email-validator>=1.0.3", # pydantic[email]
    "frictionless>=5,<6",
    "fsspec>=2024",
    "furo>=2024",
    "gcsfs>=2024",
    "gdal==3.10.1",  # pinned, because we need it to work with pudl-archiver
    "geopandas>=1.0",
    "grpcio==1.67.1", # Required by dagster, binary dependencies are flaky
    "grpcio-health-checking==1.67.1", # Required by dagster, binary dependencies are flaky
    "grpcio-status==1.67.1", # Required by dagster, binary dependencies are flaky
    "hypothesis>=6.123",
    "jellyfish>=1",
    "jinja2>=3.1",
    "jupyter",
    "matplotlib>=3.10",
    "mlflow>=2.19",
    "nbconvert>=7",
    "nbformat>=5.10",
    "networkx>=3.4",
    "numba>=0.60", # pandas[performance]; 0.60 required for Numpy 2.0
    "numexpr>=2.10", # pandas[performance]
    "numpy>=2",
    "packaging>=24",
    "pandas>=2.2.3",
    "pandera>=0.22",
    "pre-commit>=3",
    "pyarrow>=18.1", # pandas[parquet]
    "pydantic>=2.10",
    "pydantic-settings>=2.7",
    "pytest>=8",
    "pytest-cov>=5",
    "pytest-console-scripts>=1.4",
    "pytest-mock>=3.14",
    "pytest-xdist>=3.6",
    "python-calamine>=0.3", # pandas[excel]
    "python-dotenv>=1",
    "pytz>=2024",
    "pyyaml>=6",
    "readthedocs-sphinx-ext>=2",
    "requests>=2.32",
    "responses>=0.25",
    "ruff>=0.9",
    "scikit-learn>=1.6",
    "scipy>=1.15",
    "Shapely>=2",
    "splink>=4",
    "sphinx>=8",
    "sphinx-autoapi>=3",
    "sphinx-issues>=5",
    "sphinx-reredirects>=0.1.2",
    "sphinxcontrib_bibtex>=2.6",
    "sphinxcontrib_googleanalytics>=0.4",
    "sqlalchemy>=2",
    "sqlglot>=25",
    "timezonefinder>=6.2",
    "universal_pathlib>=0.2",
    "urllib3>=1.26.18",
    "xlsxwriter>=3.2", # pandas[excel]
]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Natural Language :: English",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python",
    "Topic :: Scientific/Engineering",
]
keywords = [
    "analysis",
    "climate change",
    "coal",
    "data",
    "economics",
    "eia 860",
    "eia 861",
    "eia 923",
    "electricity",
    "emissions",
    "energy",
    "epa ampd",
    "epa cems",
    "ferc form 1",
    "ferc form 2",
    "ferc form 6",
    "ferc form 60",
    "ferc form 714",
    "finance",
    "fossil fuel",
    "natural gas",
    "oil",
    "policy",
    "utility",
]

[project.scripts]
ferc_to_sqlite = "pudl.ferc_to_sqlite.cli:main"
metadata_to_rst = "pudl.convert.metadata_to_rst:metadata_to_rst"
pudl_check_fks = "pudl.etl.check_foreign_keys:pudl_check_fks"
pudl_datastore = "pudl.workspace.datastore:pudl_datastore"
pudl_etl = "pudl.etl.cli:pudl_etl"
pudl_service_territories = "pudl.analysis.service_territory:pudl_service_territories"

[project.urls]
"Homepage" = "https://catalyst.coop/pudl/"
"Source" = "https://github.com/catalyst-cooperative/pudl"
"Documentation" = "https://catalystcoop-pudl.readthedocs.io"
"Release Notes" = "https://catalystcoop-pudl.readthedocs.io/en/nightly/release_notes.html"
"Issue Tracker" = "https://github.com/catalyst-cooperative/pudl/issues"
"Support" = "https://github.com/catalyst-cooperative/pudl/discussions"
"Funding" = "https://catalyst.coop/support-the-pudl-project/"

[project.optional-dependencies]
dev = [
    "dagster-webserver>=1.9.9",
    "jupyterlab>4.3",
    "jupyter-lsp",
    "jupyterlab-lsp",
    "ruff-lsp",
    "jupyter-resource-usage",
    "pygraphviz",
    "terraform>=1.9.2"
]

[tool.setuptools]
include-package-data = true

[tool.setuptools.packages.find]
where = ["src"]

[tool.setuptools_scm]
tag_regex = "^v(?P<version>20\\d{2}\\.\\d{1,2}\\.\\d{1,2})$"
git_describe_command = [
    "git",
    "describe",
    "--dirty",
    "--tags",
    "--long",
    "--match",
    "v20*",
]

[tool.ruff]
# Configurations that apply to both the `format` and `lint` subcommands.
target-version = "py312"
line-length = 88
indent-width = 4
# Don't touch autogenerated alembic migrations and ignore notebooks for now
exclude = ["migrations/versions", "*.ipynb"]

[tool.ruff.format]
# Configuration specfic to the `format` subcommand.
# We use black compatible formatting.
quote-style = "double"
indent-style = "space"
skip-magic-trailing-comma = false
line-ending = "auto"

[tool.ruff.lint]
# Configurations that apply to the `lint` subcommand.
preview = true # Required for NPY201
explicit-preview-rules = true # and ONLY NPY201
select = [
    "A", # flake8-builtins
    "B", # flake8-bugbear
    "C", # Limit cyclomatic complexity using mccabe
    "C4", # flake8-comprehensions
    "D", # pydocstyle errors
    "E", # pycodestyle errors
    "EXE", # executable file issues
    "F", # pyflakes
    "I", # isort
    "ICN", # flake8 import conventions
    "N", # pep8-naming
    "NPY", # NumPy specific checks
    "PD", # pandas checks
    "PGH", # pygrep-hooks
    "PIE", # flake8-pie miscellaneous linting
    "PTH", # use pathlib
    "Q", # flake8-quotes
    "RET", # check return values
    "RSE", # unnecessary parenthises on raised exceptions
    "S", # flake8-bandit
    "SIM", # flake8-simplify
    "UP", # pyupgrade (use modern python syntax)
    "W", # pycodestyle warnings
    # These rules would be good, but will require more effort:
    # "ANN", # identify missing type annotations
    # "ARG", # find unused arguments
    # "DTZ", # datetime / timzeone issues flake8-datetimez
    # "ERA", # eradicate: finds commented out code
    # "FURB", # refurb: general code quality/modernization. Still in preview.
    # "PERF", # performance issues
    # "PT", # pytest style
    # "PL", # PyLint (lots of rules)
    # "RUF", # Ruff specific rules
    # "T", # Find print statements that should probably be logging
]
ignore = [
    "B006", # Don't use mutable data structures for argument defaults. TODO: FIX
    "B008", # Don't perform function calls in argument defaults. TODO: FIX
    "D401", # Require imperative mood in docstrings.
    "D417",
    "E501", # Overlong lines.
    "E203", # Space before ':' (black recommends to ignore)
    "E226", # Missing whitespace around arithmetic operator
    "E266", # Too many leading `#` before block comment
    "PD003", # Use of isna rather than isnull
    "PD004", # Use of notna rather than notnull
    "PD010", # Use of df.stack()
    "PD013", # Use of df.unstack()
    "PD015", # Use of pd.merge() rather than df.merge()
    "PD901", # df as variable name
    "RET504", # Ignore unnecessary assignment before return
    "RUF005", # Unpack iterables instead of concatenation
    "S101", # Use of assert
]
extend-select = ["NPY201"] # preview rule to easy migration to Numpy 2.0

[tool.ruff.lint.per-file-ignores]
"__init__.py" = ["F401"] # Ignore unused imports
"test/*" = ["D"]
"migrations/*" = ["D", "Q"]

[tool.ruff.lint.isort]
known-first-party = ["pudl"]

[tool.ruff.lint.pydocstyle]
convention = "google"

[tool.ruff.lint.mccabe]
max-complexity = 10

[tool.ruff.lint.flake8-quotes]
docstring-quotes = "double"
inline-quotes = "double"
multiline-quotes = "double"

[tool.doc8]
max-line-length = 88
ignore-path = ["docs/ferc/form1", "docs/_build", "docs/data_sources/phmsagas"]

[tool.pytest.ini_options]
testpaths = "./"
filterwarnings = [
    "ignore:distutils Version classes are deprecated:DeprecationWarning",
    "ignore:Deprecated call to `pkg_resources.declare_namespace:DeprecationWarning",
    "ignore:Deprecated call to.*declare_namespace:DeprecationWarning:pkg_resources",
    "ignore:Deprecated call to:DeprecationWarning:google.rpc",
    "ignore:Feature names only support names:FutureWarning:sklearn.utils.validation",
    "ignore:In a future version, the Index constructor:FutureWarning:pandas.core.reshape.merge",
    "ignore:Integration of FERC 714:UserWarning:pudl.extract.ferc714",
    "ignore:Integration of EIA 861:UserWarning:pudl.extract.eia861",
    "ignore:pkg_resources is deprecated as an API.:DeprecationWarning",
    "ignore:Subclassing validator classes is not intended to be part of their public API.:DeprecationWarning",
    "ignore:Subclassing validator classes:DeprecationWarning:tableschema",
    "ignore:The Shapely GEOS version:UserWarning:geopandas[.*]",
    "ignore:The `__fields__` attribute is deprecated:pydantic.PydanticDeprecatedSince20:unittest.mock",
    "ignore:The `__fields_set__` attribute is deprecated:pydantic.PydanticDeprecatedSince20:unittest.mock",
    "ignore:The `__fields__` attribute is deprecated:pydantic.PydanticDeprecatedSince20:pydantic.main",
    "ignore:The `__fields_set__` attribute is deprecated:pydantic.PydanticDeprecatedSince20:pydantic.main",
    "ignore:The `update_forward_refs` method is deprecated:pydantic.PydanticDeprecatedSince20:pydantic.main",
    "ignore:Support for class-based `config` is deprecated:pydantic.PydanticDeprecatedSince20:pydantic._internal._config",
    "ignore:Pydantic V1 style `@validator` validators are deprecated:pydantic.PydanticDeprecatedSince20:ferc_xbrl_extractor.instance",
    "ignore:Pydantic V1 style `@validator` validators are deprecated:pydantic.PydanticDeprecatedSince20:mlflow.gateway.config",
    "ignore:The `update_forward_refs` method is deprecated:pydantic.PydanticDeprecatedSince20:ferc_xbrl_extractor.taxonomy",
    "once:Could not infer format, so each element will be parsed individually, falling back to `dateutil`.:UserWarning:pudl.extract.eia_bulk_elec",
    "once:In a future version:FutureWarning:pudl.helpers",
    "once:open_binary is deprecated:DeprecationWarning:pudl.glue.ferc1_eia",
    "once:open_text is deprecated:DeprecationWarning:pudl.glue.ferc1_eia",
    "once:The behavior of DataFrame concatenation with empty or all-NA entries is deprecated.:FutureWarning",
    "once:Valid config keys have changed in V2:UserWarning",
]

markers = ["slow: marks tests as slow (deselect with '-m \"not slow\"')"]
# must not end with `--cov` since that will treat the first argument you pass
# to pytest as a coverage source! see
# https://github.com/catalyst-cooperative/pudl/issues/3609
addopts = """\
    --verbose \
    --cov \
    --cov-append \
    --cov-config=pyproject.toml \
    --cov-report= \
    """
log_format = "%(asctime)s [%(levelname)8s] %(name)s:%(lineno)s %(message)s"
log_date_format = "%Y-%m-%d %H:%M:%S"
log_cli = "true"
log_cli_level = "DEBUG"
doctest_optionflags = ["NORMALIZE_WHITESPACE", "IGNORE_EXCEPTION_DETAIL", "ELLIPSIS"]

[tool.conda-lock]
channels = ["conda-forge"]
platforms = [
    "linux-64",
    "osx-64",
    "osx-arm64",
    # The google-cloud-sdk conda package doesn't support Windows
    # "win-64",
]
channel-priority = "strict"
name = "pudl-dev"

[tool.conda-lock.dependencies]
curl = ">=8.4.0"
google-cloud-sdk = ">=500"
nodejs = ">=20"
pandoc = ">=2"
pip = ">=24"
prettier = ">=3.0"
python = ">=3.12,<3.13"
sqlite = ">=3.47"
zip = ">=3.0"

[tool.coverage.run]
source = ["src/pudl/", "test/integration/", "test/unit/"]
omit = [
    # Never hit by integration tests:
    "src/pudl/validate.py",
]

[tool.coverage.report]
precision = 2
sort = "miss"
skip_empty = true
fail_under = 93
exclude_lines = [
    # Have to re-enable the standard pragma
    "pragma: no cover",

    # Don't complain if tests don't hit defensive assertion code:
    "raise AssertionError",
    "raise NotImplementedError",

    # Don't complain if non-runnable code isn't run:
    "if 0:",
    "if __name__ == .__main__.:",

    # Stuff that's not expected to run normally...
    "logger.debug",
]

[tool.dagster]
modules = [{ type = "module", name = "pudl.etl" }, { type = "module", name = "pudl.ferc_to_sqlite" }]