From a377e5685f05666377615327720cc46f196b1759 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 11:39:34 -0800 Subject: [PATCH 01/20] Move environment files to the 'envs' directory Relocated 'environment.yaml' and 'requirements.txt' to a dedicated 'envs/' directory for better organization. This change improves project structure and simplifies environment file management. --- environment.yml => envs/environment.yml | 10 +++++----- requirements.txt => envs/requirements.txt | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) rename environment.yml => envs/environment.yml (67%) rename requirements.txt => envs/requirements.txt (95%) diff --git a/environment.yml b/envs/environment.yml similarity index 67% rename from environment.yml rename to envs/environment.yml index 4d99a54..5891870 100644 --- a/environment.yml +++ b/envs/environment.yml @@ -1,17 +1,17 @@ -# conda environment description file for SoG-bloomcast package development environment +# conda environment description file for SOG-Bloomcast-Ensemble package development environment # # Create a conda environment in which the `bloomcast` command can be run # with: # -# $ conda env create -f SOG-Bloomcast-Ensemble/environment.yml -# $ conda activate bloomcast +# $ mamba env create -f SOG-Bloomcast-Ensemble/envs/environment.yml +# $ mamba activate bloomcast # (bloomcast)$ python -m pip install --editable SOG # (bloomcast)$ python -m pip install --editable SOG-Bloomcast-Ensemble # # The environment will also include all the tools used to develop, -# test, and document the SoG-bloomcast package. +# test, and document the SOG-Bloomcast-Ensemble package. # -# See the requirements.txt file for an exhaustive list of all the +# See the envs/requirements.txt file for an exhaustive list of all the # packages installed in the environment and their versions used in # recent development. diff --git a/requirements.txt b/envs/requirements.txt similarity index 95% rename from requirements.txt rename to envs/requirements.txt index dd4bd06..d189653 100644 --- a/requirements.txt +++ b/envs/requirements.txt @@ -1,4 +1,4 @@ -# Python packages and their versions required for the SoG-bloomcast +# Python packages and their versions required for the SOG-Bloomcast-Ensemble # development environment # # See docs/Development.rst for instructions on how to create a conda @@ -6,7 +6,7 @@ # # Update this file by merging the output the command: # -# python -m pip list --format=freeze >> requirements.txt +# python -m pip list --format=freeze >> envs/requirements.txt alabaster==0.7.16 anyio==4.4.0 From f71c2f6816938bff6d8ee60182346ae57bf4f9b1 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 11:42:20 -0800 Subject: [PATCH 02/20] Add Dependabot config to monitor Actions versions Introduce a Dependabot configuration file to automate version checks for GitHub Actions. The setup schedules weekly checks and ensures actions are kept up to date via pull requests. This improves maintenance and security by automating dependency updates. --- .github/dependabot.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yaml diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..21b496b --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,13 @@ +# Dependabot config to enable checking for version updates on actions +# and open pull requests to apply those updates +# refs: +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + # Check for updates to GitHub Actions every week + interval: "weekly" From 17ad78c7b0afeaf7449fe21f47bcc68b9b613116 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 11:45:58 -0800 Subject: [PATCH 03/20] Add GHA workflow for auto-assigning issues/PRs This workflow triggers on issue or PR creation and reopening events. It uses a shared workflow from UBC-MOAD/gha-workflows/. --- .github/workflows/assign-issue-pr.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/assign-issue-pr.yaml diff --git a/.github/workflows/assign-issue-pr.yaml b/.github/workflows/assign-issue-pr.yaml new file mode 100644 index 0000000..2644788 --- /dev/null +++ b/.github/workflows/assign-issue-pr.yaml @@ -0,0 +1,18 @@ +name: Assign Issue/PR + +on: + issues: + types: + - reopened + - opened + pull_request: + types: + - reopened + - opened + +jobs: + auto_assign: + permissions: + issues: write + pull-requests: write + uses: UBC-MOAD/gha-workflows/.github/workflows/auto-assign.yaml@main From 4b91019c987ff321cc653233ccb1dfbc6a0e6693 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:03:31 -0800 Subject: [PATCH 04/20] Add GHA CodeQL analysis workflow This commit introduces a GitHub Actions workflow to run CodeQL analysis. It helps identify potential security vulnerabilities in Python code with scheduled weekly scans and branch-specific triggers. --- .github/workflows/codeql-analysis.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .github/workflows/codeql-analysis.yaml diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml new file mode 100644 index 0000000..a2f0984 --- /dev/null +++ b/.github/workflows/codeql-analysis.yaml @@ -0,0 +1,22 @@ +name: "CodeQL" + +on: + push: + branches: [ '*' ] + schedule: + - cron: '01 12 * * 0' + +jobs: + analyze: + name: Analyze + permissions: + actions: read + contents: read + security-events: write + strategy: + fail-fast: false + matrix: + language: [ 'python' ] + uses: UBC-MOAD/gha-workflows/.github/workflows/codeql-analysis.yaml@main + with: + language: ${{ matrix.language }} From d417e41e5ffc0c68e9b87cbe10e98b8c04ad7c07 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:15:20 -0800 Subject: [PATCH 05/20] Add test environment and GHA pytest workflow Create a new conda environment file for testing and coverage analysis, specifying required dependencies for testing the SOG-Bloomcast-Ensemble package. Add a GitHub Actions workflow to automate running pytest with coverage checks and reporting. The workflow uses a reusable workflow from UBC-MOAD/gha-workflows. --- .github/workflows/pytest-with-coverage.yaml | 25 ++++++++++++++++ envs/environment-test.yml | 32 +++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 .github/workflows/pytest-with-coverage.yaml create mode 100644 envs/environment-test.yml diff --git a/.github/workflows/pytest-with-coverage.yaml b/.github/workflows/pytest-with-coverage.yaml new file mode 100644 index 0000000..b9ca295 --- /dev/null +++ b/.github/workflows/pytest-with-coverage.yaml @@ -0,0 +1,25 @@ +name: pytest-with-coverage + +on: + push: + branches: [ '*' ] + # Enable workflow to be triggered from GitHub CLI, browser, or via API + # primarily for testing conda env solution for new Python versions + workflow_dispatch: + +jobs: + pytest-with-coverage: + permissions: + contents: read + pull-requests: write + strategy: + fail-fast: false + matrix: + python-version: [ '3.12' ] + uses: UBC-MOAD/gha-workflows/.github/workflows/pytest-with-coverage.yaml@main + with: + python-version: ${{ matrix.python-version }} + conda-env-file: envs/environment-test.yaml + conda-env-name: bloomcast-test + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/envs/environment-test.yml b/envs/environment-test.yml new file mode 100644 index 0000000..08adf6c --- /dev/null +++ b/envs/environment-test.yml @@ -0,0 +1,32 @@ +# conda environment description file for SOG-Bloomcast-Ensemble package testing environment +# +# Creates a conda environment in which the SOG-Bloomcast-Ensemble package unit tests and +# coverage analysis can be run. +# Primarily intended to create a conda env for use in a GitHub Actions workflow. + +name: bloomcast-test + +channels: + - conda-forge + - nodefaults + +dependencies: + - arrow + - beautifulsoup4 + - colander<2 + - jupyterlab + - cliff + - matplotlib + - numpy + - pip + - pyyaml + - requests + + # For unit tests and coverage monitoring + - pytest + - pytest-cov + - pytest-randomly + + - pip: + # Install the SOG-Bloomcast-Ensemble package in editable mode + - ../ From c5bd64359c6d1e119bb06859314f020f67c3c001 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:19:10 -0800 Subject: [PATCH 06/20] Correct spelling of environment-test.yaml file name --- envs/{environment-test.yml => environment-test.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename envs/{environment-test.yml => environment-test.yaml} (100%) diff --git a/envs/environment-test.yml b/envs/environment-test.yaml similarity index 100% rename from envs/environment-test.yml rename to envs/environment-test.yaml From 5a0efde47818caaaed5a80f6353ba3aa17855836 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:26:24 -0800 Subject: [PATCH 07/20] Update test env to install SOG package from GitHub Updated the test environment file to include installation of the SOG package in editable mode via a GitHub URL. This ensures the latest SOG code is pulled directly from the repository, alongside the existing SOG-Bloomcast-Ensemble package. --- envs/environment-test.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/envs/environment-test.yaml b/envs/environment-test.yaml index 08adf6c..4bc3d61 100644 --- a/envs/environment-test.yaml +++ b/envs/environment-test.yaml @@ -28,5 +28,6 @@ dependencies: - pytest-randomly - pip: - # Install the SOG-Bloomcast-Ensemble package in editable mode - - ../ + # Install theSOG and SOG-Bloomcast-Ensemble packages in editable mode + - --editable git+https://github.com/SalishSeaCast/SOG.git#egg=SOG + - --editable ../ From bba7916c8d3068b7cfe45d33e8e64d2ecb5dd9ee Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:31:22 -0800 Subject: [PATCH 08/20] Correct SOGCommand pkg identifier in test environment --- envs/environment-test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/envs/environment-test.yaml b/envs/environment-test.yaml index 4bc3d61..cc2c37f 100644 --- a/envs/environment-test.yaml +++ b/envs/environment-test.yaml @@ -29,5 +29,5 @@ dependencies: - pip: # Install theSOG and SOG-Bloomcast-Ensemble packages in editable mode - - --editable git+https://github.com/SalishSeaCast/SOG.git#egg=SOG + - --editable git+https://github.com/SalishSeaCast/SOG.git#egg=SOGcommand - --editable ../ From 32fceb6a450266c74cf8249ee6b964c42a8dde37 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:37:11 -0800 Subject: [PATCH 09/20] Rename dev env file and update dev env name Renamed `environment.yml` to `environment-dev.yaml` and updated the environment name to `bloomcast-dev` for clarity and consistency. Adjusted setup instructions to reflect these changes, aligning the naming convention with development-specific purposes. --- envs/{environment.yml => environment-dev.yaml} | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) rename envs/{environment.yml => environment-dev.yaml} (81%) diff --git a/envs/environment.yml b/envs/environment-dev.yaml similarity index 81% rename from envs/environment.yml rename to envs/environment-dev.yaml index 5891870..ccb6943 100644 --- a/envs/environment.yml +++ b/envs/environment-dev.yaml @@ -3,10 +3,9 @@ # Create a conda environment in which the `bloomcast` command can be run # with: # -# $ mamba env create -f SOG-Bloomcast-Ensemble/envs/environment.yml -# $ mamba activate bloomcast -# (bloomcast)$ python -m pip install --editable SOG -# (bloomcast)$ python -m pip install --editable SOG-Bloomcast-Ensemble +# $ mamba env create -f SOG-Bloomcast-Ensemble/envs/environment-dev.yaml +# $ mamba activate bloomcast-dev +# (bloomcast-dev)$ python -m pip install --editable SOG # # The environment will also include all the tools used to develop, # test, and document the SOG-Bloomcast-Ensemble package. @@ -15,7 +14,7 @@ # packages installed in the environment and their versions used in # recent development. -name: bloomcast +name: bloomcast-dev channels: - conda-forge From 63868130dd1083f0829dd6ccd1ccdaafc9461ecf Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:44:58 -0800 Subject: [PATCH 10/20] Add pre-commit to manage code style & repo QA Initial hooks: * Code formatting by black * Trim trailing whitespace * Ensure that files are either empty, or end with one newline * Confirm that YAML files have parsable syntax * Confirm that TOML files have parsable syntax * Prevent files larger than 500 kB from being committed --- .pre-commit-config.yaml | 21 +++++++++++++++++++++ envs/environment-dev.yaml | 5 +++++ 2 files changed, 26 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..945c167 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,21 @@ +# Git pre-commit hooks config file +# Only takes effect if you have pre-commit installed in the env, +# and after you run `pre-commit install` +# +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + # Out-of-the-box hooks from the pre-commit org + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: check-added-large-files + # Code formatting with black + - repo: https://github.com/psf/black + rev: 25.1.0 + hooks: + - id: black diff --git a/envs/environment-dev.yaml b/envs/environment-dev.yaml index ccb6943..68cf403 100644 --- a/envs/environment-dev.yaml +++ b/envs/environment-dev.yaml @@ -6,6 +6,7 @@ # $ mamba env create -f SOG-Bloomcast-Ensemble/envs/environment-dev.yaml # $ mamba activate bloomcast-dev # (bloomcast-dev)$ python -m pip install --editable SOG +# (bloomcast-dev)$ python -m pip install --editable SOG-Bloomcast-Ensemble # # The environment will also include all the tools used to develop, # test, and document the SOG-Bloomcast-Ensemble package. @@ -33,6 +34,10 @@ dependencies: - pyyaml - requests + # For coding style and repo QA + - black + - pre-commit + # For unit tests - pytest From 318505cdc42fae7100d3702ad4d5da123c16dd0f Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:46:50 -0800 Subject: [PATCH 11/20] Code style gardening by pre-commit --- .gitignore | 2 - __pkg_metadata__.py | 16 +- bloomcast/bloomcast.py | 547 +++++++++------- bloomcast/carbonate.py | 320 +++++---- bloomcast/ensemble.py | 764 +++++++++++----------- bloomcast/main.py | 9 +- bloomcast/meteo.py | 84 +-- bloomcast/rivers.py | 77 +-- bloomcast/utils.py | 277 ++++---- bloomcast/visualization.py | 309 +++++---- bloomcast/wind.py | 80 ++- cf_analysis/cf_analysis.py | 108 +-- cf_analysis/cf_hourlies.py | 72 +- docs/Deployment.rst | 1 - docs/DesignNotes.rst | 1 - docs/conf.py | 122 ++-- docs/index.rst | 1 - ensemble_analysis/ensemble_analysis.ipynb | 2 +- setup.py | 60 +- tests/test_bloomcast.py | 3 +- tests/test_ensemble.py | 268 ++++---- tests/test_meteo.py | 65 +- tests/test_rivers.py | 283 ++++---- tests/test_utils.py | 416 ++++++------ tests/test_wind.py | 69 +- 25 files changed, 2134 insertions(+), 1822 deletions(-) diff --git a/.gitignore b/.gitignore index 521be42..e2637be 100644 --- a/.gitignore +++ b/.gitignore @@ -313,5 +313,3 @@ flycheck_*.el # network security /network-security.data - - diff --git a/__pkg_metadata__.py b/__pkg_metadata__.py index e8318f3..a0a1242 100644 --- a/__pkg_metadata__.py +++ b/__pkg_metadata__.py @@ -12,16 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Python packaging metadata for SoG-bloomcast. -""" +"""Python packaging metadata for SoG-bloomcast.""" __all__ = [ - 'PROJECT', 'DESCRIPTION', 'VERSION', 'DEV_STATUS', + "PROJECT", + "DESCRIPTION", + "VERSION", + "DEV_STATUS", ] -PROJECT = 'SoG-bloomcast' -DESCRIPTION = 'Strait of Georgia spring diatom bloom predictor' -VERSION = '3.1' -DEV_STATUS = '5 - Production' +PROJECT = "SoG-bloomcast" +DESCRIPTION = "Strait of Georgia spring diatom bloom predictor" +VERSION = "3.1" +DEV_STATUS = "5 - Production" diff --git a/bloomcast/bloomcast.py b/bloomcast/bloomcast.py index bdcb4d5..170bd17 100644 --- a/bloomcast/bloomcast.py +++ b/bloomcast/bloomcast.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Driver module for SoG-bloomcast project -""" +"""Driver module for SoG-bloomcast project""" from copy import copy import datetime import logging @@ -93,11 +92,11 @@ # Marine Ecology-Progress Series, 388 (2009), pp. 147–157. # http://dx.doi.org/10.3354/meps08111 NITRATE_HALF_SATURATION_CONCENTRATION = 0.5 # uM -PHYTOPLANKTON_PEAK_WINDOW_HALF_WIDTH = 4 # days +PHYTOPLANKTON_PEAK_WINDOW_HALF_WIDTH = 4 # days -log = logging.getLogger('bloomcast') -bloom_date_log = logging.getLogger('bloomcast.bloom_date') +log = logging.getLogger("bloomcast") +bloom_date_log = logging.getLogger("bloomcast.bloom_date") class NoNewWindData(Exception): @@ -110,11 +109,12 @@ class Bloomcast(object): :arg config_file: Path for the bloomcast configuration file. :type config_file: string """ + # Colours for graph lines - nitrate_colours = {'avg': '#30b8b8', 'bounds': '#82dcdc'} - diatoms_colours = {'avg': 'green', 'bounds': '#56c056'} - temperature_colours = {'avg': 'red', 'bounds': '#ff7373'} - salinity_colours = {'avg': 'blue', 'bounds': '#7373ff'} + nitrate_colours = {"avg": "#30b8b8", "bounds": "#82dcdc"} + diatoms_colours = {"avg": "green", "bounds": "#56c056"} + temperature_colours = {"avg": "red", "bounds": "#ff7373"} + salinity_colours = {"avg": "blue", "bounds": "#7373ff"} def __init__(self, config_file, data_date): self.config = Config() @@ -139,30 +139,40 @@ def run(self): self._configure_logging() if not self.config.get_forcing_data and self.config.data_date is None: log.debug( - 'This will not end well: ' - 'get_forcing_data={0.get_forcing_data} ' - 'and data_date={0.data_date}'.format(self.config)) + "This will not end well: " + "get_forcing_data={0.get_forcing_data} " + "and data_date={0.data_date}".format(self.config) + ) return - log.debug('run start date/time is {0:%Y-%m-%d %H:%M:%S}' - .format(self.config.run_start_date)) + log.debug( + "run start date/time is {0:%Y-%m-%d %H:%M:%S}".format( + self.config.run_start_date + ) + ) # Check run start date and current date to ensure that # river flow data are available. # River flow data are only available in a rolling 18-month window. - run_start_yr_jan1 = ( - arrow.get(self.config.run_start_date).replace(month=1, day=1)) + run_start_yr_jan1 = arrow.get(self.config.run_start_date).replace( + month=1, day=1 + ) river_date_limit = arrow.now().replace(months=-18) if run_start_yr_jan1 < river_date_limit: log.error( - 'A bloomcast run starting {0.run_start_date:%Y-%m-%d} cannot ' - 'be done today because there are no river flow data availble ' - 'prior to {1}' - .format(self.config, river_date_limit.format('YYYY-MM-DD'))) + "A bloomcast run starting {0.run_start_date:%Y-%m-%d} cannot " + "be done today because there are no river flow data availble " + "prior to {1}".format( + self.config, river_date_limit.format("YYYY-MM-DD") + ) + ) return try: self._get_forcing_data() except NoNewWindData: - log.info('Wind data date {0:%Y-%m-%d} is unchanged since last run' - .format(self.config.data_date)) + log.info( + "Wind data date {0:%Y-%m-%d} is unchanged since last run".format( + self.config.data_date + ) + ) return self._run_SOG() self._get_results_timeseries() @@ -181,14 +191,12 @@ def _configure_logging(self): log.setLevel(logging.DEBUG) def patched_data_filter(record): - if (record.funcName == 'patch_data' - and 'data patched' in record.msg): + if record.funcName == "patch_data" and "data patched" in record.msg: return 0 return 1 console = logging.StreamHandler() - console.setFormatter( - logging.Formatter('%(levelname)s:%(name)s:%(message)s')) + console.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s")) console.setLevel(logging.INFO) if self.config.logging.debug: console.setLevel(logging.DEBUG) @@ -196,46 +204,55 @@ def patched_data_filter(record): log.addHandler(console) disk = logging.handlers.RotatingFileHandler( - self.config.logging.bloomcast_log_filename, maxBytes=1024 * 1024) + self.config.logging.bloomcast_log_filename, maxBytes=1024 * 1024 + ) disk.setFormatter( logging.Formatter( - '%(asctime)s %(levelname)s [%(name)s] %(message)s', - datefmt='%Y-%m-%d %H:%M')) + "%(asctime)s %(levelname)s [%(name)s] %(message)s", + datefmt="%Y-%m-%d %H:%M", + ) + ) disk.setLevel(logging.DEBUG) log.addHandler(disk) - mailhost = (('localhost', 1025) if self.config.logging.use_test_smtpd - else 'smtp.eos.ubc.ca') + mailhost = ( + ("localhost", 1025) + if self.config.logging.use_test_smtpd + else "smtp.eos.ubc.ca" + ) email = logging.handlers.SMTPHandler( - mailhost, fromaddr='SoG-bloomcast@eos.ubc.ca', + mailhost, + fromaddr="SoG-bloomcast@eos.ubc.ca", toaddrs=self.config.logging.toaddrs, - subject='Warning Message from SoG-bloomcast', + subject="Warning Message from SoG-bloomcast", timeout=10.0, ) - email.setFormatter( - logging.Formatter('%(levelname)s:%(name)s:%(message)s')) + email.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s")) email.setLevel(logging.WARNING) log.addHandler(email) bloom_date_evolution = logging.FileHandler( - self.config.logging.bloom_date_log_filename) - bloom_date_evolution.setFormatter(logging.Formatter('%(message)s')) + self.config.logging.bloom_date_log_filename + ) + bloom_date_evolution.setFormatter(logging.Formatter("%(message)s")) bloom_date_evolution.setLevel(logging.INFO) bloom_date_log.addHandler(bloom_date_evolution) bloom_date_log.propagate = False def _get_forcing_data(self): - """Collect and process forcing data. - """ + """Collect and process forcing data.""" if not self.config.get_forcing_data: - log.info('Skipped collection and processing of forcing data') + log.info("Skipped collection and processing of forcing data") return wind = WindProcessor(self.config) self.config.data_date = wind.make_forcing_data_file() - log.info('based on wind data forcing data date is {}' - .format(self.config.data_date.format('YYYY-MM-DD'))) + log.info( + "based on wind data forcing data date is {}".format( + self.config.data_date.format("YYYY-MM-DD") + ) + ) try: - with open('wind_data_date', 'rt') as f: + with open("wind_data_date", "rt") as f: last_data_date = arrow.get(f.readline().strip()).date() except IOError: # Fake a wind data date to get things rolling @@ -243,31 +260,33 @@ def _get_forcing_data(self): if self.config.data_date == last_data_date: raise NoNewWindData else: - with open('wind_data_date', 'wt') as f: - f.write( - '{}\n'.format(self.config.data_date.format('YYYY-MM-DD'))) + with open("wind_data_date", "wt") as f: + f.write("{}\n".format(self.config.data_date.format("YYYY-MM-DD"))) meteo = MeteoProcessor(self.config) meteo.make_forcing_data_files() rivers = RiversProcessor(self.config) rivers.make_forcing_data_files() def _run_SOG(self): - """Run SOG. - """ + """Run SOG.""" if not self.config.run_SOG: - log.info('Skipped running SOG') + log.info("Skipped running SOG") return processes = {} - base_infile = self.config.infiles['base'] - for key in self.config.infiles['edits']: + base_infile = self.config.infiles["base"] + for key in self.config.infiles["edits"]: proc = SOGcommand.api.run( self.config.SOG_executable, base_infile, - self.config.infiles['edits'][key], - key + '.stdout') + self.config.infiles["edits"][key], + key + ".stdout", + ) processes[key] = proc - log.info('SOG {0} run started at {1:%Y-%m-%d %H:%M:%S} as pid {2}' - .format(key, datetime.datetime.now(), proc.pid)) + log.info( + "SOG {0} run started at {1:%Y-%m-%d %H:%M:%S} as pid {2}".format( + key, datetime.datetime.now(), proc.pid + ) + ) while processes: time.sleep(30) for key, proc in copy(processes).items(): @@ -275,8 +294,11 @@ def _run_SOG(self): continue else: processes.pop(key) - log.info('SOG {0} run finished at {1:%Y-%m-%d %H:%M:%S}' - .format(key, datetime.datetime.now())) + log.info( + "SOG {0} run finished at {1:%Y-%m-%d %H:%M:%S}".format( + key, datetime.datetime.now() + ) + ) def _get_results_timeseries(self): """Read SOG results time series of interest and create @@ -285,122 +307,137 @@ def _get_results_timeseries(self): self.nitrate, self.diatoms = {}, {} self.temperature, self.salinity = {}, {} self.mixing_layer_depth = {} - for key in self.config.infiles['edits']: + for key in self.config.infiles["edits"]: std_bio_ts_outfile = self.config.std_bio_ts_outfiles[key] std_phys_ts_outfile = self.config.std_phys_ts_outfiles[key] self.nitrate[key] = SOG_Timeseries(std_bio_ts_outfile) - self.nitrate[key].read_data( - 'time', '3 m avg nitrate concentration') + self.nitrate[key].read_data("time", "3 m avg nitrate concentration") self.nitrate[key].calc_mpl_dates(self.config.run_start_date) self.diatoms[key] = SOG_Timeseries(std_bio_ts_outfile) - self.diatoms[key].read_data( - 'time', '3 m avg micro phytoplankton biomass') + self.diatoms[key].read_data("time", "3 m avg micro phytoplankton biomass") self.diatoms[key].calc_mpl_dates(self.config.run_start_date) self.temperature[key] = SOG_Timeseries(std_phys_ts_outfile) - self.temperature[key].read_data('time', '3 m avg temperature') + self.temperature[key].read_data("time", "3 m avg temperature") self.temperature[key].calc_mpl_dates(self.config.run_start_date) self.salinity[key] = SOG_Timeseries(std_phys_ts_outfile) - self.salinity[key].read_data('time', '3 m avg salinity') + self.salinity[key].read_data("time", "3 m avg salinity") self.salinity[key].calc_mpl_dates(self.config.run_start_date) self.mixing_layer_depth[key] = SOG_Timeseries(std_phys_ts_outfile) - self.mixing_layer_depth[key].read_data( - 'time', 'mixing layer depth') - self.mixing_layer_depth[key].calc_mpl_dates( - self.config.run_start_date) + self.mixing_layer_depth[key].read_data("time", "mixing layer depth") + self.mixing_layer_depth[key].calc_mpl_dates(self.config.run_start_date) def _create_timeseries_graphs(self): - """Create time series graph objects. - """ + """Create time series graph objects.""" self.fig_nitrate_diatoms_ts = self._two_axis_timeseries( - self.nitrate, self.diatoms, - titles=('3 m Avg Nitrate Concentration [uM N]', - '3 m Avg Diatom Biomass [uM N]'), - colors=(self.nitrate_colours, self.diatoms_colours)) + self.nitrate, + self.diatoms, + titles=( + "3 m Avg Nitrate Concentration [uM N]", + "3 m Avg Diatom Biomass [uM N]", + ), + colors=(self.nitrate_colours, self.diatoms_colours), + ) self.fig_temperature_salinity_ts = self._two_axis_timeseries( - self.temperature, self.salinity, - titles=('3 m Avg Temperature [deg C]', - '3 m Avg Salinity [-]'), - colors=(self.temperature_colours, self.salinity_colours)) + self.temperature, + self.salinity, + titles=("3 m Avg Temperature [deg C]", "3 m Avg Salinity [-]"), + colors=(self.temperature_colours, self.salinity_colours), + ) self.fig_mixing_layer_depth_ts = self._mixing_layer_depth_timeseries() def _two_axis_timeseries(self, left_ts, right_ts, titles, colors): """Create a time series graph figure object with 2 time series plotted on the left and right y axes. """ - fig = Figure((8, 3), facecolor='white') + fig = Figure((8, 3), facecolor="white") ax_left = fig.add_subplot(1, 1, 1) ax_left.set_position((0.125, 0.1, 0.775, 0.75)) fig.ax_left = ax_left ax_right = ax_left.twinx() ax_right.set_position(ax_left.get_position()) - predicate = (left_ts['avg_forcing'].mpl_dates - >= date2num(self.config.data_date)) - for key in 'early_bloom_forcing late_bloom_forcing'.split(): - ax_left.plot(left_ts[key].mpl_dates[predicate], - left_ts[key].dep_data[predicate], - color=colors[0]['bounds']) - ax_right.plot(right_ts[key].mpl_dates[predicate], - right_ts[key].dep_data[predicate], - color=colors[1]['bounds']) - ax_left.plot(left_ts['avg_forcing'].mpl_dates, - left_ts['avg_forcing'].dep_data, - color=colors[0]['avg']) - ax_right.plot(right_ts['avg_forcing'].mpl_dates, - right_ts['avg_forcing'].dep_data, - color=colors[1]['avg']) - ax_left.set_ylabel(titles[0], color=colors[0]['avg'], size='x-small') - ax_right.set_ylabel(titles[1], color=colors[1]['avg'], size='x-small') + predicate = left_ts["avg_forcing"].mpl_dates >= date2num(self.config.data_date) + for key in "early_bloom_forcing late_bloom_forcing".split(): + ax_left.plot( + left_ts[key].mpl_dates[predicate], + left_ts[key].dep_data[predicate], + color=colors[0]["bounds"], + ) + ax_right.plot( + right_ts[key].mpl_dates[predicate], + right_ts[key].dep_data[predicate], + color=colors[1]["bounds"], + ) + ax_left.plot( + left_ts["avg_forcing"].mpl_dates, + left_ts["avg_forcing"].dep_data, + color=colors[0]["avg"], + ) + ax_right.plot( + right_ts["avg_forcing"].mpl_dates, + right_ts["avg_forcing"].dep_data, + color=colors[1]["avg"], + ) + ax_left.set_ylabel(titles[0], color=colors[0]["avg"], size="x-small") + ax_right.set_ylabel(titles[1], color=colors[1]["avg"], size="x-small") # Add line to mark switch from actual to averaged forcing data fig.data_date_line = ax_left.axvline( - date2num(self.config.data_date), color='black') + date2num(self.config.data_date), color="black" + ) # Format x-axis ax_left.xaxis.set_major_locator(MonthLocator()) - ax_left.xaxis.set_major_formatter(DateFormatter('%j\n%b')) + ax_left.xaxis.set_major_formatter(DateFormatter("%j\n%b")) for axis in (ax_left, ax_right): for label in axis.get_xticklabels() + axis.get_yticklabels(): - label.set_size('x-small') + label.set_size("x-small") ax_left.set_xlim( - (int(left_ts['avg_forcing'].mpl_dates[0]), - math.ceil(left_ts['avg_forcing'].mpl_dates[-1]))) + ( + int(left_ts["avg_forcing"].mpl_dates[0]), + math.ceil(left_ts["avg_forcing"].mpl_dates[-1]), + ) + ) ax_left.set_xlabel( - 'Year-days in {0} and {1}' - .format(self.config.run_start_date.year, - self.config.run_start_date.year + 1), - size='x-small') + "Year-days in {0} and {1}".format( + self.config.run_start_date.year, self.config.run_start_date.year + 1 + ), + size="x-small", + ) return fig def _mixing_layer_depth_timeseries(self): """Create a time series graph figure object of the mixing layer depth on the wind data date and the 6 days preceding it. """ - fig = Figure((8, 3), facecolor='white') + fig = Figure((8, 3), facecolor="white") ax = fig.add_subplot(1, 1, 1) ax.set_position((0.125, 0.1, 0.775, 0.75)) predicate = np.logical_and( - self.mixing_layer_depth['avg_forcing'].mpl_dates + self.mixing_layer_depth["avg_forcing"].mpl_dates > date2num(self.config.data_date - datetime.timedelta(days=6)), - self.mixing_layer_depth['avg_forcing'].mpl_dates - <= date2num(self.config.data_date + datetime.timedelta(days=1))) - mpl_dates = self.mixing_layer_depth['avg_forcing'].mpl_dates[predicate] - dep_data = self.mixing_layer_depth['avg_forcing'].dep_data[predicate] - ax.plot(mpl_dates, dep_data, color='magenta') - ax.set_ylabel( - 'Mixing Layer Depth [m]', color='magenta', size='x-small') + self.mixing_layer_depth["avg_forcing"].mpl_dates + <= date2num(self.config.data_date + datetime.timedelta(days=1)), + ) + mpl_dates = self.mixing_layer_depth["avg_forcing"].mpl_dates[predicate] + dep_data = self.mixing_layer_depth["avg_forcing"].dep_data[predicate] + ax.plot(mpl_dates, dep_data, color="magenta") + ax.set_ylabel("Mixing Layer Depth [m]", color="magenta", size="x-small") # Add line to mark profile time profile_datetime = datetime.datetime.combine( - self.config.data_date, datetime.time(12)) - profile_datetime_line = ax.axvline( - date2num(profile_datetime), color='black') + self.config.data_date, datetime.time(12) + ) + profile_datetime_line = ax.axvline(date2num(profile_datetime), color="black") ax.xaxis.set_major_locator(DayLocator()) - ax.xaxis.set_major_formatter(DateFormatter('%j\n%d-%b')) + ax.xaxis.set_major_formatter(DateFormatter("%j\n%d-%b")) ax.xaxis.set_minor_locator(HourLocator(interval=6)) for label in ax.get_xticklabels() + ax.get_yticklabels(): - label.set_size('x-small') + label.set_size("x-small") ax.set_xlim((int(mpl_dates[0]), math.ceil(mpl_dates[-1]))) - ax.set_xlabel('Year-Day', size='x-small') + ax.set_xlabel("Year-Day", size="x-small") fig.legend( - [profile_datetime_line], ['Profile Time'], - loc='upper right', prop={'size': 'xx-small'}) + [profile_datetime_line], + ["Profile Time"], + loc="upper right", + prop={"size": "xx-small"}, + ) return fig def _get_results_profiles(self): @@ -409,107 +446,119 @@ def _get_results_profiles(self): """ self.nitrate_profile, self.diatoms_profile = {}, {} self.temperature_profile, self.salinity_profile = {}, {} - for key in self.config.infiles['edits']: - Hoffmueller_outfile = ( - self.config.Hoffmueller_profiles_outfiles[key]) + for key in self.config.infiles["edits"]: + Hoffmueller_outfile = self.config.Hoffmueller_profiles_outfiles[key] profile_number = ( - self.config.data_date - self.config.run_start_date.date()).days - self.nitrate_profile[key] = SOG_HoffmuellerProfile( - Hoffmueller_outfile) - self.nitrate_profile[key].read_data( - 'depth', 'nitrate', profile_number) - self.diatoms_profile[key] = SOG_HoffmuellerProfile( - Hoffmueller_outfile) + self.config.data_date - self.config.run_start_date.date() + ).days + self.nitrate_profile[key] = SOG_HoffmuellerProfile(Hoffmueller_outfile) + self.nitrate_profile[key].read_data("depth", "nitrate", profile_number) + self.diatoms_profile[key] = SOG_HoffmuellerProfile(Hoffmueller_outfile) self.diatoms_profile[key].read_data( - 'depth', 'micro phytoplankton', profile_number) - self.temperature_profile[key] = SOG_HoffmuellerProfile( - Hoffmueller_outfile) + "depth", "micro phytoplankton", profile_number + ) + self.temperature_profile[key] = SOG_HoffmuellerProfile(Hoffmueller_outfile) self.temperature_profile[key].read_data( - 'depth', 'temperature', profile_number) - self.salinity_profile[key] = SOG_HoffmuellerProfile( - Hoffmueller_outfile) - self.salinity_profile[key].read_data( - 'depth', 'salinity', profile_number) + "depth", "temperature", profile_number + ) + self.salinity_profile[key] = SOG_HoffmuellerProfile(Hoffmueller_outfile) + self.salinity_profile[key].read_data("depth", "salinity", profile_number) def _create_profile_graphs(self): - """Create profile graph objects. - """ + """Create profile graph objects.""" profile_datetime = datetime.datetime.combine( - self.config.data_date, datetime.time(12)) + self.config.data_date, datetime.time(12) + ) profile_dt = profile_datetime - self.config.run_start_date profile_hour = profile_dt.days * 24 + profile_dt.seconds / 3600 - self.mixing_layer_depth['avg_forcing'].boolean_slice( - self.mixing_layer_depth['avg_forcing'].indep_data >= profile_hour) - mixing_layer_depth = self.mixing_layer_depth['avg_forcing'].dep_data[0] + self.mixing_layer_depth["avg_forcing"].boolean_slice( + self.mixing_layer_depth["avg_forcing"].indep_data >= profile_hour + ) + mixing_layer_depth = self.mixing_layer_depth["avg_forcing"].dep_data[0] self.fig_temperature_salinity_profile = self._two_axis_profile( - self.temperature_profile['avg_forcing'], - self.salinity_profile['avg_forcing'], + self.temperature_profile["avg_forcing"], + self.salinity_profile["avg_forcing"], mixing_layer_depth, - titles=('Temperature [deg C]', 'Salinity [-]'), + titles=("Temperature [deg C]", "Salinity [-]"), colors=(self.temperature_colours, self.salinity_colours), - limits=((4, 10), (20, 30))) + limits=((4, 10), (20, 30)), + ) self.fig_nitrate_diatoms_profile = self._two_axis_profile( - self.nitrate_profile['avg_forcing'], - self.diatoms_profile['avg_forcing'], + self.nitrate_profile["avg_forcing"], + self.diatoms_profile["avg_forcing"], mixing_layer_depth, - titles=('Nitrate Concentration [uM N]', 'Diatom Biomass [uM N]'), - colors=(self.nitrate_colours, self.diatoms_colours)) + titles=("Nitrate Concentration [uM N]", "Diatom Biomass [uM N]"), + colors=(self.nitrate_colours, self.diatoms_colours), + ) - def _two_axis_profile(self, top_profile, bottom_profile, - mixing_layer_depth, titles, colors, limits=None): + def _two_axis_profile( + self, + top_profile, + bottom_profile, + mixing_layer_depth, + titles, + colors, + limits=None, + ): """Create a profile graph figure object with 2 profiles plotted on the top and bottom x axes. """ - fig = Figure((4, 8), facecolor='white') + fig = Figure((4, 8), facecolor="white") ax_bottom = fig.add_subplot(1, 1, 1) ax_bottom.set_position((0.19, 0.1, 0.5, 0.8)) ax_top = ax_bottom.twiny() ax_top.set_position(ax_bottom.get_position()) ax_top.plot( - top_profile.dep_data, top_profile.indep_data, - color=colors[0]['avg']) - ax_top.set_xlabel(titles[0], color=colors[0]['avg'], size='small') - ax_bottom.plot(bottom_profile.dep_data, bottom_profile.indep_data, - color=colors[1]['avg']) - ax_bottom.set_xlabel(titles[1], color=colors[1]['avg'], size='small') + top_profile.dep_data, top_profile.indep_data, color=colors[0]["avg"] + ) + ax_top.set_xlabel(titles[0], color=colors[0]["avg"], size="small") + ax_bottom.plot( + bottom_profile.dep_data, bottom_profile.indep_data, color=colors[1]["avg"] + ) + ax_bottom.set_xlabel(titles[1], color=colors[1]["avg"], size="small") for axis in (ax_bottom, ax_top): for label in axis.get_xticklabels() + axis.get_yticklabels(): - label.set_size('x-small') + label.set_size("x-small") if limits is not None: ax_top.set_xlim(limits[0]) ax_bottom.set_xlim(limits[1]) - ax_bottom.axhline(mixing_layer_depth, color='black') + ax_bottom.axhline(mixing_layer_depth, color="black") ax_bottom.text( - x=ax_bottom.get_xlim()[1], y=mixing_layer_depth, - s=' Mixing Layer\n Depth = {0:.2f} m'.format(mixing_layer_depth), - verticalalignment='center', size='small') + x=ax_bottom.get_xlim()[1], + y=mixing_layer_depth, + s=" Mixing Layer\n Depth = {0:.2f} m".format(mixing_layer_depth), + verticalalignment="center", + size="small", + ) ax_bottom.set_ylim( - (bottom_profile.indep_data[-1], bottom_profile.indep_data[0])) - ax_bottom.set_ylabel('Depth [m]', size='small') + (bottom_profile.indep_data[-1], bottom_profile.indep_data[0]) + ) + ax_bottom.set_ylabel("Depth [m]", size="small") return fig def _calc_bloom_date(self): - """Calculate the predicted spring bloom date. - """ - key = 'avg_forcing' + """Calculate the predicted spring bloom date.""" + key = "avg_forcing" self.bloom_date, self.bloom_biomass = {}, {} - for key in self.config.infiles['edits']: + for key in self.config.infiles["edits"]: self._clip_results_to_jan1(key) self._reduce_results_to_daily(key) first_low_nitrate_days = self._find_low_nitrate_days( - key, NITRATE_HALF_SATURATION_CONCENTRATION) + key, NITRATE_HALF_SATURATION_CONCENTRATION + ) self._find_phytoplankton_peak( - key, first_low_nitrate_days, - PHYTOPLANKTON_PEAK_WINDOW_HALF_WIDTH) + key, first_low_nitrate_days, PHYTOPLANKTON_PEAK_WINDOW_HALF_WIDTH + ) if self.config.get_forcing_data or self.config.run_SOG: - line = (' {0} {1} {2:.4f}' - .format(self.config.data_date.format('YYYY-MM-DD'), - self.bloom_date['avg_forcing'], - self.bloom_biomass['avg_forcing'])) - for key in 'early_bloom_forcing late_bloom_forcing'.split(): - line += (' {0} {1:.4f}' - .format(self.bloom_date[key], - self.bloom_biomass[key])) + line = " {0} {1} {2:.4f}".format( + self.config.data_date.format("YYYY-MM-DD"), + self.bloom_date["avg_forcing"], + self.bloom_biomass["avg_forcing"], + ) + for key in "early_bloom_forcing late_bloom_forcing".split(): + line += " {0} {1:.4f}".format( + self.bloom_date[key], self.bloom_biomass[key] + ) bloom_date_log.info(line) def _clip_results_to_jan1(self, key): @@ -537,34 +586,47 @@ def _reduce_results_to_daily(self, key): # day day_slice = 86400 // self.config.SOG_timestep day_iterator = range( - 0, self.nitrate[key].dep_data.shape[0] - day_slice, day_slice) + 0, self.nitrate[key].dep_data.shape[0] - day_slice, day_slice + ) jan1 = datetime.date(self.config.run_start_date.year + 1, 1, 1) self.nitrate[key].dep_data = np.array( - [self.nitrate[key].dep_data[i:i + day_slice].min() - for i in day_iterator]) + [self.nitrate[key].dep_data[i : i + day_slice].min() for i in day_iterator] + ) self.nitrate[key].indep_data = np.array( - [jan1 + datetime.timedelta(days=i) - for i in range(self.nitrate[key].dep_data.size)]) + [ + jan1 + datetime.timedelta(days=i) + for i in range(self.nitrate[key].dep_data.size) + ] + ) day_iterator = range( - 0, self.diatoms[key].dep_data.shape[0] - day_slice, day_slice) + 0, self.diatoms[key].dep_data.shape[0] - day_slice, day_slice + ) self.diatoms[key].dep_data = np.array( - [self.diatoms[key].dep_data[i:i + day_slice].max() - for i in day_iterator]) + [self.diatoms[key].dep_data[i : i + day_slice].max() for i in day_iterator] + ) self.diatoms[key].indep_data = np.array( - [jan1 + datetime.timedelta(days=i) - for i in range(self.diatoms[key].dep_data.size)]) + [ + jan1 + datetime.timedelta(days=i) + for i in range(self.diatoms[key].dep_data.size) + ] + ) def _find_low_nitrate_days(self, key, threshold): """Return the start and end dates of the first 2 day period in which the nitrate concentration is below the ``threshold``. """ - key_string = key.replace('_', ' ') - self.nitrate[key].boolean_slice( - self.nitrate[key].dep_data <= threshold) - log.debug('Dates on which nitrate was <= {0} uM N with {1}:\n{2}' - .format(threshold, key_string, self.nitrate[key].indep_data)) - log.debug('Nitrate <= {0} uM N with {1}:\n{2}' - .format(threshold, key_string, self.nitrate[key].dep_data)) + key_string = key.replace("_", " ") + self.nitrate[key].boolean_slice(self.nitrate[key].dep_data <= threshold) + log.debug( + "Dates on which nitrate was <= {0} uM N with {1}:\n{2}".format( + threshold, key_string, self.nitrate[key].indep_data + ) + ) + log.debug( + "Nitrate <= {0} uM N with {1}:\n{2}".format( + threshold, key_string, self.nitrate[key].dep_data + ) + ) for i in range(self.nitrate[key].dep_data.shape[0]): low_nitrate_day_1 = self.nitrate[key].indep_data[i] days = self.nitrate[key].indep_data[i + 1] - low_nitrate_day_1 @@ -573,35 +635,44 @@ def _find_low_nitrate_days(self, key, threshold): break return low_nitrate_day_1, low_nitrate_day_2 - def _find_phytoplankton_peak(self, key, first_low_nitrate_days, - peak_half_width): + def _find_phytoplankton_peak(self, key, first_low_nitrate_days, peak_half_width): """Return the date with ``peak_half_width`` of the ``first_low_nitrate_days`` on which the diatoms biomass is the greatest. """ - key_string = key.replace('_', ' ') + key_string = key.replace("_", " ") half_width_days = datetime.timedelta(days=peak_half_width) early_bloom_date = first_low_nitrate_days[0] - half_width_days late_bloom_date = first_low_nitrate_days[1] + half_width_days - log.debug('Bloom window for {0} is between {1} and {2}' - .format(key_string, early_bloom_date, late_bloom_date)) - self.diatoms[key].boolean_slice( - self.diatoms[key].indep_data >= early_bloom_date) + log.debug( + "Bloom window for {0} is between {1} and {2}".format( + key_string, early_bloom_date, late_bloom_date + ) + ) self.diatoms[key].boolean_slice( - self.diatoms[key].indep_data <= late_bloom_date) - log.debug('Dates in {0} bloom window:\n{1}' - .format(key_string, self.diatoms[key].indep_data)) - log.debug('Micro phytoplankton biomass values in ' - '{0} bloom window:\n{1}' - .format(key_string, self.diatoms[key].dep_data)) + self.diatoms[key].indep_data >= early_bloom_date + ) + self.diatoms[key].boolean_slice(self.diatoms[key].indep_data <= late_bloom_date) + log.debug( + "Dates in {0} bloom window:\n{1}".format( + key_string, self.diatoms[key].indep_data + ) + ) + log.debug( + "Micro phytoplankton biomass values in " + "{0} bloom window:\n{1}".format(key_string, self.diatoms[key].dep_data) + ) bloom_date_index = self.diatoms[key].dep_data.argmax() self.bloom_date[key] = self.diatoms[key].indep_data[bloom_date_index] self.bloom_biomass[key] = self.diatoms[key].dep_data[bloom_date_index] - log.info('Predicted {0} bloom date is {1}' - .format(key_string, self.bloom_date[key])) + log.info( + "Predicted {0} bloom date is {1}".format(key_string, self.bloom_date[key]) + ) log.debug( - 'Phytoplankton biomass on {0} bloom date is {1} uM N' - .format(key_string, self.bloom_biomass[key])) + "Phytoplankton biomass on {0} bloom date is {1} uM N".format( + key_string, self.bloom_biomass[key] + ) + ) def clip_results_to_jan1(nitrate, diatoms, run_start_date): @@ -660,20 +731,26 @@ def reduce_results_to_daily(nitrate, diatoms, run_start_date, SOG_timestep): last_day = nitrate[member].dep_data.shape[0] - day_slice day_iterator = range(0, last_day, day_slice) nitrate[member].dep_data = np.array( - [nitrate[member].dep_data[i:i + day_slice].min() - for i in day_iterator]) + [nitrate[member].dep_data[i : i + day_slice].min() for i in day_iterator] + ) nitrate[member].indep_data = np.array( - [jan1 + datetime.timedelta(days=i) - for i in range(nitrate[member].dep_data.size)]) + [ + jan1 + datetime.timedelta(days=i) + for i in range(nitrate[member].dep_data.size) + ] + ) last_day = diatoms[member].dep_data.shape[0] - day_slice day_iterator = range(0, last_day, day_slice) diatoms[member].dep_data = np.array( - [diatoms[member].dep_data[i:i + day_slice].max() - for i in day_iterator]) + [diatoms[member].dep_data[i : i + day_slice].max() for i in day_iterator] + ) diatoms[member].indep_data = np.array( - [jan1 + datetime.timedelta(days=i) - for i in range(diatoms[member].dep_data.size)]) + [ + jan1 + datetime.timedelta(days=i) + for i in range(diatoms[member].dep_data.size) + ] + ) def find_low_nitrate_days(nitrate, threshold): @@ -701,14 +778,10 @@ def find_phytoplankton_peak(diatoms, first_low_nitrate_days, peak_half_width): half_width_days = datetime.timedelta(days=peak_half_width) bloom_dates, bloom_biomasses = {}, {} for member in diatoms: - bloom_window_start = ( - first_low_nitrate_days[member][0] - half_width_days) - bloom_window_end = ( - first_low_nitrate_days[member][1] + half_width_days) - diatoms[member].boolean_slice( - diatoms[member].indep_data >= bloom_window_start) - diatoms[member].boolean_slice( - diatoms[member].indep_data <= bloom_window_end) + bloom_window_start = first_low_nitrate_days[member][0] - half_width_days + bloom_window_end = first_low_nitrate_days[member][1] + half_width_days + diatoms[member].boolean_slice(diatoms[member].indep_data >= bloom_window_start) + diatoms[member].boolean_slice(diatoms[member].indep_data <= bloom_window_end) bloom_date_index = diatoms[member].dep_data.argmax() bloom_dates[member] = diatoms[member].indep_data[bloom_date_index] bloom_biomasses[member] = diatoms[member].dep_data[bloom_date_index] @@ -719,12 +792,12 @@ def main(): try: config_file = sys.argv[1] except IndexError: - print('Expected config file path/name') + print("Expected config file path/name") sys.exit(1) try: data_date = arrow.get(sys.argv[2]) except ValueError: - print('Expected %Y-%m-%d for data date, got: {0[2]}'.format(sys.argv)) + print("Expected %Y-%m-%d for data date, got: {0[2]}".format(sys.argv)) sys.exit(1) except IndexError: data_date = None diff --git a/bloomcast/carbonate.py b/bloomcast/carbonate.py index b028597..5d73f42 100644 --- a/bloomcast/carbonate.py +++ b/bloomcast/carbonate.py @@ -38,9 +38,9 @@ def calc_carbonate(TAlk, DIC, sigma_t, S, T, P, PO4, Si): # Convert from uM to mol/kg TAlk = TAlk * 1.0e-3 / (sigma_t + 1.0e3) - DIC = DIC * 1.0e-3 / (sigma_t + 1.0e3) - PO4 = PO4 * 1.0e-3 / (sigma_t + 1.0e3) - Si = Si * 1.0e-3 / (sigma_t + 1.0e3) + DIC = DIC * 1.0e-3 / (sigma_t + 1.0e3) + PO4 = PO4 * 1.0e-3 / (sigma_t + 1.0e3) + Si = Si * 1.0e-3 / (sigma_t + 1.0e3) # Calculate pH and Omega_A pH = CalculatepHfromTATC(TAlk, DIC, PO4, Si) @@ -67,10 +67,10 @@ def set_constants(Sal, TempK, Pdbar): R_gas = 83.1451 # ml bar-1 K-1 mol-1, DOEv2 # Preallocate common operations - Pbar = Pdbar / 10.0 + Pbar = Pdbar / 10.0 TempK100 = TempK / 100.0 logTempK = np.log(TempK) - sqrSal = np.sqrt(Sal) + sqrSal = np.sqrt(Sal) # Calculate IonS: # This is from the DOE handbook, Chapter 5, p. 13/22, eq. 7.2.4: @@ -82,18 +82,18 @@ def set_constants(Sal, TempK, Pdbar): # Uppstrom, L., Deep-Sea Research 21:161-162, 1974: # this is 0.000416 * Sali / 35 = 0.0000119 * Sali # TB = (0.000232d0 / 10.811d0) * (Sal / 1.80655d0) ! in mol/kg-SW - TB = 0.0004157 * Sal / 35.0 # in mol/kg-SW + TB = 0.0004157 * Sal / 35.0 # in mol/kg-SW # Calculate total sulfate: # Morris, A. W., and Riley, J. P., Deep-Sea Research 13:699-705, 1966: # this is .02824 * Sali / 35 = .0008067 * Sali - TS = (0.14 / 96.062) * (Sal / 1.80655) # in mol/kg-SW + TS = (0.14 / 96.062) * (Sal / 1.80655) # in mol/kg-SW # Calculate total fluoride: # Riley, J. P., Deep-Sea Research 12:219-220, 1965: # this is .000068 * Sali / 35 = .00000195 * Sali # Approximate [F-] of Fraser River is 3 umol/kg - TF = np.maximum((0.000067 / 18.998) * (Sal / 1.80655), 3.0e-6) # in mol/kg-SW + TF = np.maximum((0.000067 / 18.998) * (Sal / 1.80655), 3.0e-6) # in mol/kg-SW # CALCULATE EQUILIBRIUM CONSTANTS (SW scale) # Calculate KS: @@ -102,27 +102,38 @@ def set_constants(Sal, TempK, Pdbar): # It was given in mol/kg-H2O. I convert it to mol/kg-SW. # TYPO on p. 121: the constant e9 should be e8. # This is from eqs 22 and 23 on p. 123, and Table 4 on p 121: - lnKS = (-4276.1 / TempK + 141.328 - 23.093 * logTempK + - (-13856.0 / TempK + 324.57 - 47.986 * logTempK) * sqrIonS + - (35474.0 / TempK - 771.54 + 114.723 * logTempK) * IonS + - (-2698.0 / TempK) * sqrIonS * IonS + (1776.0 / TempK) * IonS**2) - KS = (np.exp(lnKS) # this is on the free pH scale in mol/kg-H2O - * (1.0 - 0.001005 * Sal)) # convert to mol/kg-SW + lnKS = ( + -4276.1 / TempK + + 141.328 + - 23.093 * logTempK + + (-13856.0 / TempK + 324.57 - 47.986 * logTempK) * sqrIonS + + (35474.0 / TempK - 771.54 + 114.723 * logTempK) * IonS + + (-2698.0 / TempK) * sqrIonS * IonS + + (1776.0 / TempK) * IonS**2 + ) + KS = np.exp(lnKS) * ( # this is on the free pH scale in mol/kg-H2O + 1.0 - 0.001005 * Sal + ) # convert to mol/kg-SW # Calculate KF: # Dickson, A. G. and Riley, J. P., Marine Chemistry 7:89-99, 1979: lnKF = 1590.2 / TempK - 12.641 + 1.525 * sqrIonS - KF = (np.exp(lnKF) # this is on the free pH scale in mol/kg-H2O - * (1.0 - 0.001005 * Sal)) # convert to mol/kg-SW + KF = np.exp(lnKF) * ( # this is on the free pH scale in mol/kg-H2O + 1.0 - 0.001005 * Sal + ) # convert to mol/kg-SW # Calculate pH scale conversion factors ( NOT pressure-corrected) - SWStoTOT = (1 + TS / KS) / (1 + TS / KS + TF / KF) + SWStoTOT = (1 + TS / KS) / (1 + TS / KS + TF / KF) # Calculate K0: # Weiss, R. F., Marine Chemistry 2:203-215, 1974. - lnK0 = (-60.2409 + 93.4517 / TempK100 + 23.3585 * np.log(TempK100) + - Sal * (0.023517 - 0.023656 * TempK100 + 0.0047036 * TempK100**2)) - K0 = np.exp(lnK0) # this is in mol/kg-SW/atm + lnK0 = ( + -60.2409 + + 93.4517 / TempK100 + + 23.3585 * np.log(TempK100) + + Sal * (0.023517 - 0.023656 * TempK100 + 0.0047036 * TempK100**2) + ) + K0 = np.exp(lnK0) # this is in mol/kg-SW/atm # From Millero, 2010, also for estuarine use. # Marine and Freshwater Research, v. 61, p. 139–142. @@ -133,83 +144,112 @@ def set_constants(Sal, TempK, Pdbar): # This is from page 141 pK10 = -126.34048 + 6320.813 / TempK + 19.568224 * np.log(TempK) # This is from their table 2, page 140. - A1 = 13.4038 * Sal**0.5 + 0.03206 * Sal - 5.242e-5 * Sal**2 - B1 = -530.659 * Sal**0.5 - 5.8210 * Sal - C1 = -2.0664 * Sal**0.5 - pK1 = pK10 + A1 + B1 / TempK + C1 * np.log(TempK) - K1 = 10**(-pK1) + A1 = 13.4038 * Sal**0.5 + 0.03206 * Sal - 5.242e-5 * Sal**2 + B1 = -530.659 * Sal**0.5 - 5.8210 * Sal + C1 = -2.0664 * Sal**0.5 + pK1 = pK10 + A1 + B1 / TempK + C1 * np.log(TempK) + K1 = 10 ** (-pK1) # This is from page 141 - pK20 = -90.18333 + 5143.692 / TempK + 14.613358 * np.log(TempK) + pK20 = -90.18333 + 5143.692 / TempK + 14.613358 * np.log(TempK) # This is from their table 3, page 140. - A2 = 21.3728 * Sal**0.5 + 0.1218 * Sal - 3.688e-4 * Sal**2 - B2 = -788.289 * Sal**0.5 - 19.189 * Sal - C2 = -3.374 * Sal**0.5 - pK2 = pK20 + A2 + B2 / TempK + C2 * np.log(TempK) - K2 = 10**(-pK2) + A2 = 21.3728 * Sal**0.5 + 0.1218 * Sal - 3.688e-4 * Sal**2 + B2 = -788.289 * Sal**0.5 - 19.189 * Sal + C2 = -3.374 * Sal**0.5 + pK2 = pK20 + A2 + B2 / TempK + C2 * np.log(TempK) + K2 = 10 ** (-pK2) # Calculate KW: # Millero, Geochemica et Cosmochemica Acta 59:661-677, 1995. # his check value of 1.6 umol/kg-SW should be 6.2 - lnKW = (148.9802 - 13847.26 / TempK - 23.6521 * logTempK + - (-5.977 + 118.67 / TempK + 1.0495 * logTempK) * sqrSal - - 0.01615 * Sal) - KW = np.exp(lnKW) # this is on the SWS pH scale in (mol/kg-SW)^2 + lnKW = ( + 148.9802 + - 13847.26 / TempK + - 23.6521 * logTempK + + (-5.977 + 118.67 / TempK + 1.0495 * logTempK) * sqrSal + - 0.01615 * Sal + ) + KW = np.exp(lnKW) # this is on the SWS pH scale in (mol/kg-SW)^2 # Calculate KB: # Dickson, A. G., Deep-Sea Research 37:755-766, 1990: - lnKB = ((-8966.9 - 2890.53 * sqrSal - 77.942 * Sal + - 1.728 * sqrSal * Sal - 0.0996 * Sal**2) / TempK + - 148.0248 + 137.1942 * sqrSal + 1.62142 * Sal + - (-24.4344 - 25.085 * sqrSal - 0.2474 * Sal) * logTempK + - 0.053105 * sqrSal * TempK) - KB = (np.exp(lnKB) # this is on the total pH scale in mol/kg-SW - / SWStoTOT) # convert to SWS pH scale + lnKB = ( + ( + -8966.9 + - 2890.53 * sqrSal + - 77.942 * Sal + + 1.728 * sqrSal * Sal + - 0.0996 * Sal**2 + ) + / TempK + + 148.0248 + + 137.1942 * sqrSal + + 1.62142 * Sal + + (-24.4344 - 25.085 * sqrSal - 0.2474 * Sal) * logTempK + + 0.053105 * sqrSal * TempK + ) + KB = ( + np.exp(lnKB) / SWStoTOT # this is on the total pH scale in mol/kg-SW + ) # convert to SWS pH scale # Calculate KP1, KP2, KP3, and KSi: # Yao and Millero, Aquatic Geochemistry 1:53-88, 1995 # KP1, KP2, KP3 are on the SWS pH scale in mol/kg-SW. # KSi was given on the SWS pH scale in molal units. - lnKP1 = (-4576.752 / TempK + 115.54 - 18.453 * logTempK + - (-106.736 / TempK + 0.69171) * sqrSal + - (-0.65643 / TempK - 0.01844) * Sal) + lnKP1 = ( + -4576.752 / TempK + + 115.54 + - 18.453 * logTempK + + (-106.736 / TempK + 0.69171) * sqrSal + + (-0.65643 / TempK - 0.01844) * Sal + ) KP1 = np.exp(lnKP1) - lnKP2 = (-8814.715 / TempK + 172.1033 - 27.927 * logTempK + - (-160.34 / TempK + 1.3566) * sqrSal + - (0.37335 / TempK - 0.05778) * Sal) + lnKP2 = ( + -8814.715 / TempK + + 172.1033 + - 27.927 * logTempK + + (-160.34 / TempK + 1.3566) * sqrSal + + (0.37335 / TempK - 0.05778) * Sal + ) KP2 = np.exp(lnKP2) - lnKP3 = (-3070.75 / TempK - 18.126 + - (17.27039 / TempK + 2.81197) * sqrSal + - (-44.99486 / TempK - 0.09984) * Sal) + lnKP3 = ( + -3070.75 / TempK + - 18.126 + + (17.27039 / TempK + 2.81197) * sqrSal + + (-44.99486 / TempK - 0.09984) * Sal + ) KP3 = np.exp(lnKP3) - lnKSi = (-8904.2 / TempK + 117.4 - 19.334 * logTempK + - (-458.79 / TempK + 3.5913) * sqrIonS + - (188.74 / TempK - 1.5998) * IonS + - (-12.1652 / TempK + 0.07871) * IonS**2) - KSi = (np.exp(lnKSi) # this is on the SWS pH scale in mol/kg-H2O - * (1.0 - 0.001005 * Sal)) # convert to mol/kg-SW + lnKSi = ( + -8904.2 / TempK + + 117.4 + - 19.334 * logTempK + + (-458.79 / TempK + 3.5913) * sqrIonS + + (188.74 / TempK - 1.5998) * IonS + + (-12.1652 / TempK + 0.07871) * IonS**2 + ) + KSi = np.exp(lnKSi) * ( # this is on the SWS pH scale in mol/kg-H2O + 1.0 - 0.001005 * Sal + ) # convert to mol/kg-SW # Correct constants for pressure pressure_corrections(TempK, Pbar) def pressure_corrections(TempK, Pbar): - """Calculate pressure corrections for constants defined in set_constants - """ + """Calculate pressure corrections for constants defined in set_constants""" # Declare global constants global R_gas, K1, K2, KW, KB, KF, KS, KP1, KP2, KP3, KSi, TB, TS, TF # Temperature and gas constant - RT = R_gas * TempK + RT = R_gas * TempK TempC = TempK - 273.15 # Fugacity Factor Delta = 57.7 - 0.118 * TempK - b = (-1636.75 + 12.0408 * TempK - 0.0327957 * TempK**2 + - 3.16528 * 1.0e-5 * TempK**3) + b = -1636.75 + 12.0408 * TempK - 0.0327957 * TempK**2 + 3.16528 * 1.0e-5 * TempK**3 FugFac = np.exp((b + 2.0 * Delta) * 1.01325 / RT) # Pressure effects on K1 & K2: @@ -247,7 +287,7 @@ def pressure_corrections(TempK, Pbar): # deltaV = -29.48 + 0.1622 * TempC + 0.295 * (Sal - 34.8) # Millero, 1992 # deltaV = -29.48 - 0.1622 * TempC - 0.002608 * TempC**2 # Millero, 1995 # deltaV = deltaV + 0.295 * (Sal - 34.8) # Millero, 1979 - Kappa = -2.84 / 1000.0 # Millero, 1979 + Kappa = -2.84 / 1000.0 # Millero, 1979 # Millero, 1992 and Millero, 1995 also have this. # Kappa = Kappa + 0.354 * (Sal - 34.8) / 1000 # Millero, 1979 # Kappa = (-3.0 + 0.0427 * TempC) / 1000 # Millero, 1983 @@ -256,24 +296,24 @@ def pressure_corrections(TempK, Pbar): # Pressure effects on KF & KS: # These are from Millero, 1995, which is the same as Millero, 1983. # It is assumed that KF and KS are on the free pH scale. - deltaV = -9.78 - 0.009 * TempC - 0.000942 * TempC**2 - Kappa = (-3.91 + 0.054 * TempC) / 1000.0 + deltaV = -9.78 - 0.009 * TempC - 0.000942 * TempC**2 + Kappa = (-3.91 + 0.054 * TempC) / 1000.0 lnKFfac = (-1.0 * deltaV + 0.5 * Kappa * Pbar) * Pbar / RT - deltaV = -18.03 + 0.0466 * TempC + 0.000316 * TempC**2 - Kappa = (-4.53 + 0.09 * TempC) / 1000.0 + deltaV = -18.03 + 0.0466 * TempC + 0.000316 * TempC**2 + Kappa = (-4.53 + 0.09 * TempC) / 1000.0 lnKSfac = (-1.0 * deltaV + 0.5 * Kappa * Pbar) * Pbar / RT # Correct KP1, KP2, & KP3 for pressure: # The corrections for KP1, KP2, and KP3 are from Millero, 1995, # which are the same as Millero, 1983. - deltaV = -14.51 + 0.1211 * TempC - 0.000321 * TempC**2 - Kappa = (-2.67 + 0.0427 * TempC) / 1000.0 + deltaV = -14.51 + 0.1211 * TempC - 0.000321 * TempC**2 + Kappa = (-2.67 + 0.0427 * TempC) / 1000.0 lnKP1fac = (-1.0 * deltaV + 0.5 * Kappa * Pbar) * Pbar / RT - deltaV = -23.12 + 0.1758 * TempC - 0.002647 * TempC**2 - Kappa = (-5.15 + 0.09 * TempC) / 1000.0 + deltaV = -23.12 + 0.1758 * TempC - 0.002647 * TempC**2 + Kappa = (-5.15 + 0.09 * TempC) / 1000.0 lnKP2fac = (-1.0 * deltaV + 0.5 * Kappa * Pbar) * Pbar / RT - deltaV = -26.57 + 0.202 * TempC - 0.003042 * TempC**2 - Kappa = (-4.08 + 0.0714 * TempC) / 1000.0 + deltaV = -26.57 + 0.202 * TempC - 0.003042 * TempC**2 + Kappa = (-4.08 + 0.0714 * TempC) / 1000.0 lnKP3fac = (-1.0 * deltaV + 0.5 * Kappa * Pbar) * Pbar / RT # Pressure effects on KSi: @@ -281,17 +321,17 @@ def pressure_corrections(TempK, Pbar): # values have been estimated from the values of boric acid. HOWEVER, # there is no listing of the values in the table. # I used the values for boric acid from above. - deltaV = -29.48 + 0.1622 * TempC - 0.002608 * TempC**2 - Kappa = -2.84 / 1000.0 + deltaV = -29.48 + 0.1622 * TempC - 0.002608 * TempC**2 + Kappa = -2.84 / 1000.0 lnKSifac = (-1.0 * deltaV + 0.5 * Kappa * Pbar) * Pbar / RT # Correct K's for pressure here: - K1 = K1 * np.exp(lnK1fac) - K2 = K2 * np.exp(lnK2fac) - KW = KW * np.exp(lnKWfac) - KB = KB * np.exp(lnKBfac) - KF = KF * np.exp(lnKFfac) - KS = KS * np.exp(lnKSfac) + K1 = K1 * np.exp(lnK1fac) + K2 = K2 * np.exp(lnK2fac) + KW = KW * np.exp(lnKWfac) + KB = KB * np.exp(lnKBfac) + KF = KF * np.exp(lnKFfac) + KS = KS * np.exp(lnKSfac) KP1 = KP1 * np.exp(lnKP1fac) KP2 = KP2 * np.exp(lnKP2fac) KP3 = KP3 * np.exp(lnKP3fac) @@ -299,7 +339,7 @@ def pressure_corrections(TempK, Pbar): def CalculatepHfromTATC(TA, TC, TP, TSi): - """ SUB CalculatepHfromTATC, version 04.01, 10-13-96, written by Ernie Lewis. + """SUB CalculatepHfromTATC, version 04.01, 10-13-96, written by Ernie Lewis. Inputs: TA, TC, TP, TSi Output: pH This calculates pH from TA and TC using K1 and K2 by Newton's method. @@ -314,36 +354,42 @@ def CalculatepHfromTATC(TA, TC, TP, TSi): global R_gas, K1, K2, KW, KB, KF, KS, KP1, KP2, KP3, KSi, TB, TS, TF # Set iteration parameters - pHGuess = 8.0 # this is the first guess - pHTol = 1.0e-4 # tolerance for iterations end - ln10 = np.log(10.0) - pH = np.ones(TA.shape[0]) * pHGuess # creates a vector holding the first guess for all samples - deltapH = pHTol + 1.0 + pHGuess = 8.0 # this is the first guess + pHTol = 1.0e-4 # tolerance for iterations end + ln10 = np.log(10.0) + pH = ( + np.ones(TA.shape[0]) * pHGuess + ) # creates a vector holding the first guess for all samples + deltapH = pHTol + 1.0 # Begin iteration to find pH while np.any(abs(deltapH) > pHTol): - H = 10.0**(-1.0 * pH) - Denom = (H * H + K1 * H + K1 * K2) - CAlk = TC * K1 * (H + 2.0 * K2) / Denom - BAlk = TB * KB / (KB + H) - OH = KW / H - PhosTop = KP1 * KP2 * H + 2 * KP1 * KP2 * KP3 - H * H * H - PhosBot = H * H * H + KP1 * H * H + KP1 * KP2 * H + KP1 * KP2 * KP3 - PAlk = TP * PhosTop / PhosBot - SiAlk = TSi * KSi / (KSi + H) - FREEtoTOT = (1 + TS / KS) # pH scale conversion factor - Hfree = H / FREEtoTOT # for H on the total scale - HSO4 = TS / (1 + KS / Hfree) # since KS is on the free scale - HF = TF / (1 + KF / Hfree) # since KF is on the free scale - Residual = TA - CAlk - BAlk - OH - PAlk - SiAlk + Hfree + HSO4 + HF - # find Slope dTA/dpH (not exact, but keeps all important terms) - Slope = (ln10 * (TC * K1 * H * (H * H + K1 * K2 + 4.0 * H * K2) - / Denom / Denom + BAlk * H / (KB + H) + OH + H)) - deltapH = Residual / Slope # this is Newton's method - # to keep the jump from being too big - while np.any(abs(deltapH) > 1): - deltapH = deltapH / 2.0 - pH = pH + deltapH # Is on the same scale as K1 and K2 were calculated + H = 10.0 ** (-1.0 * pH) + Denom = H * H + K1 * H + K1 * K2 + CAlk = TC * K1 * (H + 2.0 * K2) / Denom + BAlk = TB * KB / (KB + H) + OH = KW / H + PhosTop = KP1 * KP2 * H + 2 * KP1 * KP2 * KP3 - H * H * H + PhosBot = H * H * H + KP1 * H * H + KP1 * KP2 * H + KP1 * KP2 * KP3 + PAlk = TP * PhosTop / PhosBot + SiAlk = TSi * KSi / (KSi + H) + FREEtoTOT = 1 + TS / KS # pH scale conversion factor + Hfree = H / FREEtoTOT # for H on the total scale + HSO4 = TS / (1 + KS / Hfree) # since KS is on the free scale + HF = TF / (1 + KF / Hfree) # since KF is on the free scale + Residual = TA - CAlk - BAlk - OH - PAlk - SiAlk + Hfree + HSO4 + HF + # find Slope dTA/dpH (not exact, but keeps all important terms) + Slope = ln10 * ( + TC * K1 * H * (H * H + K1 * K2 + 4.0 * H * K2) / Denom / Denom + + BAlk * H / (KB + H) + + OH + + H + ) + deltapH = Residual / Slope # this is Newton's method + # to keep the jump from being too big + while np.any(abs(deltapH) > 1): + deltapH = deltapH / 2.0 + pH = pH + deltapH # Is on the same scale as K1 and K2 were calculated return pH @@ -381,9 +427,9 @@ def ca_solubility(S, TempK, P, DIC, pH): global R_gas, K1, K2, KW, KB, KF, KS, KP1, KP2, KP3, KSi, TB, TS, TF # Precalculate quantities - TempC = TempK - 273.15 + TempC = TempK - 273.15 logTempK = np.log(TempK) - sqrtS = np.sqrt(S) + sqrtS = np.sqrt(S) # Calculate Ca^2+: # Riley, J. P. and Tongudai, M., Chemical Geology 2:263-269, 1967: @@ -392,24 +438,34 @@ def ca_solubility(S, TempK, P, DIC, pH): # Calcite solubility: # Mucci, Alphonso, Amer. J. of Science 283:781-799, 1983. - KCa = (10.0**(-171.9065 - 0.077993 * TempK + 2839.319 / TempK - + 71.595 * logTempK / np.log(10.0) - + (-0.77712 + 0.0028426 * TempK + 178.34 / TempK) * sqrtS - - 0.07711 * S + 0.0041249 * sqrtS * S)) + KCa = 10.0 ** ( + -171.9065 + - 0.077993 * TempK + + 2839.319 / TempK + + 71.595 * logTempK / np.log(10.0) + + (-0.77712 + 0.0028426 * TempK + 178.34 / TempK) * sqrtS + - 0.07711 * S + + 0.0041249 * sqrtS * S + ) # Aragonite solubility: # Mucci, Alphonso, Amer. J. of Science 283:781-799, 1983. - KAr = (10.0**(-171.945 - 0.077993 * TempK + 2903.293 / TempK - + 71.595 * logTempK / np.log(10.0) - + (-0.068393 + 0.0017276 * TempK + 88.135 / TempK) * sqrtS - - 0.10018 * S + 0.0059415 * sqrtS * S)) + KAr = 10.0 ** ( + -171.945 + - 0.077993 * TempK + + 2903.293 / TempK + + 71.595 * logTempK / np.log(10.0) + + (-0.068393 + 0.0017276 * TempK + 88.135 / TempK) * sqrtS + - 0.10018 * S + + 0.0059415 * sqrtS * S + ) # Pressure correction for calcite: # Ingle, Marine Chemistry 3:301-319, 1975 # same as in Millero, GCA 43:1651-1661, 1979, but Millero, GCA 1995 # has typos (-0.5304, -0.3692, and 10^3 for Kappa factor) deltaV_KCa = -48.76 + 0.5304 * TempC - Kappa_KCa = (-11.76 + 0.3692 * TempC) / 1000.0 + Kappa_KCa = (-11.76 + 0.3692 * TempC) / 1000.0 KCa = KCa * np.exp((-deltaV_KCa + 0.5 * Kappa_KCa * P) * P / (R_gas * TempK)) # Pressure correction for aragonite: @@ -417,11 +473,11 @@ def ca_solubility(S, TempK, P, DIC, pH): # same as Millero, GCA 1995 except for typos (-0.5304, -0.3692, # and 10^3 for Kappa factor) deltaV_KAr = deltaV_KCa + 2.8 - Kappa_KAr = Kappa_KCa + Kappa_KAr = Kappa_KCa KAr = KAr * np.exp((-deltaV_KAr + 0.5 * Kappa_KAr * P) * P / (R_gas * TempK)) # Calculate Omegas: - H = 10.0**(-pH) + H = 10.0 ** (-pH) CO3 = DIC * K1 * K2 / (K1 * H + H * H + K1 * K2) Omega_C = CO3 * Ca / KCa Omega_A = CO3 * Ca / KAr @@ -429,8 +485,7 @@ def ca_solubility(S, TempK, P, DIC, pH): def calc_rho(Sal, TempK, P): - """ Calculate rho: Based on SOG code - """ + """Calculate rho: Based on SOG code""" # Convert the temperature to Celsius TempC = TempK - 273.15 @@ -441,10 +496,17 @@ def calc_rho(Sal, TempK, P): # Calculate the density profile at the grid point depths # Pure water density at atmospheric pressure # (Bigg P.H., (1967) Br. J. Applied Physics 8 pp 521-537) - R1 = ((((6.536332e-9 * TempC - 1.120083e-6) * TempC + 1.001685e-4) - * TempC - 9.095290e-3) * TempC + 6.793952e-2) * TempC - 28.263737 - R2 = (((5.3875e-9 * TempC - 8.2467e-7) * TempC + 7.6438e-5) - * TempC - 4.0899e-3) * TempC + 8.24493e-1 + R1 = ( + ( + ((6.536332e-9 * TempC - 1.120083e-6) * TempC + 1.001685e-4) * TempC + - 9.095290e-3 + ) + * TempC + + 6.793952e-2 + ) * TempC - 28.263737 + R2 = ( + ((5.3875e-9 * TempC - 8.2467e-7) * TempC + 7.6438e-5) * TempC - 4.0899e-3 + ) * TempC + 8.24493e-1 R3 = (-1.6546e-6 * TempC + 1.0227e-4) * TempC - 5.72466e-3 # International one-atmosphere equation of state of seawater @@ -452,7 +514,7 @@ def calc_rho(Sal, TempK, P): # Specific volume at atmospheric pressure V350P = 1.0 / 1028.1063 - SVA = -SIG * V350P / (1028.1063 + SIG) + SVA = -SIG * V350P / (1028.1063 + SIG) # Density anomoly at atmospheric pressure sigma_t = 28.106331 - SVA / (V350P * (V350P + SVA)) diff --git a/bloomcast/ensemble.py b/bloomcast/ensemble.py index 32c5275..8aedfff 100644 --- a/bloomcast/ensemble.py +++ b/bloomcast/ensemble.py @@ -37,53 +37,53 @@ ) -__all__ = ['Ensemble'] +__all__ = ["Ensemble"] # Colours for plots COLORS = { - 'axes': '#ebebeb', # bootswatch superhero theme text - 'bg': '#2B3E50', # bootswatch superhero theme background - 'diatoms': '#7CC643', - 'nitrate': '#82AFDC', - 'temperature': '#D83F83', - 'salinity': '#82DCDC', - 'temperature_lines': { - 'early': '#F00C27', - 'median': '#D83F83', - 'late': '#BD9122', + "axes": "#ebebeb", # bootswatch superhero theme text + "bg": "#2B3E50", # bootswatch superhero theme background + "diatoms": "#7CC643", + "nitrate": "#82AFDC", + "temperature": "#D83F83", + "salinity": "#82DCDC", + "temperature_lines": { + "early": "#F00C27", + "median": "#D83F83", + "late": "#BD9122", }, - 'salinity_lines': { - 'early': '#0EB256', - 'median': '#82DCDC', - 'late': '#224EBD', + "salinity_lines": { + "early": "#0EB256", + "median": "#82DCDC", + "late": "#224EBD", }, - 'mld': '#df691a', - 'wind_speed': '#ebebeb', + "mld": "#df691a", + "wind_speed": "#ebebeb", } class Ensemble(cliff.command.Command): - """run the ensemble bloomcast - """ - log = logging.getLogger('bloomcast.ensemble') - bloom_date_log = logging.getLogger('bloomcast.ensemble.bloom_date') + """run the ensemble bloomcast""" + + log = logging.getLogger("bloomcast.ensemble") + bloom_date_log = logging.getLogger("bloomcast.ensemble.bloom_date") def get_parser(self, prog_name): parser = super().get_parser(prog_name) - parser.description = ''' + parser.description = """ Run an ensemble forecast to predict the first spring diatom phytoplankton bloom in the Strait of Georgia. - ''' + """ parser.add_argument( - 'config_file', - help='path and name of configuration file', + "config_file", + help="path and name of configuration file", ) parser.add_argument( - '--data-date', + "--data-date", type=arrow.get, - help='data date for development and debugging; overridden if ' - 'wind forcing data is collected and processed', + help="data date for development and debugging; overridden if " + "wind forcing data is collected and processed", ) return parser @@ -96,31 +96,40 @@ def take_action(self, parsed_args): self.config.data_date = parsed_args.data_date if not self.config.get_forcing_data and self.config.data_date is None: self.log.debug( - 'This will not end well: ' - 'get_forcing_data={0.get_forcing_data} ' - 'and data_date={0.data_date}'.format(self.config)) + "This will not end well: " + "get_forcing_data={0.get_forcing_data} " + "and data_date={0.data_date}".format(self.config) + ) return - self.log.debug('run start date/time is {0:%Y-%m-%d %H:%M:%S}' - .format(self.config.run_start_date)) + self.log.debug( + "run start date/time is {0:%Y-%m-%d %H:%M:%S}".format( + self.config.run_start_date + ) + ) # Check run start date and current date to ensure that # river flow data are available. # River flow data are only available in a rolling 18-month window. - run_start_yr_jan1 = ( - arrow.get(self.config.run_start_date).replace(month=1, day=1)) + run_start_yr_jan1 = arrow.get(self.config.run_start_date).replace( + month=1, day=1 + ) river_date_limit = arrow.now().shift(months=-18) if run_start_yr_jan1 < river_date_limit: self.log.error( - 'A bloomcast run starting {0.run_start_date:%Y-%m-%d} cannot ' - 'be done today because there are no river flow data available ' - 'prior to {1}' - .format(self.config, river_date_limit.format('YYYY-MM-DD'))) + "A bloomcast run starting {0.run_start_date:%Y-%m-%d} cannot " + "be done today because there are no river flow data available " + "prior to {1}".format( + self.config, river_date_limit.format("YYYY-MM-DD") + ) + ) return try: get_forcing_data(self.config, self.log) except ValueError: self.log.info( - 'Wind data date {} is unchanged since last run' - .format(self.config.data_date.format('YYYY-MM-DD'))) + "Wind data date {} is unchanged since last run".format( + self.config.data_date.format("YYYY-MM-DD") + ) + ) return self._create_infile_edits() self._create_batch_description() @@ -130,211 +139,218 @@ def take_action(self, parsed_args): prediction, bloom_dates = self._calc_bloom_dates() self._load_physics_timeseries(prediction) timeseries_plots = self._create_timeseries_graphs( - COLORS, prediction, bloom_dates) + COLORS, prediction, bloom_dates + ) self._load_profiles(prediction) profile_plots = self._create_profile_graphs(COLORS) - self.render_results( - prediction, bloom_dates, timeseries_plots, profile_plots) + self.render_results(prediction, bloom_dates, timeseries_plots, profile_plots) def _create_infile_edits(self): - """Create YAML infile edit files for each ensemble member SOG run. - """ + """Create YAML infile edit files for each ensemble member SOG run.""" ensemble_config = self.config.ensemble start_year = ensemble_config.start_year end_year = ensemble_config.end_year + 1 forcing_data_file_roots = ensemble_config.forcing_data_file_roots forcing_data_key_pairs = ( - ('wind', 'avg_historical_wind_file'), - ('air_temperature', 'avg_historical_air_temperature_file'), - ('cloud_fraction', 'avg_historical_cloud_file'), - ('relative_humidity', 'avg_historical_humidity_file'), - ('major_river', 'avg_historical_major_river_file'), - ('minor_river', 'avg_historical_minor_river_file'), + ("wind", "avg_historical_wind_file"), + ("air_temperature", "avg_historical_air_temperature_file"), + ("cloud_fraction", "avg_historical_cloud_file"), + ("relative_humidity", "avg_historical_humidity_file"), + ("major_river", "avg_historical_major_river_file"), + ("minor_river", "avg_historical_minor_river_file"), ) timeseries_key_pairs = ( - ('std_phys_ts_outfile', 'std_physics'), - ('user_phys_ts_outfile', 'user_physics'), - ('std_bio_ts_outfile', 'std_biology'), - ('user_bio_ts_outfile', 'user_biology'), - ('std_chem_ts_outfile', 'std_chemistry'), - ('user_chem_ts_outfile', 'user_chemistry'), + ("std_phys_ts_outfile", "std_physics"), + ("user_phys_ts_outfile", "user_physics"), + ("std_bio_ts_outfile", "std_biology"), + ("user_bio_ts_outfile", "user_biology"), + ("std_chem_ts_outfile", "std_chemistry"), + ("user_chem_ts_outfile", "user_chemistry"), ) profiles_key_pairs = ( - ('profiles_outfile_base', 'profile_file_base'), - ('user_profiles_outfile_base', 'user_profile_file_base'), - ('halocline_outfile', 'halocline_file'), - ('Hoffmueller_profiles_outfile', 'hoffmueller_file'), - ('user_Hoffmueller_profiles_outfile', 'user_hoffmueller_file'), + ("profiles_outfile_base", "profile_file_base"), + ("user_profiles_outfile_base", "user_profile_file_base"), + ("halocline_outfile", "halocline_file"), + ("Hoffmueller_profiles_outfile", "hoffmueller_file"), + ("user_Hoffmueller_profiles_outfile", "user_hoffmueller_file"), ) self.edit_files = [] for year in range(start_year, end_year): suffix = two_yr_suffix(year) member_infile_edits = infile_edits_template.copy() - forcing_data = member_infile_edits['forcing_data'] - timeseries_results = member_infile_edits['timeseries_results'] - profiles_results = member_infile_edits['profiles_results'] + forcing_data = member_infile_edits["forcing_data"] + timeseries_results = member_infile_edits["timeseries_results"] + profiles_results = member_infile_edits["profiles_results"] for config_key, infile_key in forcing_data_key_pairs: - filename = ''.join( - (forcing_data_file_roots[config_key], suffix)) - forcing_data[infile_key]['value'] = filename + filename = "".join((forcing_data_file_roots[config_key], suffix)) + forcing_data[infile_key]["value"] = filename for config_key, infile_key in timeseries_key_pairs: - filename = ''.join((getattr(self.config, config_key), suffix)) - timeseries_results[infile_key]['value'] = filename + filename = "".join((getattr(self.config, config_key), suffix)) + timeseries_results[infile_key]["value"] = filename for config_key, infile_key in profiles_key_pairs: - filename = ''.join((getattr(self.config, config_key), suffix)) - profiles_results[infile_key]['value'] = filename + filename = "".join((getattr(self.config, config_key), suffix)) + profiles_results[infile_key]["value"] = filename name, ext = os.path.splitext(ensemble_config.base_infile) - filename = ''.join((name, suffix, ext)) - with open(filename, 'wt') as f: + filename = "".join((name, suffix, ext)) + with open(filename, "wt") as f: yaml.safe_dump(member_infile_edits, f) self.edit_files.append((year, filename, suffix)) - self.log.debug('wrote infile edit file {}'.format(filename)) + self.log.debug("wrote infile edit file {}".format(filename)) def _create_batch_description(self): - """Create the YAML batch description file for the ensemble runs. - """ + """Create the YAML batch description file for the ensemble runs.""" batch_description = { - 'max_concurrent_jobs': self.config.ensemble.max_concurrent_jobs, - 'SOG_executable': self.config.SOG_executable, - 'base_infile': self.config.ensemble.base_infile, - 'jobs': [], + "max_concurrent_jobs": self.config.ensemble.max_concurrent_jobs, + "SOG_executable": self.config.SOG_executable, + "base_infile": self.config.ensemble.base_infile, + "jobs": [], } for year, edit_file, suffix in self.edit_files: job = { - ''.join(('bloomcast', suffix)): { - 'edit_files': [edit_file], + "".join(("bloomcast", suffix)): { + "edit_files": [edit_file], } } - batch_description['jobs'].append(job) - filename = 'bloomcast_ensemble_jobs.yaml' - with open(filename, 'wt') as f: + batch_description["jobs"].append(job) + filename = "bloomcast_ensemble_jobs.yaml" + with open(filename, "wt") as f: yaml.safe_dump(batch_description, f) - self.log.debug( - 'wrote ensemble batch description file: {}'.format(filename)) + self.log.debug("wrote ensemble batch description file: {}".format(filename)) def _run_SOG_batch(self): - """Run the ensemble of SOG runs at a batch job. - """ + """Run the ensemble of SOG runs at a batch job.""" if not self.config.run_SOG: - self.log.info('Skipped running SOG') + self.log.info("Skipped running SOG") return - returncode = SOGcommand.api.batch('bloomcast_ensemble_jobs.yaml') + returncode = SOGcommand.api.batch("bloomcast_ensemble_jobs.yaml") self.log.info( - 'ensemble batch SOG runs completed with return code {}' - .format(returncode)) + "ensemble batch SOG runs completed with return code {}".format(returncode) + ) def _load_biology_timeseries(self): - """Load biological timeseries results from all ensemble SOG runs. - """ + """Load biological timeseries results from all ensemble SOG runs.""" self.nitrate_ts, self.diatoms_ts = {}, {} for member, edit_file, suffix in self.edit_files: - filename = ''.join((self.config.std_bio_ts_outfile, suffix)) + filename = "".join((self.config.std_bio_ts_outfile, suffix)) self.nitrate_ts[member] = utils.SOG_Timeseries(filename) - self.nitrate_ts[member].read_data( - 'time', '3 m avg nitrate concentration') + self.nitrate_ts[member].read_data("time", "3 m avg nitrate concentration") self.nitrate_ts[member].calc_mpl_dates(self.config.run_start_date) self.diatoms_ts[member] = utils.SOG_Timeseries(filename) self.diatoms_ts[member].read_data( - 'time', '3 m avg micro phytoplankton biomass') + "time", "3 m avg micro phytoplankton biomass" + ) self.diatoms_ts[member].calc_mpl_dates(self.config.run_start_date) - self.log.debug( - 'read nitrate & diatoms timeseries from {}'.format(filename)) + self.log.debug("read nitrate & diatoms timeseries from {}".format(filename)) self.nitrate = copy.deepcopy(self.nitrate_ts) self.diatoms = copy.deepcopy(self.diatoms_ts) def _load_chemistry_timeseries(self): - """Load carbon chemistry timeseries results from all ensemble SOG runs. - """ + """Load carbon chemistry timeseries results from all ensemble SOG runs.""" self.DIC_ts, self.alkalinity_ts = {}, {} for member, edit_file, suffix in self.edit_files: - filename = ''.join((self.config.std_chem_ts_outfile, suffix)) + filename = "".join((self.config.std_chem_ts_outfile, suffix)) self.DIC_ts[member] = utils.SOG_Timeseries(filename) - self.DIC_ts[member].read_data( - 'time', '3 m avg DIC concentration') + self.DIC_ts[member].read_data("time", "3 m avg DIC concentration") self.DIC_ts[member].calc_mpl_dates(self.config.run_start_date) self.alkalinity_ts[member] = utils.SOG_Timeseries(filename) - self.alkalinity_ts[member].read_data( - 'time', '3 m avg alkalinity') - self.alkalinity_ts[member].calc_mpl_dates( - self.config.run_start_date) - self.log.debug( - 'read DIC & alkalinity timeseries from {}'.format(filename)) + self.alkalinity_ts[member].read_data("time", "3 m avg alkalinity") + self.alkalinity_ts[member].calc_mpl_dates(self.config.run_start_date) + self.log.debug("read DIC & alkalinity timeseries from {}".format(filename)) self.DIC = copy.deepcopy(self.DIC_ts) self.alkalinity = copy.deepcopy(self.alkalinity_ts) def _calc_bloom_dates(self): - """Calculate the predicted spring bloom date. - """ + """Calculate the predicted spring bloom date.""" run_start_date = self.config.run_start_date - bloomcast.clip_results_to_jan1( - self.nitrate, self.diatoms, run_start_date) + bloomcast.clip_results_to_jan1(self.nitrate, self.diatoms, run_start_date) bloomcast.reduce_results_to_daily( - self.nitrate, self.diatoms, run_start_date, - self.config.SOG_timestep) + self.nitrate, self.diatoms, run_start_date, self.config.SOG_timestep + ) first_low_nitrate_days = bloomcast.find_low_nitrate_days( - self.nitrate, bloomcast.NITRATE_HALF_SATURATION_CONCENTRATION) + self.nitrate, bloomcast.NITRATE_HALF_SATURATION_CONCENTRATION + ) bloom_dates, bloom_biomasses = bloomcast.find_phytoplankton_peak( - self.diatoms, first_low_nitrate_days, - bloomcast.PHYTOPLANKTON_PEAK_WINDOW_HALF_WIDTH) + self.diatoms, + first_low_nitrate_days, + bloomcast.PHYTOPLANKTON_PEAK_WINDOW_HALF_WIDTH, + ) ord_days = np.array( - [bloom_date.toordinal() for bloom_date in bloom_dates.values()]) + [bloom_date.toordinal() for bloom_date in bloom_dates.values()] + ) median = np.rint(np.median(ord_days)) early_bound, late_bound = np.percentile(ord_days, (5, 95)) - prediction = OrderedDict([ - ('early', find_member(bloom_dates, np.trunc(early_bound))), - ('median', find_member(bloom_dates, median)), - ('late', find_member(bloom_dates, np.ceil(late_bound))), - ]) + prediction = OrderedDict( + [ + ("early", find_member(bloom_dates, np.trunc(early_bound))), + ("median", find_member(bloom_dates, median)), + ("late", find_member(bloom_dates, np.ceil(late_bound))), + ] + ) min_bound, max_bound = np.percentile(ord_days, (0, 100)) - extremes = OrderedDict([ - ('min', find_member(bloom_dates, np.trunc(min_bound))), - ('max', find_member(bloom_dates, np.ceil(max_bound))), - ]) + extremes = OrderedDict( + [ + ("min", find_member(bloom_dates, np.trunc(min_bound))), + ("max", find_member(bloom_dates, np.ceil(max_bound))), + ] + ) self.log.info( - 'Predicted earliest bloom date is {}' - .format(bloom_dates[extremes['min']])) + "Predicted earliest bloom date is {}".format(bloom_dates[extremes["min"]]) + ) self.log.info( - 'Earliest bloom date is based on forcing from {}/{}' - .format(extremes['min'] - 1, extremes['min'])) + "Earliest bloom date is based on forcing from {}/{}".format( + extremes["min"] - 1, extremes["min"] + ) + ) self.log.info( - 'Predicted early bound bloom date is {}' - .format(bloom_dates[prediction['early']])) + "Predicted early bound bloom date is {}".format( + bloom_dates[prediction["early"]] + ) + ) self.log.info( - 'Early bound bloom date is based on forcing from {}/{}' - .format(prediction['early'] - 1, prediction['early'])) + "Early bound bloom date is based on forcing from {}/{}".format( + prediction["early"] - 1, prediction["early"] + ) + ) self.log.info( - 'Predicted median bloom date is {}' - .format(bloom_dates[prediction['median']])) + "Predicted median bloom date is {}".format( + bloom_dates[prediction["median"]] + ) + ) self.log.info( - 'Median bloom date is based on forcing from {}/{}' - .format(prediction['median'] - 1, prediction['median'])) + "Median bloom date is based on forcing from {}/{}".format( + prediction["median"] - 1, prediction["median"] + ) + ) self.log.info( - 'Predicted late bound bloom date is {}' - .format(bloom_dates[prediction['late']])) + "Predicted late bound bloom date is {}".format( + bloom_dates[prediction["late"]] + ) + ) self.log.info( - 'Late bound bloom date is based on forcing from {}/{}' - .format(prediction['late'] - 1, prediction['late'])) + "Late bound bloom date is based on forcing from {}/{}".format( + prediction["late"] - 1, prediction["late"] + ) + ) self.log.info( - 'Predicted latest bloom date is {}' - .format(bloom_dates[extremes['max']])) + "Predicted latest bloom date is {}".format(bloom_dates[extremes["max"]]) + ) self.log.info( - 'Latest bloom date is based on forcing from {}/{}' - .format(extremes['max'] - 1, extremes['max'])) - line = ( - ' {data_date}' - .format(data_date=self.config.data_date.format('YYYY-MM-DD'))) - for member in 'median early late'.split(): - line += ( - ' {bloom_date} {forcing_year}' - .format( - bloom_date=bloom_dates[prediction[member]], - forcing_year=prediction[member])) + "Latest bloom date is based on forcing from {}/{}".format( + extremes["max"] - 1, extremes["max"] + ) + ) + line = " {data_date}".format( + data_date=self.config.data_date.format("YYYY-MM-DD") + ) + for member in "median early late".split(): + line += " {bloom_date} {forcing_year}".format( + bloom_date=bloom_dates[prediction[member]], + forcing_year=prediction[member], + ) for member in extremes.values(): - line += ( - ' {bloom_date} {forcing_year}' - .format( - bloom_date=bloom_dates[member], - forcing_year=member)) + line += " {bloom_date} {forcing_year}".format( + bloom_date=bloom_dates[member], forcing_year=member + ) self.bloom_date_log.info(line) return prediction, bloom_dates @@ -349,64 +365,58 @@ def _load_physics_timeseries(self, prediction): self.mixing_layer_depth = {} for member in prediction.values(): suffix = two_yr_suffix(member) - filename = ''.join((self.config.std_phys_ts_outfile, suffix)) + filename = "".join((self.config.std_phys_ts_outfile, suffix)) self.temperature[member] = utils.SOG_Timeseries(filename) - self.temperature[member].read_data('time', '3 m avg temperature') + self.temperature[member].read_data("time", "3 m avg temperature") self.temperature[member].calc_mpl_dates(self.config.run_start_date) self.salinity[member] = utils.SOG_Timeseries(filename) - self.salinity[member].read_data('time', '3 m avg salinity') + self.salinity[member].read_data("time", "3 m avg salinity") self.salinity[member].calc_mpl_dates(self.config.run_start_date) self.log.debug( - 'read temperature and salinity timeseries from {}' - .format(filename)) - suffix = two_yr_suffix(prediction['median']) - filename = ''.join((self.config.std_phys_ts_outfile, suffix)) + "read temperature and salinity timeseries from {}".format(filename) + ) + suffix = two_yr_suffix(prediction["median"]) + filename = "".join((self.config.std_phys_ts_outfile, suffix)) self.mixing_layer_depth = utils.SOG_Timeseries(filename) - self.mixing_layer_depth.read_data( - 'time', 'mixing layer depth') - self.mixing_layer_depth.calc_mpl_dates( - self.config.run_start_date) - self.log.debug( - 'read mixing layer depth timeseries from {}'.format(filename)) - filename = 'Sandheads_wind' + self.mixing_layer_depth.read_data("time", "mixing layer depth") + self.mixing_layer_depth.calc_mpl_dates(self.config.run_start_date) + self.log.debug("read mixing layer depth timeseries from {}".format(filename)) + filename = "Sandheads_wind" self.wind = wind.WindTimeseries(filename) self.wind.read_data(self.config.run_start_date) self.wind.calc_mpl_dates(self.config.run_start_date) - self.log.debug( - 'read wind speed forcing timeseries from {}'.format(filename)) + self.log.debug("read wind speed forcing timeseries from {}".format(filename)) def _create_timeseries_graphs(self, colors, prediction, bloom_dates): - """Create time series plot figure objects. - """ + """Create time series plot figure objects.""" timeseries_plots = { - 'nitrate_diatoms': visualization.nitrate_diatoms_timeseries( + "nitrate_diatoms": visualization.nitrate_diatoms_timeseries( self.nitrate_ts, self.diatoms_ts, colors, self.config.data_date, prediction, bloom_dates, - titles=('3 m Avg Nitrate Concentration [µM N]', - '3 m Avg Diatom Biomass [µM N]'), - ), - 'temperature_salinity': - visualization.temperature_salinity_timeseries( - self.temperature, - self.salinity, - colors, - self.config.data_date, - prediction, - bloom_dates, - titles=('3 m Avg Temperature [°C]', - '3 m Avg Salinity [-]'), + titles=( + "3 m Avg Nitrate Concentration [µM N]", + "3 m Avg Diatom Biomass [µM N]", ), - 'mld_wind': visualization.mixing_layer_depth_wind_timeseries( + ), + "temperature_salinity": visualization.temperature_salinity_timeseries( + self.temperature, + self.salinity, + colors, + self.config.data_date, + prediction, + bloom_dates, + titles=("3 m Avg Temperature [°C]", "3 m Avg Salinity [-]"), + ), + "mld_wind": visualization.mixing_layer_depth_wind_timeseries( self.mixing_layer_depth, self.wind, colors, self.config.data_date, - titles=('Mixing Layer Depth [m]', - 'Wind Speed [m/s]'), + titles=("Mixing Layer Depth [m]", "Wind Speed [m/s]"), ), } return timeseries_plots @@ -422,103 +432,114 @@ def _load_profiles(self, prediction): """ self.temperature_profile, self.salinity_profile = {}, {} self.diatoms_profile, self.nitrate_profile = {}, {} - member = prediction['median'] + member = prediction["median"] suffix = two_yr_suffix(member) - filename = ''.join( - (self.config.Hoffmueller_profiles_outfile, suffix)) + filename = "".join((self.config.Hoffmueller_profiles_outfile, suffix)) profile_number = ( - self.config.data_date.date() - - self.config.run_start_date.date()).days - 1 - self.log.debug('use profile number {}'.format(profile_number)) + self.config.data_date.date() - self.config.run_start_date.date() + ).days - 1 + self.log.debug("use profile number {}".format(profile_number)) self.temperature_profile = utils.SOG_HoffmuellerProfile(filename) - self.temperature_profile.read_data( - 'depth', 'temperature', profile_number) - self.log.debug('read temperature profile from {}'.format(filename)) + self.temperature_profile.read_data("depth", "temperature", profile_number) + self.log.debug("read temperature profile from {}".format(filename)) self.salinity_profile = utils.SOG_HoffmuellerProfile(filename) - self.salinity_profile.read_data( - 'depth', 'salinity', profile_number) - self.log.debug('read salinity profile from {}'.format(filename)) + self.salinity_profile.read_data("depth", "salinity", profile_number) + self.log.debug("read salinity profile from {}".format(filename)) self.diatoms_profile = utils.SOG_HoffmuellerProfile(filename) - self.diatoms_profile.read_data( - 'depth', 'micro phytoplankton', profile_number) - self.log.debug('read diatom biomass profile from {}'.format(filename)) + self.diatoms_profile.read_data("depth", "micro phytoplankton", profile_number) + self.log.debug("read diatom biomass profile from {}".format(filename)) self.nitrate_profile = utils.SOG_HoffmuellerProfile(filename) - self.nitrate_profile.read_data( - 'depth', 'nitrate', profile_number) - self.log.debug( - 'read nitrate concentration profile from {}'.format(filename)) + self.nitrate_profile.read_data("depth", "nitrate", profile_number) + self.log.debug("read nitrate concentration profile from {}".format(filename)) def _create_profile_graphs(self, colors): - """Create profile plot figure objects. - """ + """Create profile plot figure objects.""" profile_datetime = self.config.data_date.replace(hour=12) profile_dt = profile_datetime.naive - self.config.run_start_date profile_hour = profile_dt.days * 24 + profile_dt.seconds / 3600 self.mixing_layer_depth.boolean_slice( - self.mixing_layer_depth.indep_data >= profile_hour) + self.mixing_layer_depth.indep_data >= profile_hour + ) profile_plots = visualization.profiles( profiles=( - self.temperature_profile, self.salinity_profile, - self.diatoms_profile, self.nitrate_profile, + self.temperature_profile, + self.salinity_profile, + self.diatoms_profile, + self.nitrate_profile, ), titles=( - 'Temperature [°C]', 'Salinity [-]', - 'Diatom Biomass [µM N]', 'Nitrate Concentration [µM N]', + "Temperature [°C]", + "Salinity [-]", + "Diatom Biomass [µM N]", + "Nitrate Concentration [µM N]", ), limits=((4, 10), (16, 32), None, (0, 32)), mixing_layer_depth=self.mixing_layer_depth.dep_data[0], label_colors=( - 'temperature', 'salinity', 'diatoms', 'nitrate', 'mld', + "temperature", + "salinity", + "diatoms", + "nitrate", + "mld", ), colors=colors, ) return profile_plots def render_results( - self, prediction, bloom_dates, timeseries_plots, profile_plots, + self, + prediction, + bloom_dates, + timeseries_plots, + profile_plots, ): - """Render bloomcast results and plots to files. - """ + """Render bloomcast results and plots to files.""" ts_plot_files = {} for key, fig in timeseries_plots.items(): - filename = '{}_timeseries.svg'.format(key) - visualization.save_image( - fig, filename, facecolor=fig.get_facecolor()) + filename = "{}_timeseries.svg".format(key) + visualization.save_image(fig, filename, facecolor=fig.get_facecolor()) ts_plot_files[key] = filename - self.log.debug( - 'saved {} time series figure as {}'.format(key, filename)) + self.log.debug("saved {} time series figure as {}".format(key, filename)) visualization.save_image( - profile_plots, 'profiles.svg', facecolor=fig.get_facecolor()) - self.log.debug('saved profiles figure as profiles.svg') + profile_plots, "profiles.svg", facecolor=fig.get_facecolor() + ) + self.log.debug("saved profiles figure as profiles.svg") if self.config.results.push_to_web: results_path = self.config.results.path latest_bloomcast = { - 'run_start_date': self.config.run_start_date.strftime( - '%Y-%m-%d'), - 'data_date': self.config.data_date.format('YYYY-MM-DD'), - 'prediction': dict(prediction), - 'bloom_dates': { - data_year: date.strftime('%Y-%m-%d') - for data_year, date in bloom_dates.items()}, - 'ts_plot_files': ts_plot_files, - 'profiles_plot_file': 'profiles.svg', + "run_start_date": self.config.run_start_date.strftime("%Y-%m-%d"), + "data_date": self.config.data_date.format("YYYY-MM-DD"), + "prediction": dict(prediction), + "bloom_dates": { + data_year: date.strftime("%Y-%m-%d") + for data_year, date in bloom_dates.items() + }, + "ts_plot_files": ts_plot_files, + "profiles_plot_file": "profiles.svg", } - with (results_path/'latest_bloomcast.yaml').open('wt') as f: + with (results_path / "latest_bloomcast.yaml").open("wt") as f: yaml.safe_dump(latest_bloomcast, f) - self.log.debug('saved most bloomcast results to {}'.format( - results_path / 'latest_bloomcast.yaml')) + self.log.debug( + "saved most bloomcast results to {}".format( + results_path / "latest_bloomcast.yaml" + ) + ) for plot_file in ts_plot_files.values(): shutil.copy2(plot_file, str(results_path)) - self.log.debug('copied {} to {}'.format( - plot_file, results_path/plot_file)) - shutil.copy2('profiles.svg', str(results_path)) - self.log.debug('copied profiles.svg to {}'.format( - results_path/'profiles.svg')) - shutil.copy2( - self.config.logging.bloom_date_log_filename, str(results_path)) - self.log.debug('copied {} to {}'.format( - self.config.logging.bloom_date_log_filename, - results_path/self.config.logging.bloom_date_log_filename)) + self.log.debug( + "copied {} to {}".format(plot_file, results_path / plot_file) + ) + shutil.copy2("profiles.svg", str(results_path)) + self.log.debug( + "copied profiles.svg to {}".format(results_path / "profiles.svg") + ) + shutil.copy2(self.config.logging.bloom_date_log_filename, str(results_path)) + self.log.debug( + "copied {} to {}".format( + self.config.logging.bloom_date_log_filename, + results_path / self.config.logging.bloom_date_log_filename, + ) + ) def configure_logging(config, bloom_date_log): @@ -528,66 +549,73 @@ def configure_logging(config, bloom_date_log): Debug logging on/off & email recipient(s) for warning messages are set in config file. """ - root_logger = logging.getLogger('') + root_logger = logging.getLogger("") console_handler = root_logger.handlers[0] def patched_data_filter(record): - if (record.funcName == 'patch_data' - and 'data patched' in record.msg): + if record.funcName == "patch_data" and "data patched" in record.msg: return 0 return 1 + console_handler.addFilter(patched_data_filter) def requests_info_debug_filter(record): - if (record.name.startswith('requests.') - and record.levelname in {'INFO', 'DEBUG'}): + if record.name.startswith("requests.") and record.levelname in { + "INFO", + "DEBUG", + }: return 0 return 1 + console_handler.addFilter(requests_info_debug_filter) disk = logging.handlers.RotatingFileHandler( - config.logging.bloomcast_log_filename, maxBytes=1024 * 1024) + config.logging.bloomcast_log_filename, maxBytes=1024 * 1024 + ) disk.setFormatter( logging.Formatter( - '%(asctime)s %(levelname)s [%(name)s] %(message)s', - datefmt='%Y-%m-%d %H:%M')) + "%(asctime)s %(levelname)s [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M" + ) + ) disk.setLevel(logging.DEBUG) disk.addFilter(requests_info_debug_filter) root_logger.addHandler(disk) - mailhost = (('localhost', 1025) if config.logging.use_test_smtpd - else 'smtp.eos.ubc.ca') + mailhost = ( + ("localhost", 1025) if config.logging.use_test_smtpd else "smtp.eos.ubc.ca" + ) email = logging.handlers.SMTPHandler( - mailhost, fromaddr='SoG-bloomcast@eos.ubc.ca', + mailhost, + fromaddr="SoG-bloomcast@eos.ubc.ca", toaddrs=config.logging.toaddrs, - subject='Warning Message from SoG-bloomcast', + subject="Warning Message from SoG-bloomcast", timeout=10.0, ) - email.setFormatter( - logging.Formatter('%(levelname)s:%(name)s:%(message)s')) + email.setFormatter(logging.Formatter("%(levelname)s:%(name)s:%(message)s")) email.setLevel(logging.WARNING) root_logger.addHandler(email) - bloom_date_evolution = logging.FileHandler( - config.logging.bloom_date_log_filename) - bloom_date_evolution.setFormatter(logging.Formatter('%(message)s')) + bloom_date_evolution = logging.FileHandler(config.logging.bloom_date_log_filename) + bloom_date_evolution.setFormatter(logging.Formatter("%(message)s")) bloom_date_evolution.setLevel(logging.INFO) bloom_date_log.addHandler(bloom_date_evolution) bloom_date_log.propagate = False def get_forcing_data(config, log): - """Collect and process forcing data. - """ + """Collect and process forcing data.""" if not config.get_forcing_data: - log.info('Skipped collection and processing of forcing data') + log.info("Skipped collection and processing of forcing data") return wind_processor = wind.WindProcessor(config) config.data_date = wind_processor.make_forcing_data_file() - log.info('based on wind data forcing data date is {}' - .format(config.data_date.format('YYYY-MM-DD'))) + log.info( + "based on wind data forcing data date is {}".format( + config.data_date.format("YYYY-MM-DD") + ) + ) try: - with open('wind_data_date', 'rt') as f: + with open("wind_data_date", "rt") as f: last_data_date = arrow.get(f.readline().strip()).date() except IOError: # Fake a wind data date to get things rolling @@ -595,8 +623,8 @@ def get_forcing_data(config, log): if config.data_date.date() == last_data_date: raise ValueError else: - with open('wind_data_date', 'wt') as f: - f.write('{}\n'.format(config.data_date.format('YYYY-MM-DD'))) + with open("wind_data_date", "wt") as f: + f.write("{}\n".format(config.data_date.format("YYYY-MM-DD"))) meteo_processor = meteo.MeteoProcessor(config) meteo_processor.make_forcing_data_files() rivers_processor = rivers.RiversProcessor(config) @@ -615,10 +643,7 @@ def two_yr_suffix(year): :returns: String of the form ``_XXYY`` like ``_8081`` for 1981 """ - return ('_{year_m1}{year}' - .format( - year_m1=str(year - 1)[-2:], - year=str(year)[-2:])) + return "_{year_m1}{year}".format(year_m1=str(year - 1)[-2:], year=str(year)[-2:]) def find_member(bloom_dates, ord_day): @@ -640,11 +665,14 @@ def find_member(bloom_dates, ord_day): :returns: Ensemble member identifier :rtype: str """ + def find_matches(day): return [ - member for member, bloom_date in bloom_dates.items() + member + for member, bloom_date in bloom_dates.items() if bloom_date.toordinal() == day ] + matches = find_matches(ord_day) if not matches: for i in range(1, 11): @@ -655,103 +683,101 @@ def find_matches(day): return max(matches) -infile_edits_template = { # pragma: no cover - 'forcing_data': { - 'use_average_forcing_data': { - 'description': 'yes=avg only; no=fail if data runs out; fill=historic then avg', - 'value': 'histfill', - 'variable_name': 'use_average_forcing_data' +infile_edits_template = { # pragma: no cover + "forcing_data": { + "use_average_forcing_data": { + "description": "yes=avg only; no=fail if data runs out; fill=historic then avg", + "value": "histfill", + "variable_name": "use_average_forcing_data", }, - 'avg_historical_wind_file': { - 'description': 'average/historical wind forcing data path/filename', - 'value': None, - 'variable_name': 'n/a', + "avg_historical_wind_file": { + "description": "average/historical wind forcing data path/filename", + "value": None, + "variable_name": "n/a", }, - 'avg_historical_air_temperature_file': { - 'description': 'average/historical air temperature forcing data path/filename', - 'value': None, - 'variable_name': 'n/a', + "avg_historical_air_temperature_file": { + "description": "average/historical air temperature forcing data path/filename", + "value": None, + "variable_name": "n/a", }, - 'avg_historical_cloud_file': { - 'description': 'average/historical cloud fraction forcing data path/filename', - 'value': None, - 'variable_name': 'n/a' + "avg_historical_cloud_file": { + "description": "average/historical cloud fraction forcing data path/filename", + "value": None, + "variable_name": "n/a", }, - 'avg_historical_humidity_file': { - 'description': 'average/historical humidity forcing data path/filename', - 'value': None, - 'variable_name': 'n/a', + "avg_historical_humidity_file": { + "description": "average/historical humidity forcing data path/filename", + "value": None, + "variable_name": "n/a", }, - 'avg_historical_major_river_file': { - 'description': 'average/historical major river forcing data path/filename', - 'value': None, - 'variable_name': 'n/a', + "avg_historical_major_river_file": { + "description": "average/historical major river forcing data path/filename", + "value": None, + "variable_name": "n/a", }, - 'avg_historical_minor_river_file': { - 'description': 'average/historical minor river forcing data path/filename', - 'value': None, - 'variable_name': 'n/a', + "avg_historical_minor_river_file": { + "description": "average/historical minor river forcing data path/filename", + "value": None, + "variable_name": "n/a", }, }, - - 'timeseries_results': { - 'std_physics': { - 'description': 'path/filename for standard physics time series output', - 'value': None, - 'variable_name': 'std_phys_ts_out', + "timeseries_results": { + "std_physics": { + "description": "path/filename for standard physics time series output", + "value": None, + "variable_name": "std_phys_ts_out", }, - 'user_physics': { - 'description': 'path/filename for user physics time series output', - 'value': None, - 'variable_name': 'user_phys_ts_out', + "user_physics": { + "description": "path/filename for user physics time series output", + "value": None, + "variable_name": "user_phys_ts_out", }, - 'std_biology': { - 'description': 'path/filename for standard biology time series output', - 'value': None, - 'variable_name': 'std_bio_ts_out', + "std_biology": { + "description": "path/filename for standard biology time series output", + "value": None, + "variable_name": "std_bio_ts_out", }, - 'user_biology': { - 'description': 'path/filename for user biology time series output', - 'value': None, - 'variable_name': 'user_bio_ts_out', + "user_biology": { + "description": "path/filename for user biology time series output", + "value": None, + "variable_name": "user_bio_ts_out", }, - 'std_chemistry': { - 'description': 'path/filename for standard chemistry time series output', - 'value': None, - 'variable_name': 'std_chem_ts_out', + "std_chemistry": { + "description": "path/filename for standard chemistry time series output", + "value": None, + "variable_name": "std_chem_ts_out", }, - 'user_chemistry': { - 'description': 'path/filename for user chemistry time series output', - 'value': None, - 'variable_name': 'user_chem_ts_out', + "user_chemistry": { + "description": "path/filename for user chemistry time series output", + "value": None, + "variable_name": "user_chem_ts_out", }, }, - - 'profiles_results': { - 'profile_file_base': { - 'description': 'path/filename base for profiles (datetime will be appended)', - 'value': None, - 'variable_name': 'profilesBase_fn', + "profiles_results": { + "profile_file_base": { + "description": "path/filename base for profiles (datetime will be appended)", + "value": None, + "variable_name": "profilesBase_fn", }, - 'user_profile_file_base': { - 'description': 'path/filename base for user profiles (datetime appended)', - 'value': None, - 'variable_name': 'userprofilesBase_fn', + "user_profile_file_base": { + "description": "path/filename base for user profiles (datetime appended)", + "value": None, + "variable_name": "userprofilesBase_fn", }, - 'halocline_file': { - 'description': 'path/filename for halocline results', - 'value': None, - 'variable_name': 'haloclines_fn', + "halocline_file": { + "description": "path/filename for halocline results", + "value": None, + "variable_name": "haloclines_fn", }, - 'hoffmueller_file': { - 'description': 'path/filename for Hoffmueller results', - 'value': None, - 'variable_name': 'Hoffmueller_fn', + "hoffmueller_file": { + "description": "path/filename for Hoffmueller results", + "value": None, + "variable_name": "Hoffmueller_fn", }, - 'user_hoffmueller_file': { - 'description': 'path/filename for user Hoffmueller results', - 'value': None, - 'variable_name': 'userHoffmueller_fn', + "user_hoffmueller_file": { + "description": "path/filename for user Hoffmueller results", + "value": None, + "variable_name": "userHoffmueller_fn", }, }, } diff --git a/bloomcast/main.py b/bloomcast/main.py index 7299ad1..dc4bdc0 100644 --- a/bloomcast/main.py +++ b/bloomcast/main.py @@ -28,15 +28,16 @@ __all__ = [ - 'BloomcastApp', 'main', + "BloomcastApp", + "main", ] class BloomcastApp(cliff.app.App): - CONSOLE_MESSAGE_FORMAT = '%(levelname)s:%(name)s:%(message)s' + CONSOLE_MESSAGE_FORMAT = "%(levelname)s:%(name)s:%(message)s" def __init__(self): - app_namespace = 'bloomcast.app' + app_namespace = "bloomcast.app" super(BloomcastApp, self).__init__( description=__pkg_metadata__.DESCRIPTION, version=__pkg_metadata__.VERSION, @@ -49,5 +50,5 @@ def main(argv=sys.argv[1:]): return app.run(argv) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv[1:])) diff --git a/bloomcast/meteo.py b/bloomcast/meteo.py index 5cd8a15..64551ba 100644 --- a/bloomcast/meteo.py +++ b/bloomcast/meteo.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Meteorolgical forcing data processing module for SoG-bloomcast project. -""" +"""Meteorolgical forcing data processing module for SoG-bloomcast project.""" import logging import sys import contextlib @@ -26,17 +25,17 @@ ) -log = logging.getLogger('bloomcast.meteo') +log = logging.getLogger("bloomcast.meteo") class MeteoProcessor(ClimateDataProcessor): - """Meteorological forcing data processor. - """ + """Meteorological forcing data processor.""" + def __init__(self, config): data_readers = { - 'air_temperature': self.read_temperature, - 'relative_humidity': self.read_humidity, - 'cloud_fraction': self.read_cloud_fraction, + "air_temperature": self.read_temperature, + "relative_humidity": self.read_humidity, + "cloud_fraction": self.read_cloud_fraction, } super(MeteoProcessor, self).__init__(config, data_readers) @@ -51,21 +50,27 @@ def make_forcing_data_files(self): contexts = [] for qty in self.config.climate.meteo.quantities: output_file = self.config.climate.meteo.output_files[qty] - file_objs[qty] = open(output_file, 'wt') + file_objs[qty] = open(output_file, "wt") contexts.append(file_objs[qty]) self.raw_data = [] for data_month in self._get_data_months(): - self.get_climate_data('meteo', data_month) - log.debug('got meteo data for {0:%Y-%m}'.format(data_month)) + self.get_climate_data("meteo", data_month) + log.debug("got meteo data for {0:%Y-%m}".format(data_month)) with contextlib.ExitStack() as stack: files = dict( - [(qty, - stack.enter_context(open( - self.config.climate.meteo.output_files[qty], 'wt'))) - for qty in self.config.climate.meteo.quantities]) + [ + ( + qty, + stack.enter_context( + open(self.config.climate.meteo.output_files[qty], "wt") + ), + ) + for qty in self.config.climate.meteo.quantities + ] + ) for qty in self.config.climate.meteo.quantities: self.process_data(qty, end_date=self.config.data_date) - log.debug('latest {0} {1}'.format(qty, self.data[qty][-1])) + log.debug("latest {0} {1}".format(qty, self.data[qty][-1])) files[qty].writelines(self.format_data(qty)) def read_temperature(self, record): @@ -74,7 +79,7 @@ def read_temperature(self, record): SOG expects air temperature to be in 10ths of degrees Celcius due to legacy data formating of files from Environment Canada. """ - temperature = record.find('temp').text + temperature = record.find("temp").text try: temperature = float(temperature) * 10 except TypeError: @@ -83,9 +88,8 @@ def read_temperature(self, record): return temperature def read_humidity(self, record): - """Read relative humidity from XML data object. - """ - humidity = record.find('relhum').text + """Read relative humidity from XML data object.""" + humidity = record.find("relhum").text try: humidity = float(humidity) except TypeError: @@ -97,19 +101,21 @@ def read_cloud_fraction(self, record): """Read weather description from XML data object and transform it to cloud fraction via Susan's heuristic mapping. """ - weather_desc = record.find('weather').text + weather_desc = record.find("weather").text mapping = self.config.climate.meteo.cloud_fraction_mapping try: cloud_fraction = mapping[weather_desc] except KeyError: - if weather_desc is None or weather_desc == 'NA': + if weather_desc is None or weather_desc == "NA": # None indicates missing data cloud_fraction = [None] else: log.warning( - 'Unrecognized weather description: {0} at {1}; ' - 'cloud fraction set to 10' - .format(weather_desc, self.read_timestamp(record))) + "Unrecognized weather description: {0} at {1}; " + "cloud fraction set to 10".format( + weather_desc, self.read_timestamp(record) + ) + ) cloud_fraction = [10] if len(cloud_fraction) == 1: cloud_fraction = cloud_fraction[0] @@ -134,15 +140,16 @@ def format_data(self, qty): follow expressed as floats with 2 decimal place. """ for i in range(len(self.data[qty]) // 24): - data = self.data[qty][i * 24:(i + 1) * 24] + data = self.data[qty][i * 24 : (i + 1) * 24] timestamp = data[0][0] - line = '{0} {1:%Y %m %d} 42'.format( - self.config.climate.meteo.station_id, timestamp) + line = "{0} {1:%Y %m %d} 42".format( + self.config.climate.meteo.station_id, timestamp + ) for j, hour in enumerate(data): try: - line += ' {0:.2f}'.format(hour[1]) - if qty == 'cloud_fraction': - last_cf = hour[1] or data[j-1][1] + line += " {0:.2f}".format(hour[1]) + if qty == "cloud_fraction": + last_cf = hour[1] or data[j - 1][1] except TypeError: # This is a hack to work around NavCanada not reporting # a YVR weather description (from which we get cloud @@ -150,14 +157,15 @@ def format_data(self, qty): # happening. The upshot is that the final few values # in the dataset can be None, so we persist the last valid # value for that very special case, and log a warning. - if qty == 'cloud_fraction': - line += ' {0:.2f}'.format(last_cf) + if qty == "cloud_fraction": + line += " {0:.2f}".format(last_cf) log.warning( - f'missing cloud fraction value {hour} ' - f'filled with {last_cf}') + f"missing cloud fraction value {hour} " + f"filled with {last_cf}" + ) else: raise - line += '\n' + line += "\n" yield line @@ -168,10 +176,10 @@ def run(config_file): logging.basicConfig(level=logging.DEBUG) config = Config() config.load_config(config_file) - config.data_date = arrow.now().floor('day') + config.data_date = arrow.now().floor("day") meteo = MeteoProcessor(config) meteo.make_forcing_data_files() -if __name__ == '__main__': +if __name__ == "__main__": run(sys.argv[1]) diff --git a/bloomcast/rivers.py b/bloomcast/rivers.py index 696844e..71d8004 100644 --- a/bloomcast/rivers.py +++ b/bloomcast/rivers.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Rivers flows forcing data processing module for SoG-bloomcast project. -""" +"""Rivers flows forcing data processing module for SoG-bloomcast project.""" import datetime import logging import sys @@ -28,12 +27,12 @@ ) -log = logging.getLogger('bloomcast.rivers') +log = logging.getLogger("bloomcast.rivers") class RiversProcessor(ForcingDataProcessor): - """River flows forcing data processor. - """ + """River flows forcing data processor.""" + def __init__(self, config): super(RiversProcessor, self).__init__(config) @@ -44,7 +43,7 @@ def make_forcing_data_files(self): from the end, patch missing values, and write the data to files in the format that SOG expects. """ - for river in 'major minor'.split(): + for river in "major minor".split(): self.get_river_data(river) try: scale_factor = getattr(self.config.rivers, river).scale_factor @@ -53,11 +52,9 @@ def make_forcing_data_files(self): scale_factor = 1 self.process_data(river, scale_factor, end_date=self.config.data_date) output_file = self.config.rivers.output_files[river] - with open(output_file, 'wt') as file_obj: + with open(output_file, "wt") as file_obj: file_obj.writelines(self.format_data(river)) - log.debug( - 'latest {0} river flow {1}' - .format(river, self.data[river][-1])) + log.debug("latest {0} river flow {1}".format(river, self.data[river][-1])) def get_river_data(self, river): """Return a BeautifulSoup parser object containing the river @@ -65,23 +62,26 @@ def get_river_data(self, river): WaterOffice page. """ params = self.config.rivers.params - params['stn'] = getattr(self.config.rivers, river).station_id + params["stn"] = getattr(self.config.rivers, river).station_id today = arrow.now().date() - start_year = (self.config.run_start_date.year - if self.config.run_start_date.year != today.year - else today.year) + start_year = ( + self.config.run_start_date.year + if self.config.run_start_date.year != today.year + else today.year + ) params.update(self._date_params(start_year)) response = requests.get( self.config.rivers.data_url, params=params, - cookies=self.config.rivers.disclaimer_cookie) + cookies=self.config.rivers.disclaimer_cookie, + ) log.debug( - 'got {0} river data for {1}-01-01 to {2}' - .format( - river, start_year, - self.config.data_date.format('YYYY-MM-DD'))) - soup = bs4.BeautifulSoup(response.content, 'html.parser') - self.raw_data = soup.find('table') + "got {0} river data for {1}-01-01 to {2}".format( + river, start_year, self.config.data_date.format("YYYY-MM-DD") + ) + ) + soup = bs4.BeautifulSoup(response.content, "html.parser") + self.raw_data = soup.find("table") def _date_params(self, start_year): """Return a dict of the components of start and end dates for @@ -94,16 +94,16 @@ def _date_params(self, start_year): """ end_date = self.config.data_date.shift(days=+1) params = { - 'startDate': arrow.get(start_year, 1, 1).format('YYYY-MM-DD'), - 'endDate': end_date.format('YYYY-MM-DD') + "startDate": arrow.get(start_year, 1, 1).format("YYYY-MM-DD"), + "endDate": end_date.format("YYYY-MM-DD"), } return params - def process_data(self, qty, scale_factor=1, end_date=arrow.now().floor('day')): + def process_data(self, qty, scale_factor=1, end_date=arrow.now().floor("day")): """Process data from BeautifulSoup parser object to a list of hourly timestamps and data values. """ - tds = self.raw_data.findAll('td') + tds = self.raw_data.findAll("td") timestamps = (td.string for td in tds[::4]) flows = (td.text for td in tds[1::4]) data_day = self.read_datestamp(tds[0].string) @@ -131,20 +131,19 @@ def _convert_flow(self, flow_string, scale_factor): the end of the string. """ try: - return float(flow_string.replace(',', '')) * scale_factor + return float(flow_string.replace(",", "")) * scale_factor except ValueError: # Ignore training `*` - return float(flow_string[:-1].replace(',', '')) * scale_factor + return float(flow_string[:-1].replace(",", "")) * scale_factor def read_datestamp(self, string): """Read datestamp from BeautifulSoup parser object and return it as a date instance. """ - return datetime.datetime.strptime(string, '%Y-%m-%d %H:%M:%S').date() + return datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S").date() def patch_data(self, qty): - """Patch missing data values by interpolation. - """ + """Patch missing data values by interpolation.""" i = 0 data = self.data[qty] gap_count = 0 @@ -159,17 +158,19 @@ def patch_data(self, qty): missing_date = data[i][0] + j * datetime.timedelta(days=1) data.insert(i + j, (missing_date, None)) log.debug( - '{qty} river data patched for {date}' - .format(qty=qty, date=missing_date)) + "{qty} river data patched for {date}".format( + qty=qty, date=missing_date + ) + ) gap_count += 1 gap_end = i + delta - 1 self.interpolate_values(qty, gap_start, gap_end) i += delta if gap_count: log.debug( - '{count} {qty} river data values patched; ' - 'see debug log on disk for details' - .format(count=gap_count, qty=qty)) + "{count} {qty} river data values patched; " + "see debug log on disk for details".format(count=gap_count, qty=qty) + ) def format_data(self, qty): """Generate lines of river flow forcing data in the format @@ -188,7 +189,7 @@ def format_data(self, qty): for data in self.data[qty]: datestamp = data[0] flow = data[1] - line = '{0:%Y %m %d} {1:e}\n'.format(datestamp, flow) + line = "{0:%Y %m %d} {1:e}\n".format(datestamp, flow) yield line @@ -199,10 +200,10 @@ def run(config_file): logging.basicConfig(level=logging.DEBUG) config = Config() config.load_config(config_file) - config.data_date = arrow.now().floor('day') + config.data_date = arrow.now().floor("day") rivers = RiversProcessor(config) rivers.make_forcing_data_files() -if __name__ == '__main__': +if __name__ == "__main__": run(sys.argv[1]) diff --git a/bloomcast/utils.py b/bloomcast/utils.py index a4ef67e..fb0c841 100644 --- a/bloomcast/utils.py +++ b/bloomcast/utils.py @@ -31,7 +31,7 @@ import SOGcommand -log = logging.getLogger('bloomcast.utils') +log = logging.getLogger("bloomcast.utils") class _Container(object): @@ -39,158 +39,150 @@ class _Container(object): class Config(object): - """Placeholder for a config object that reads values from a file. - """ + """Placeholder for a config object that reads values from a file.""" + def load_config(self, config_file): """Load values from the specified config file into the attributes of the Config object. """ config_dict = self._read_yaml_file(config_file) self.logging = _Container() - self.logging.__dict__.update(config_dict['logging']) - self.get_forcing_data = config_dict['get_forcing_data'] - self.run_SOG = config_dict['run_SOG'] - self.SOG_executable = config_dict['SOG_executable'] + self.logging.__dict__.update(config_dict["logging"]) + self.get_forcing_data = config_dict["get_forcing_data"] + self.run_SOG = config_dict["run_SOG"] + self.SOG_executable = config_dict["SOG_executable"] self.ensemble = _Container() - self.ensemble.__dict__.update(config_dict['ensemble']) - self._load_results_config(config_dict['results']) + self.ensemble.__dict__.update(config_dict["ensemble"]) + self._load_results_config(config_dict["results"]) infile_dict = self._read_SOG_infile(self.ensemble.base_infile) - self.run_start_date = ( - infile_dict['run_start_date'] - .replace(hour=0, minute=0, second=0, microsecond=0)) - self.SOG_timestep = int(infile_dict['SOG_timestep']) + self.run_start_date = infile_dict["run_start_date"].replace( + hour=0, minute=0, second=0, microsecond=0 + ) + self.SOG_timestep = int(infile_dict["SOG_timestep"]) timeseries_keys = ( - 'std_phys_ts_outfile user_phys_ts_outfile ' - 'std_bio_ts_outfile user_bio_ts_outfile ' - 'std_chem_ts_outfile user_chem_ts_outfile' - .split()) + "std_phys_ts_outfile user_phys_ts_outfile " + "std_bio_ts_outfile user_bio_ts_outfile " + "std_chem_ts_outfile user_chem_ts_outfile".split() + ) for key in timeseries_keys: setattr(self, key, infile_dict[key]) profiles_keys = ( - 'profiles_outfile_base user_profiles_outfile_base ' - 'halocline_outfile ' - 'Hoffmueller_profiles_outfile user_Hoffmueller_profiles_outfile' - .split()) + "profiles_outfile_base user_profiles_outfile_base " + "halocline_outfile " + "Hoffmueller_profiles_outfile user_Hoffmueller_profiles_outfile".split() + ) for key in profiles_keys: setattr(self, key, infile_dict[key]) self.climate = _Container() - self.climate.__dict__.update(config_dict['climate']) + self.climate.__dict__.update(config_dict["climate"]) self._load_meteo_config(config_dict, infile_dict) self._load_wind_config(config_dict, infile_dict) self.rivers = _Container() - self.rivers.__dict__.update(config_dict['rivers']) + self.rivers.__dict__.update(config_dict["rivers"]) self._load_rivers_config(config_dict, infile_dict) def _load_results_config(self, config_dict): - """Load Config values for website results generation. - """ + """Load Config values for website results generation.""" self.results = _Container() for key, value in config_dict.items(): - if key == 'push_to_web': + if key == "push_to_web": self.results.__dict__[key] = value else: self.results.__dict__[key] = pathlib.Path(value) def _load_meteo_config(self, config_dict, infile_dict): - """Load Config values for meteorological forcing data. - """ + """Load Config values for meteorological forcing data.""" self.climate.meteo = _Container() - self.climate.meteo.__dict__.update( - config_dict['climate']['meteo']) + self.climate.meteo.__dict__.update(config_dict["climate"]["meteo"]) self.climate.meteo.cloud_fraction_mapping = self._read_yaml_file( - config_dict['climate']['meteo']['cloud_fraction_mapping']) - forcing_data_files = infile_dict['forcing_data_files'] + config_dict["climate"]["meteo"]["cloud_fraction_mapping"] + ) + forcing_data_files = infile_dict["forcing_data_files"] self.climate.meteo.output_files = {} for qty in self.climate.meteo.quantities: self.climate.meteo.output_files[qty] = forcing_data_files[qty] def _load_wind_config(self, config_dict, infile_dict): - """Load Config values for wind forcing data. - """ + """Load Config values for wind forcing data.""" self.climate.wind = _Container() - wind = config_dict['climate']['wind'] - self.climate.wind.station_id = wind['station_id'] - forcing_data_files = infile_dict['forcing_data_files'] + wind = config_dict["climate"]["wind"] + self.climate.wind.station_id = wind["station_id"] + forcing_data_files = infile_dict["forcing_data_files"] self.climate.wind.output_files = {} - self.climate.wind.output_files['wind'] = forcing_data_files['wind'] + self.climate.wind.output_files["wind"] = forcing_data_files["wind"] def _load_rivers_config(self, config_dict, infile_dict): - """Load Config values for river flows forcing data. - """ + """Load Config values for river flows forcing data.""" self.rivers.major = _Container() - major_river = config_dict['rivers']['major'] - self.rivers.major.station_id = major_river['station_id'] + major_river = config_dict["rivers"]["major"] + self.rivers.major.station_id = major_river["station_id"] self.rivers.minor = _Container() - minor_river = config_dict['rivers']['minor'] - self.rivers.minor.station_id = minor_river['station_id'] - self.rivers.minor.scale_factor = minor_river['scale_factor'] - forcing_data_files = infile_dict['forcing_data_files'] + minor_river = config_dict["rivers"]["minor"] + self.rivers.minor.station_id = minor_river["station_id"] + self.rivers.minor.scale_factor = minor_river["scale_factor"] + forcing_data_files = infile_dict["forcing_data_files"] self.rivers.output_files = {} - for river in 'major minor'.split(): - self.rivers.output_files[river] = ( - forcing_data_files[river + '_river']) + for river in "major minor".split(): + self.rivers.output_files[river] = forcing_data_files[river + "_river"] def _read_yaml_file(self, config_file): """Return the dict that results from loading the contents of the specified config file as YAML. """ - with open(config_file, 'rt') as file_obj: + with open(config_file, "rt") as file_obj: config = yaml.safe_load(file_obj.read()) - log.debug( - 'data structure read from {}'.format(config_file)) + log.debug("data structure read from {}".format(config_file)) return config def _read_SOG_infile(self, yaml_file): - """Return a dict of selected values read from the SOG infile. - """ + """Return a dict of selected values read from the SOG infile.""" # Mappings between SOG YAML infile keys and Config object attributes infile_values = { - 'initial_conditions.init_datetime': 'run_start_date', - 'numerics.dt': 'SOG_timestep', - 'timeseries_results.std_physics': 'std_phys_ts_outfile', - 'timeseries_results.user_physics': 'user_phys_ts_outfile', - 'timeseries_results.std_biology': 'std_bio_ts_outfile', - 'timeseries_results.user_biology': 'user_bio_ts_outfile', - 'timeseries_results.std_chemistry': 'std_chem_ts_outfile', - 'timeseries_results.user_chemistry': 'user_chem_ts_outfile', - 'profiles_results.profile_file_base': ( - 'profiles_outfile_base'), - 'profiles_results.user_profile_file_base': ( - 'user_profiles_outfile_base'), - 'profiles_results.halocline_file': ( - 'halocline_outfile'), - 'profiles_results.hoffmueller_file': ( - 'Hoffmueller_profiles_outfile'), - 'profiles_results.user_hoffmueller_file': ( - 'user_Hoffmueller_profiles_outfile'), + "initial_conditions.init_datetime": "run_start_date", + "numerics.dt": "SOG_timestep", + "timeseries_results.std_physics": "std_phys_ts_outfile", + "timeseries_results.user_physics": "user_phys_ts_outfile", + "timeseries_results.std_biology": "std_bio_ts_outfile", + "timeseries_results.user_biology": "user_bio_ts_outfile", + "timeseries_results.std_chemistry": "std_chem_ts_outfile", + "timeseries_results.user_chemistry": "user_chem_ts_outfile", + "profiles_results.profile_file_base": ("profiles_outfile_base"), + "profiles_results.user_profile_file_base": ("user_profiles_outfile_base"), + "profiles_results.halocline_file": ("halocline_outfile"), + "profiles_results.hoffmueller_file": ("Hoffmueller_profiles_outfile"), + "profiles_results.user_hoffmueller_file": ( + "user_Hoffmueller_profiles_outfile" + ), } forcing_data_files = { - 'forcing_data.wind_forcing_file': 'wind', - 'forcing_data.air_temperature_forcing_file': 'air_temperature', - 'forcing_data.humidity_forcing_file': 'relative_humidity', - 'forcing_data.cloud_fraction_forcing_file': 'cloud_fraction', - 'forcing_data.major_river_forcing_file': 'major_river', - 'forcing_data.minor_river_forcing_file': 'minor_river', + "forcing_data.wind_forcing_file": "wind", + "forcing_data.air_temperature_forcing_file": "air_temperature", + "forcing_data.humidity_forcing_file": "relative_humidity", + "forcing_data.cloud_fraction_forcing_file": "cloud_fraction", + "forcing_data.major_river_forcing_file": "major_river", + "forcing_data.minor_river_forcing_file": "minor_river", } - infile_dict = {'forcing_data_files': {}} + infile_dict = {"forcing_data_files": {}} for infile_key in infile_values: value = SOGcommand.api.read_infile(yaml_file, [], infile_key) result_key = infile_values[infile_key] infile_dict[result_key] = value log.debug( - 'run start date, time step, and output file names read from {}' - .format(yaml_file)) + "run start date, time step, and output file names read from {}".format( + yaml_file + ) + ) for infile_key in forcing_data_files: value = SOGcommand.api.read_infile(yaml_file, [], infile_key) result_key = forcing_data_files[infile_key] - infile_dict['forcing_data_files'][result_key] = value - log.debug('forcing data file names read from {}'.format(yaml_file)) + infile_dict["forcing_data_files"][result_key] = value + log.debug("forcing data file names read from {}".format(yaml_file)) return infile_dict class ForcingDataProcessor(object): - """Base class for forcing data processors. - """ + """Base class for forcing data processors.""" + def __init__(self, config): self.config = config self.data = {} @@ -212,14 +204,13 @@ def _trim_data(self, qty): data at 23:00 are deleted. """ while self.data[qty]: - if any([self._valuegetter(data[1]) - for data in self.data[qty][-24:]]): + if any([self._valuegetter(data[1]) for data in self.data[qty][-24:]]): break else: del self.data[qty][-24:] else: - raise ValueError('Forcing data list is empty') - if qty != 'cloud_fraction': + raise ValueError("Forcing data list is empty") + if qty != "cloud_fraction": # This is a hack to work around NavCanada not reporting # a YVR weather description (from which we get cloud # fraction) at 23:00 when there is no precipitation @@ -234,29 +225,26 @@ def _trim_data(self, qty): else: break else: - raise ValueError('Forcing data list is empty') + raise ValueError("Forcing data list is empty") def patch_data(self, qty): - """Patch missing data values by interpolation. - """ + """Patch missing data values by interpolation.""" gap_start = gap_end = None gap_count = 0 for i, data in enumerate(self.data[qty]): if self._valuegetter(data[1]) is None: gap_start = i if gap_start is None else gap_start gap_end = i - log.debug( - '{qty} data patched for {date}' - .format(qty=qty, date=data[0])) + log.debug("{qty} data patched for {date}".format(qty=qty, date=data[0])) gap_count += 1 elif gap_start is not None: self.interpolate_values(qty, gap_start, gap_end) gap_start = gap_end = None if gap_count: log.debug( - '{count} {qty} data values patched; ' - 'see debug log on disk for details' - .format(count=gap_count, qty=qty)) + "{count} {qty} data values patched; " + "see debug log on disk for details".format(count=gap_count, qty=qty) + ) def interpolate_values(self, qty, gap_start, gap_end): """Calculate values for missing data via linear interpolation. @@ -274,12 +262,11 @@ def interpolate_values(self, qty, gap_start, gap_end): gap_hours = gap_end - gap_start + 1 if gap_hours > 11: log.warning( - 'A {qty} forcing data gap > 11 hr starting at ' - '{gap_start:%Y-%m-%d %H:00} has been patched ' - 'by linear interpolation' - .format( - qty=qty, - gap_start=self.data[qty][gap_start][0]) + "A {qty} forcing data gap > 11 hr starting at " + "{gap_start:%Y-%m-%d %H:00} has been patched " + "by linear interpolation".format( + qty=qty, gap_start=self.data[qty][gap_start][0] + ) ) last_value = self.data[qty][gap_start - 1][1] next_value = self.data[qty][gap_end + 1][1] @@ -291,8 +278,8 @@ def interpolate_values(self, qty, gap_start, gap_end): class ClimateDataProcessor(ForcingDataProcessor): - """Climate forcing data processor base class. - """ + """Climate forcing data processor base class.""" + def __init__(self, config, data_readers): self.data_readers = data_readers super(ClimateDataProcessor, self).__init__(config) @@ -304,13 +291,12 @@ def get_climate_data(self, data_type, data_month): The XML objects are :class:`ElementTree` subelement instances. """ params = self.config.climate.params - params['stationID'] = getattr( - self.config.climate, data_type).station_id + params["stationID"] = getattr(self.config.climate, data_type).station_id params.update(self._date_params(data_month)) response = requests.get(self.config.climate.url, params=params) tree = ElementTree.parse(io.StringIO(response.text)) root = tree.getroot() - self.raw_data.extend(root.findall('stationdata')) + self.raw_data.extend(root.findall("stationdata")) def _date_params(self, data_month=None): """Return a dict of the components of the specified data month @@ -327,9 +313,9 @@ def _date_params(self, data_month=None): if not data_month: data_month = datetime.date.today() - datetime.timedelta(days=1) params = { - 'Year': data_month.year, - 'Month': data_month.month, - 'Day': 1, + "Year": data_month.year, + "Month": data_month.month, + "Day": 1, } return params @@ -343,15 +329,17 @@ def _get_data_months(self): """ today = datetime.date.today() this_year = today.year - data_months = [datetime.date(this_year, month, 1) - for month in range(1, today.month + 1)] + data_months = [ + datetime.date(this_year, month, 1) for month in range(1, today.month + 1) + ] if self.config.run_start_date.year != this_year: last_year = self.config.run_start_date.year - data_months = [datetime.date(last_year, month, 1) - for month in range(1, 13)] + data_months + data_months = [ + datetime.date(last_year, month, 1) for month in range(1, 13) + ] + data_months return data_months - def process_data(self, qty, end_date=arrow.now().floor('day')): + def process_data(self, qty, end_date=arrow.now().floor("day")): """Process data from XML data records to a list of hourly timestamps and data values. """ @@ -362,14 +350,16 @@ def process_data(self, qty, end_date=arrow.now().floor('day')): timestamp = self.read_timestamp(record) if timestamp.date() > end_date.date(): break - if qty != 'wind' and (timestamp.date() < YVR_STN_CHG_DATE): + if qty != "wind" and (timestamp.date() < YVR_STN_CHG_DATE): # Handle YVR station nummber change self.data[qty].append((timestamp, 0)) - elif all(( - qty == 'cloud_fraction', - self.raw_data.index(record) == 0, - reader(record) is None - )): + elif all( + ( + qty == "cloud_fraction", + self.raw_data.index(record) == 0, + reader(record) is None, + ) + ): # Handle no cloud fraction value observed at beginning # of time series; avoid interpolation failure self.data[qty].append((timestamp, 5)) @@ -383,10 +373,10 @@ def read_timestamp(self, record): datetime instance. """ timestamp = datetime.datetime( - int(record.get('year')), - int(record.get('month')), - int(record.get('day')), - int(record.get('hour')), + int(record.get("year")), + int(record.get("month")), + int(record.get("day")), + int(record.get("hour")), ) return timestamp @@ -404,6 +394,7 @@ class SOG_Relation(object): (e.g. Hoffmueller profile file requires a custom :meth:`read_data` method). """ + def __init__(self, datafile): """Create a SOG_Relation instance with its datafile attribute initialized. @@ -416,15 +407,15 @@ def read_header(self, file_obj): """ for line in file_obj: line = line.strip() - if line.startswith('*FieldNames:'): + if line.startswith("*FieldNames:"): # Drop the *FieldNames: label and keep the # comma-delimited list - field_names = line.split(': ', 1)[1].split(', ') - if line.startswith('*FieldUnits:'): + field_names = line.split(": ", 1)[1].split(", ") + if line.startswith("*FieldUnits:"): # Drop the *FieldUnits: label and keep the # comma-delimited list - field_units = line.split(': ', 1)[1].split(', ') - if line.startswith('*EndOfHeader'): + field_units = line.split(": ", 1)[1].split(", ") + if line.startswith("*EndOfHeader"): break return field_names, field_units @@ -436,7 +427,7 @@ def read_data(self, indep_field, dep_field): and the indep_units and dep_units attributes to units strings for the data fields. """ - with open(self.datafile, 'rt') as file_obj: + with open(self.datafile, "rt") as file_obj: (field_names, field_units) = self.read_header(file_obj) indep_col = field_names.index(indep_field) dep_col = field_names.index(dep_field) @@ -451,8 +442,8 @@ def read_data(self, indep_field, dep_field): class SOG_Timeseries(SOG_Relation): - """SOG timeseries relation. - """ + """SOG timeseries relation.""" + def boolean_slice(self, predicate, in_place=True): """Slice the independent and dependent data arrays using the Boolean ``predicate`` array. @@ -472,15 +463,21 @@ def calc_mpl_dates(self, run_start_date): """Calculate matplotlib dates from the independent data array and the ``run_start_date``. """ - self.mpl_dates = np.array(matplotlib.dates.date2num( - [run_start_date + datetime.timedelta(hours=hours) - for hours in self.indep_data])) + self.mpl_dates = np.array( + matplotlib.dates.date2num( + [ + run_start_date + datetime.timedelta(hours=hours) + for hours in self.indep_data + ] + ) + ) class SOG_HoffmuellerProfile(SOG_Relation): """SOG profile relation with data read from a Hoffmueller diagram results file. """ + def read_data(self, indep_field, dep_field, profile_number): """Read the data for the specified independent and dependent fields from the data file. @@ -489,7 +486,7 @@ def read_data(self, indep_field, dep_field, profile_number): and the indep_units and dep_units attributes to units strings for the data fields. """ - with open(self.datafile, 'rt') as file_obj: + with open(self.datafile, "rt") as file_obj: (field_names, field_units) = self.read_header(file_obj) indep_col = field_names.index(indep_field) dep_col = field_names.index(dep_field) @@ -498,7 +495,7 @@ def read_data(self, indep_field, dep_field, profile_number): self.indep_data, self.dep_data = [], [] profile_count = 1 for line in file_obj: - if line == '\n': + if line == "\n": profile_count += 1 if profile_count < profile_number: continue diff --git a/bloomcast/visualization.py b/bloomcast/visualization.py index 0f560d2..214f67a 100644 --- a/bloomcast/visualization.py +++ b/bloomcast/visualization.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Results visualization functions for SoG-bloomcast. -""" +"""Results visualization functions for SoG-bloomcast.""" import datetime import matplotlib.backends.backend_agg @@ -24,12 +23,18 @@ def nitrate_diatoms_timeseries( - nitrate, diatoms, colors, data_date, prediction, bloom_dates, titles, + nitrate, + diatoms, + colors, + data_date, + prediction, + bloom_dates, + titles, ): """Create a time series plot figure object showing nitrate and diatom biomass for median and bounds bloom predictions. """ - fig = matplotlib.figure.Figure(figsize=(15, 10), facecolor=colors['bg']) + fig = matplotlib.figure.Figure(figsize=(15, 10), facecolor=colors["bg"]) ax_late = fig.add_subplot(3, 1, 3) ax_median = fig.add_subplot(3, 1, 2, sharex=ax_late) ax_early = fig.add_subplot(3, 1, 1, sharex=ax_late) @@ -37,214 +42,261 @@ def nitrate_diatoms_timeseries( axes_right = [ax.twinx() for ax in axes_left] # Set colours of background, spines, ticks, and labels for ax in axes_left: - ax.set_facecolor(colors['bg']) - set_spine_and_tick_colors(ax, colors, yticks='nitrate') + ax.set_facecolor(colors["bg"]) + set_spine_and_tick_colors(ax, colors, yticks="nitrate") for ax in axes_right: - set_spine_and_tick_colors(ax, colors, yticks='diatoms') + set_spine_and_tick_colors(ax, colors, yticks="diatoms") # Set titles above each sub-plot ax_titles = ( - 'Early Bound Prediction', - 'Median Prediction', - 'Late Bound Prediction', + "Early Bound Prediction", + "Median Prediction", + "Late Bound Prediction", ) for i, title in enumerate(ax_titles): axes_left[i].annotate( - title, xy=(0, 1), xytext=(0, 5), - xycoords='axes fraction', textcoords='offset points', - size='large', color=colors['axes']) + title, + xy=(0, 1), + xytext=(0, 5), + xycoords="axes fraction", + textcoords="offset points", + size="large", + color=colors["axes"], + ) # Plot time series for i, member in enumerate(prediction.values()): axes_left[i].plot( nitrate[member].mpl_dates, nitrate[member].dep_data, - color=colors['nitrate'], + color=colors["nitrate"], ) axes_right[i].plot( diatoms[member].mpl_dates, diatoms[member].dep_data, - color=colors['diatoms'], + color=colors["diatoms"], ) # Set y-axes ticks and labels axes_left[i].set_ybound(0, 30) axes_left[i].set_yticks(range(0, 31, 5)) - axes_left[i].grid(linestyle=(0, (1, 3)), color=colors['axes'], alpha=0.5) + axes_left[i].grid(linestyle=(0, (1, 3)), color=colors["axes"], alpha=0.5) axes_right[i].set_ybound(0, 18) axes_right[i].set_yticks(range(0, 19, 3)) # Add lines at bloom date and actual to ensemble forcing transition add_transition_date_line(axes_left[i], data_date, colors) add_bloom_date_line(axes_left[i], bloom_dates[member], colors) # Set x-axes limits, tick intervals, title, and grid visibility - set_timeseries_x_limits_ticks_label( - ax_late, nitrate[prediction['median']], colors) - hide_ticklabels(ax_early, 'x') - hide_ticklabels(ax_median, 'x') - axes_left[1].set_ylabel(titles[0], color=colors['nitrate']) - axes_right[1].set_ylabel(titles[1], color=colors['diatoms']) + set_timeseries_x_limits_ticks_label(ax_late, nitrate[prediction["median"]], colors) + hide_ticklabels(ax_early, "x") + hide_ticklabels(ax_median, "x") + axes_left[1].set_ylabel(titles[0], color=colors["nitrate"]) + axes_right[1].set_ylabel(titles[1], color=colors["diatoms"]) return fig def temperature_salinity_timeseries( - temperature, salinity, colors, data_date, prediction, bloom_dates, titles, + temperature, + salinity, + colors, + data_date, + prediction, + bloom_dates, + titles, ): """Create a time series plot figure object showing temperature on the left axis and salinity on the right. """ - fig = matplotlib.figure.Figure(figsize=(15, 4.25), facecolor=colors['bg']) + fig = matplotlib.figure.Figure(figsize=(15, 4.25), facecolor=colors["bg"]) ax_left = fig.add_subplot(1, 1, 1) ax_right = ax_left.twinx() # Set colours of background, spines, ticks, and labels - ax_left.set_facecolor(colors['bg']) - set_spine_and_tick_colors(ax_left, colors, yticks='temperature') - set_spine_and_tick_colors(ax_right, colors, yticks='salinity') + ax_left.set_facecolor(colors["bg"]) + set_spine_and_tick_colors(ax_left, colors, yticks="temperature") + set_spine_and_tick_colors(ax_right, colors, yticks="salinity") ax_left.annotate( - 'Temperature and Salinity', xy=(0, 1), xytext=(0, 5), - xycoords='axes fraction', textcoords='offset points', - size='large', color=colors['axes']) + "Temperature and Salinity", + xy=(0, 1), + xytext=(0, 5), + xycoords="axes fraction", + textcoords="offset points", + size="large", + color=colors["axes"], + ) # Plot time series - lines, labels = [0]*6, [0]*6 - for i, key in enumerate('early late median'.split()): - line, = ax_left.plot( + lines, labels = [0] * 6, [0] * 6 + for i, key in enumerate("early late median".split()): + (line,) = ax_left.plot( temperature[prediction[key]].mpl_dates, temperature[prediction[key]].dep_data, - color=colors['temperature_lines'][key]) + color=colors["temperature_lines"][key], + ) lines[i] = line labels[i] = key.title() - line, = ax_right.plot( + (line,) = ax_right.plot( salinity[prediction[key]].mpl_dates, salinity[prediction[key]].dep_data, - color=colors['salinity_lines'][key]) + color=colors["salinity_lines"][key], + ) lines[i + 3] = line labels[i + 3] = key.title() leg = ax_left.legend( - lines, labels, title='Forcing Data Source', ncol=2, loc='lower left', - fancybox=True, fontsize='small') + lines, + labels, + title="Forcing Data Source", + ncol=2, + loc="lower left", + fancybox=True, + fontsize="small", + ) leg.get_frame().set_alpha(0.5) # Set x-axes limits, tick intervals, title, and grid visibility set_timeseries_x_limits_ticks_label( - ax_left, temperature[prediction['median']], colors) + ax_left, temperature[prediction["median"]], colors + ) fig.subplots_adjust(bottom=0.17) # Set y-axes ticks and labels ax_left.set_ybound(4, 18) - ax_left.grid(linestyle=(0, (1, 3)), color=colors['axes'], alpha=0.5) + ax_left.grid(linestyle=(0, (1, 3)), color=colors["axes"], alpha=0.5) ax_right.set_ybound(16, 30) - ax_left.set_ylabel(titles[0], color=colors['temperature']) - ax_right.set_ylabel(titles[1], color=colors['salinity']) + ax_left.set_ylabel(titles[0], color=colors["temperature"]) + ax_right.set_ylabel(titles[1], color=colors["salinity"]) # Add line at actual to ensemble forcing transition add_transition_date_line(ax_left, data_date, colors) return fig def mixing_layer_depth_wind_timeseries( - mixing_layer_depth, wind, colors, data_date, titles, + mixing_layer_depth, + wind, + colors, + data_date, + titles, ): - fig = matplotlib.figure.Figure(figsize=(15, 4.75), facecolor=colors['bg']) + fig = matplotlib.figure.Figure(figsize=(15, 4.75), facecolor=colors["bg"]) ax_left = fig.add_subplot(1, 1, 1) ax_right = ax_left.twinx() # Set colours of background, spines, ticks, and labels - ax_left.set_facecolor(colors['bg']) - set_spine_and_tick_colors(ax_left, colors, yticks='mld') - set_spine_and_tick_colors(ax_right, colors, yticks='wind_speed') + ax_left.set_facecolor(colors["bg"]) + set_spine_and_tick_colors(ax_left, colors, yticks="mld") + set_spine_and_tick_colors(ax_right, colors, yticks="wind_speed") ax_left.annotate( - 'Mixing Layer Depth and Wind Speed', xy=(0, 1), xytext=(0, 5), - xycoords='axes fraction', textcoords='offset points', - size='large', color=colors['axes']) + "Mixing Layer Depth and Wind Speed", + xy=(0, 1), + xytext=(0, 5), + xycoords="axes fraction", + textcoords="offset points", + size="large", + color=colors["axes"], + ) # Plot time series def calc_slice(data): slice = np.logical_and( - data.mpl_dates > matplotlib.dates.date2num( - data_date.shift(days=-6).datetime), - data.mpl_dates <= matplotlib.dates.date2num( - data_date.shift(days=+1).datetime)) + data.mpl_dates + > matplotlib.dates.date2num(data_date.shift(days=-6).datetime), + data.mpl_dates + <= matplotlib.dates.date2num(data_date.shift(days=+1).datetime), + ) return slice mld_slice = calc_slice(mixing_layer_depth) mld_dates = mixing_layer_depth.mpl_dates[mld_slice] ax_left.fill_between( - mld_dates, -mixing_layer_depth.dep_data[mld_slice], - color=colors['mld'], alpha=0.5, + mld_dates, + -mixing_layer_depth.dep_data[mld_slice], + color=colors["mld"], + alpha=0.5, ) wind_slice = calc_slice(wind) ax_right.fill_between( - wind.mpl_dates[wind_slice], wind.dep_data[wind_slice], - color=colors['wind_speed'], alpha=0.25, + wind.mpl_dates[wind_slice], + wind.dep_data[wind_slice], + color=colors["wind_speed"], + alpha=0.25, ) # Set x-axes limits, tick intervals, title, and grid visibility ax_left.xaxis.set_major_locator(matplotlib.dates.DayLocator()) - ax_left.xaxis.set_major_formatter( - matplotlib.dates.DateFormatter('%j\n%d-%b')) + ax_left.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%j\n%d-%b")) ax_left.xaxis.set_minor_locator(matplotlib.dates.HourLocator(interval=6)) - ax_left.xaxis.set_tick_params(which='minor', color=colors['axes']) + ax_left.xaxis.set_tick_params(which="minor", color=colors["axes"]) start_date = matplotlib.dates.num2date(np.trunc(mld_dates[0])) end_date = matplotlib.dates.num2date(np.ceil(mld_dates[-1])) ax_left.set_xlim((start_date, end_date)) if start_date.year == end_date.year: - label = 'Year-days in {year}'.format(year=start_date.year) + label = "Year-days in {year}".format(year=start_date.year) else: - label = ( - 'Year-days in {first_year} and {second_year}' - .format( - first_year=start_date.year, - second_year=end_date.year)) - ax_left.set_xlabel(label, color=colors['axes']) + label = "Year-days in {first_year} and {second_year}".format( + first_year=start_date.year, second_year=end_date.year + ) + ax_left.set_xlabel(label, color=colors["axes"]) fig.subplots_adjust(bottom=0.15) # Set y-axes ticks and labels ax_left.set_ybound(-30, 30) ax_left.set_yticks(range(-30, 31, 5)) ax_left.set_yticklabels( - ('30', '25', '20', '15', '10', '5', '0', '', '', '', '', '', '')) - ax_left.grid(linestyle=(0, (1, 3)), color=colors['axes'], alpha=0.5) + ("30", "25", "20", "15", "10", "5", "0", "", "", "", "", "", "") + ) + ax_left.grid(linestyle=(0, (1, 3)), color=colors["axes"], alpha=0.5) ax_right.set_ybound(-24, 24) ax_right.set_yticks(range(-24, 25, 4)) ax_right.set_yticklabels( - ('', '', '', '', '', '', '0', '4', '8', '12', '16', '20', '24')) - ax_left.set_ylabel(titles[0], color=colors['mld']) - ax_right.set_ylabel(titles[1], color=colors['wind_speed']) + ("", "", "", "", "", "", "0", "4", "8", "12", "16", "20", "24") + ) + ax_left.set_ylabel(titles[0], color=colors["mld"]) + ax_right.set_ylabel(titles[1], color=colors["wind_speed"]) # Add line to mark profile time - profile_datetime = matplotlib.dates.date2num( - data_date.shift(hours=12).datetime) - ax_left.axvline(profile_datetime, color=colors['axes']) + profile_datetime = matplotlib.dates.date2num(data_date.shift(hours=12).datetime) + ax_left.axvline(profile_datetime, color=colors["axes"]) ax_left.annotate( - 'Profile Time', + "Profile Time", xy=(profile_datetime, ax_left.get_ylim()[1]), - xytext=(0, 5), xycoords='data', textcoords='offset points', - size='small', color=colors['axes']) + xytext=(0, 5), + xycoords="data", + textcoords="offset points", + size="small", + color=colors["axes"], + ) return fig def add_bloom_date_line(axes, bloom_date, colors): d = datetime.datetime.combine(bloom_date, datetime.time(12)) - axes.axvline( - matplotlib.dates.date2num(d), color=colors['diatoms']) + axes.axvline(matplotlib.dates.date2num(d), color=colors["diatoms"]) axes.annotate( - 'Bloom Date', xy=(d, axes.get_ylim()[1]), xytext=(2, -12), - xycoords='data', textcoords='offset points', - size='small', color=colors['axes']) + "Bloom Date", + xy=(d, axes.get_ylim()[1]), + xytext=(2, -12), + xycoords="data", + textcoords="offset points", + size="small", + color=colors["axes"], + ) def add_transition_date_line(axes, data_date, colors): - axes.axvline( - matplotlib.dates.date2num(data_date.datetime), color=colors['axes']) + axes.axvline(matplotlib.dates.date2num(data_date.datetime), color=colors["axes"]) axes.annotate( - 'Actual to Ensemble\nForcing Transition', + "Actual to Ensemble\nForcing Transition", xy=(matplotlib.dates.date2num(data_date.datetime), axes.get_ylim()[1]), - xytext=(-70, 5), xycoords='data', textcoords='offset points', - size='small', color=colors['axes']) + xytext=(-70, 5), + xycoords="data", + textcoords="offset points", + size="small", + color=colors["axes"], + ) -def hide_ticklabels(axes, axis='both'): - if axis in 'x both'.split(): +def hide_ticklabels(axes, axis="both"): + if axis in "x both".split(): for t in axes.get_xticklabels(): t.set_visible(False) - if axis in 'y both'.split(): + if axis in "y both".split(): for t in axes.get_yticklabels(): t.set_visible(False) -def set_spine_and_tick_colors(axes, colors, xticks='axes', yticks='axes'): - for side in 'top bottom left right'.split(): - axes.spines[side].set_color(colors['axes']) - axes.tick_params(color=colors['axes']) +def set_spine_and_tick_colors(axes, colors, xticks="axes", yticks="axes"): + for side in "top bottom left right".split(): + axes.spines[side].set_color(colors["axes"]) + axes.tick_params(color=colors["axes"]) for label in axes.get_xticklabels(): label.set_color(colors[xticks]) for label in axes.get_yticklabels(): @@ -256,23 +308,25 @@ def set_timeseries_x_limits_ticks_label(axes, timeseries, colors): end_date = matplotlib.dates.num2date(np.ceil(timeseries.mpl_dates[-1])) axes.set_xlim((start_date, end_date)) axes.xaxis.set_major_locator(matplotlib.dates.MonthLocator()) - axes.xaxis.set_major_formatter( - matplotlib.dates.DateFormatter('%j\n%b')) + axes.xaxis.set_major_formatter(matplotlib.dates.DateFormatter("%j\n%b")) if start_date.year == end_date.year: - label = 'Year-days in {year}'.format(year=start_date.year) + label = "Year-days in {year}".format(year=start_date.year) else: - label = ( - 'Year-days in {first_year} and {second_year}' - .format( - first_year=start_date.year, - second_year=end_date.year)) - axes.set_xlabel(label, color=colors['axes']) + label = "Year-days in {first_year} and {second_year}".format( + first_year=start_date.year, second_year=end_date.year + ) + axes.set_xlabel(label, color=colors["axes"]) def profiles( - profiles, titles, limits, mixing_layer_depth, label_colors, colors, + profiles, + titles, + limits, + mixing_layer_depth, + label_colors, + colors, ): - fig = matplotlib.figure.Figure(figsize=(15, 10), facecolor=colors['bg']) + fig = matplotlib.figure.Figure(figsize=(15, 10), facecolor=colors["bg"]) axs = [] axs.append(fig.add_subplot(1, 4, 1)) for i in range(1, 4): @@ -280,42 +334,47 @@ def profiles( # Plot profiles with colour coordinated tick labels on the top axis for i, ax in enumerate(axs): ax.plot( - profiles[i].dep_data, profiles[i].indep_data, - color=colors[label_colors[i]] + profiles[i].dep_data, profiles[i].indep_data, color=colors[label_colors[i]] ) if limits[i] is not None: ax.set_xlim(limits[i]) - ax.xaxis.set_ticks_position('both') - ax.xaxis.set_label_position('top') - ax.tick_params(labelbottom='off', labeltop='on') + ax.xaxis.set_ticks_position("both") + ax.xaxis.set_label_position("top") + ax.tick_params(labelbottom="off", labeltop="on") set_spine_and_tick_colors(ax, colors, xticks=label_colors[i]) ax.set_xlabel(titles[i], color=colors[label_colors[i]]) - ax.set_facecolor(colors['bg']) - ax.grid(linestyle=(0, (1, 3)), color=colors['axes'], alpha=0.5) + ax.set_facecolor(colors["bg"]) + ax.grid(linestyle=(0, (1, 3)), color=colors["axes"], alpha=0.5) # Add line to mark mixing layer depth with its value on left axes for ax in axs: ax.axhline(mixing_layer_depth, color=colors[label_colors[-1]]) trans = matplotlib.transforms.blended_transform_factory( - axs[0].transAxes, axs[0].transData) + axs[0].transAxes, axs[0].transData + ) axs[0].text( - x=-0.025, y=mixing_layer_depth, transform=trans, - s='{:.2f} m'.format(mixing_layer_depth), - verticalalignment='center', horizontalalignment='right', - color=colors['mld'], + x=-0.025, + y=mixing_layer_depth, + transform=trans, + s="{:.2f} m".format(mixing_layer_depth), + verticalalignment="center", + horizontalalignment="right", + color=colors["mld"], ) y_offset = -0.3 if mixing_layer_depth > 1 else 0.8 axs[0].text( - x=0.975, y=mixing_layer_depth + y_offset, transform=trans, - s='Mixing Layer Depth', - horizontalalignment='right', - color=colors['mld'], + x=0.975, + y=mixing_layer_depth + y_offset, + transform=trans, + s="Mixing Layer Depth", + horizontalalignment="right", + color=colors["mld"], ) # Set left y-axis limits & label and hide y-axis labels on other axes axs[0].set_ylim((profiles[0].indep_data[0], profiles[0].indep_data[-1])) axs[0].invert_yaxis() - axs[0].set_ylabel('Depth [m]', color=colors['axes']) + axs[0].set_ylabel("Depth [m]", color=colors["axes"]) for ax in axs[1:]: - hide_ticklabels(ax, 'y') + hide_ticklabels(ax, "y") return fig diff --git a/bloomcast/wind.py b/bloomcast/wind.py index 0f08591..4b21b47 100644 --- a/bloomcast/wind.py +++ b/bloomcast/wind.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Wind forcing data processing module for SoG-bloomcast project. -""" +"""Wind forcing data processing module for SoG-bloomcast project.""" import datetime import logging import math @@ -29,14 +28,14 @@ ) -log = logging.getLogger('bloomcast.wind') +log = logging.getLogger("bloomcast.wind") class WindProcessor(ClimateDataProcessor): - """Wind forcing data processor. - """ + """Wind forcing data processor.""" + def __init__(self, config): - data_readers = {'wind': self.read_wind_velocity} + data_readers = {"wind": self.read_wind_velocity} super(WindProcessor, self).__init__(config, data_readers) def make_forcing_data_file(self): @@ -49,13 +48,13 @@ def make_forcing_data_file(self): """ self.raw_data = [] for data_month in self._get_data_months(): - self.get_climate_data('wind', data_month) - log.debug('got wind data for {0:%Y-%m}'.format(data_month)) - self.process_data('wind') - log.debug('latest wind {0}'.format(self.data['wind'][-1])) - data_date = arrow.get(self.data['wind'][-1][0]).replace(hour=0) - output_file = self.config.climate.wind.output_files['wind'] - with open(output_file, 'wt') as file_obj: + self.get_climate_data("wind", data_month) + log.debug("got wind data for {0:%Y-%m}".format(data_month)) + self.process_data("wind") + log.debug("latest wind {0}".format(self.data["wind"][-1])) + data_date = arrow.get(self.data["wind"][-1][0]).replace(hour=0) + output_file = self.config.climate.wind.output_files["wind"] + with open(output_file, "wt") as file_obj: file_obj.writelines(self.format_data()) return data_date @@ -63,8 +62,8 @@ def read_wind_velocity(self, record): """Read wind velocity from XML data object and transform it to along- and cross-strait components. """ - speed = record.find('windspd').text - direction = record.find('winddir').text + speed = record.find("windspd").text + direction = record.find("winddir").text try: # Convert from km/hr to m/s speed = float(speed) * 1000 / (60 * 60) @@ -84,12 +83,12 @@ def read_wind_velocity(self, record): v_wind = speed * math.cos(radian_direction) # Rotate components to align u direction with Strait strait_heading = math.radians(305) - cross_wind = ( - u_wind * math.cos(strait_heading) - - v_wind * math.sin(strait_heading)) - along_wind = ( - u_wind * math.sin(strait_heading) - + v_wind * math.cos(strait_heading)) + cross_wind = u_wind * math.cos(strait_heading) - v_wind * math.sin( + strait_heading + ) + along_wind = u_wind * math.sin(strait_heading) + v_wind * math.cos( + strait_heading + ) # Resolve atmosphere/ocean direction difference in favour of # oceanography cross_wind = -cross_wind @@ -97,8 +96,7 @@ def read_wind_velocity(self, record): return cross_wind, along_wind def _valuegetter(self, data_item): - """Return the along-strait wind velocity component. - """ + """Return the along-strait wind velocity component.""" return data_item[0] def interpolate_values(self, qty, gap_start, gap_end): @@ -110,21 +108,20 @@ def interpolate_values(self, qty, gap_start, gap_end): gap_hours = gap_end - gap_start + 1 if gap_hours > 11: log.warning( - 'A wind forcing data gap > 11 hr starting at ' - '{0:%Y-%m-%d %H:00} has been patched by linear interpolation' - .format(self.data[qty][gap_start][0])) + "A wind forcing data gap > 11 hr starting at " + "{0:%Y-%m-%d %H:00} has been patched by linear interpolation".format( + self.data[qty][gap_start][0] + ) + ) last_cross_wind, last_along_wind = self.data[qty][gap_start - 1][1] next_cross_wind, next_along_wind = self.data[qty][gap_end + 1][1] - delta_cross_wind = ( - (next_cross_wind - last_cross_wind) / (gap_hours + 1)) - delta_along_wind = ( - (next_along_wind - last_along_wind) / (gap_hours + 1)) + delta_cross_wind = (next_cross_wind - last_cross_wind) / (gap_hours + 1) + delta_along_wind = (next_along_wind - last_along_wind) / (gap_hours + 1) for i in range(gap_end - gap_start + 1): timestamp = self.data[qty][gap_start + i][0] cross_wind = last_cross_wind + delta_cross_wind * (i + 1) along_wind = last_along_wind + delta_along_wind * (i + 1) - self.data[qty][gap_start + i] = ( - timestamp, (cross_wind, along_wind)) + self.data[qty][gap_start + i] = (timestamp, (cross_wind, along_wind)) def format_data(self): """Generate lines of wind forcing data in the format expected @@ -142,22 +139,23 @@ def format_data(self): * Cross-strait wind component * Along-strait wind component """ - for data in self.data['wind']: + for data in self.data["wind"]: timestamp = data[0] wind = data[1] - line = '{0:%d %m %Y} {1:.1f} {2:f} {3:f}\n'.format( - timestamp, timestamp.hour, wind[0], wind[1]) + line = "{0:%d %m %Y} {1:.1f} {2:f} {3:f}\n".format( + timestamp, timestamp.hour, wind[0], wind[1] + ) yield line class WindTimeseries(SOG_Timeseries): - """Wind speed data expressed as a SOG timerseries object. - """ + """Wind speed data expressed as a SOG timerseries object.""" + def read_data( self, run_start_date, - indep_field='timestamp', - dep_field='wind speed', + indep_field="timestamp", + dep_field="wind speed", ): def interesting(data): for row in data: @@ -173,7 +171,7 @@ def interesting(data): yield timestamp, wind_speed self.indep_data, self.dep_data = [], [] - with open(self.datafile, 'rt') as data: + with open(self.datafile, "rt") as data: for timestamp, wind_speed in interesting(data): self.indep_data.append(timestamp) self.dep_data.append(wind_speed) @@ -192,5 +190,5 @@ def run(config_file): wind.make_forcing_data_file() -if __name__ == '__main__': +if __name__ == "__main__": run(sys.argv[1]) diff --git a/cf_analysis/cf_analysis.py b/cf_analysis/cf_analysis.py index c4c32f7..07957e3 100644 --- a/cf_analysis/cf_analysis.py +++ b/cf_analysis/cf_analysis.py @@ -62,27 +62,27 @@ import yaml -EC_URL = 'http://www.climate.weatheroffice.gc.ca/climateData/bulkdata_e.html' +EC_URL = "http://www.climate.weatheroffice.gc.ca/climateData/bulkdata_e.html" START_YEAR = 2002 END_YEAR = 2011 -YVR_CF_FILE = '../../SOG-forcing/met/YVRhistCF' +YVR_CF_FILE = "../../SOG-forcing/met/YVRhistCF" DUMP_HOURLY_RESULTS = False -HOURLY_FILE = 'cf_analysis.txt' +HOURLY_FILE = "cf_analysis.txt" # Threshold for number of observations of a given weather description # at which to switch from averaging all values to averaging values for # each month AVERAGING_THRESHOLD = 500 -MAPPING_FILE = 'cloud_fraction_mapping.yaml' +MAPPING_FILE = "cloud_fraction_mapping.yaml" root_log = logging.getLogger() -log = logging.getLogger('cf_analysis') +log = logging.getLogger("cf_analysis") logging.basicConfig(level=logging.DEBUG) -formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s') +formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s") console = logging.StreamHandler() console.setFormatter(formatter) log.addHandler(console) -disk = logging.FileHandler('cf_analysis.log', mode='w') +disk = logging.FileHandler("cf_analysis.log", mode="w") disk.setFormatter(formatter) log.addHandler(disk) root_log.addHandler(disk) @@ -91,55 +91,54 @@ def run(): data_months = ( - date(year, month, 1) - for year in range(2002, 2012) - for month in range(1, 13) + date(year, month, 1) for year in range(2002, 2012) for month in range(1, 13) ) request_params = { - 'timeframe': 1, # Daily - 'Prov': 'BC', - 'format': 'xml', - 'StationID': 889, # YVR - 'Day': 1, + "timeframe": 1, # Daily + "Prov": "BC", + "format": "xml", + "StationID": 889, # YVR + "Day": 1, } mapping = {} - yvr_file = open(YVR_CF_FILE, 'rt') + yvr_file = open(YVR_CF_FILE, "rt") context = contextlib.nested(yvr_file) if DUMP_HOURLY_RESULTS: - hourly_file = open(HOURLY_FILE, 'wt') + hourly_file = open(HOURLY_FILE, "wt") context = contextlib.nested(yvr_file, hourly_file) with context: for data_month in data_months: ec_data = get_EC_data(data_month, request_params) yvr_data = get_yvr_line(yvr_file, START_YEAR).next() - for record in ec_data.findall('stationdata'): - parts = [record.get(part) - for part in 'year month day hour'.split()] + for record in ec_data.findall("stationdata"): + parts = [record.get(part) for part in "year month day hour".split()] timestamp = datetime(*map(int, parts)) - weather_desc = record.find('weather').text + weather_desc = record.find("weather").text if weather_desc is None: log.info( - 'Missing weather description at {0:%Y-%m-%d %H:%M} ' - 'skipped'.format(timestamp)) + "Missing weather description at {0:%Y-%m-%d %H:%M} " + "skipped".format(timestamp) + ) continue - while timestamp.date() > yvr_data['date']: + while timestamp.date() > yvr_data["date"]: yvr_data = get_yvr_line(yvr_file, START_YEAR).next() if DUMP_HOURLY_RESULTS: - write_hourly_line( - timestamp, weather_desc, yvr_data, hourly_file) + write_hourly_line(timestamp, weather_desc, yvr_data, hourly_file) build_raw_mapping(mapping, weather_desc, timestamp, yvr_data) calc_mapping_averages(mapping) - with open(MAPPING_FILE, 'wt') as mapping_file: + with open(MAPPING_FILE, "wt") as mapping_file: yaml.safe_dump(mapping, mapping_file) def get_EC_data(data_month, request_params): - request_params.update({ - 'Year': data_month.year, - 'Month': data_month.month, - }) + request_params.update( + { + "Year": data_month.year, + "Month": data_month.month, + } + ) response = requests.get(EC_URL, params=request_params) - log.info('got meteo data for {0:%Y-%m}'.format(data_month)) + log.info("got meteo data for {0:%Y-%m}".format(data_month)) tree = ElementTree.parse(StringIO(response.content)) ec_data = tree.getroot() return ec_data @@ -152,30 +151,43 @@ def get_yvr_line(yvr_file, start_year): data_date = date(*map(int, parts[1:4])) else: yvr_data = { - 'date': data_date, - 'hourly_cfs': map(float, parts[5:29]), + "date": data_date, + "hourly_cfs": map(float, parts[5:29]), } yield yvr_data def write_hourly_line(timestamp, weather_desc, yvr_data, hourly_file): - result_line = ( - '{0:%Y-%m-%d %H:%M:%S} {1} {2}\n' - .format(timestamp, weather_desc, - yvr_data['hourly_cfs'][timestamp.hour])) + result_line = "{0:%Y-%m-%d %H:%M:%S} {1} {2}\n".format( + timestamp, weather_desc, yvr_data["hourly_cfs"][timestamp.hour] + ) hourly_file.write(result_line) def build_raw_mapping(mapping, weather_desc, timestamp, yvr_data): try: mapping[weather_desc][timestamp.month].append( - yvr_data['hourly_cfs'][timestamp.hour]) + yvr_data["hourly_cfs"][timestamp.hour] + ) except KeyError: mapping[weather_desc] = [ - [], [], [], [], [], [], [], [], [], [], [], [], [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], + [], ] mapping[weather_desc][timestamp.month].append( - yvr_data['hourly_cfs'][timestamp.hour]) + yvr_data["hourly_cfs"][timestamp.hour] + ) log.info('"{0}" added to mapping'.format(weather_desc)) @@ -184,8 +196,10 @@ def calc_mapping_averages(mapping): total_observations = sum(len(month) for month in months) if total_observations > AVERAGING_THRESHOLD: log.info( - 'using monthly averaging for {0} "{1}" observation(s)' - .format(total_observations, weather_desc)) + 'using monthly averaging for {0} "{1}" observation(s)'.format( + total_observations, weather_desc + ) + ) for i, month in enumerate(months): try: mapping[weather_desc][i] = sum(month) / len(month) @@ -194,12 +208,14 @@ def calc_mapping_averages(mapping): mapping[weather_desc].pop(0) else: log.info( - 'using all value averaging for {0} "{1}" observation(s)' - .format(total_observations, weather_desc)) + 'using all value averaging for {0} "{1}" observation(s)'.format( + total_observations, weather_desc + ) + ) mapping[weather_desc] = [ sum(sum(month) for month in months) / total_observations ] -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/cf_analysis/cf_hourlies.py b/cf_analysis/cf_hourlies.py index b499200..ce92d23 100644 --- a/cf_analysis/cf_hourlies.py +++ b/cf_analysis/cf_hourlies.py @@ -34,29 +34,29 @@ import yaml -EC_URL = 'http://www.climate.weatheroffice.gc.ca/climateData/bulkdata_e.html' +EC_URL = "http://www.climate.weatheroffice.gc.ca/climateData/bulkdata_e.html" START_YEAR = 2002 END_YEAR = 2012 STATION_ID = 889 # YVR -MAPPING_FILE = 'cloud_fraction_mapping.yaml' -HOURLY_FILE_ROOT = 'cf_hourly_yvr' +MAPPING_FILE = "cloud_fraction_mapping.yaml" +HOURLY_FILE_ROOT = "cf_hourly_yvr" root_log = logging.getLogger() -log = logging.getLogger('cf_hourlies') +log = logging.getLogger("cf_hourlies") logging.basicConfig(level=logging.DEBUG) -formatter = logging.Formatter('%(levelname)s:%(name)s:%(message)s') +formatter = logging.Formatter("%(levelname)s:%(name)s:%(message)s") console = logging.StreamHandler() console.setFormatter(formatter) log.addHandler(console) -disk = logging.FileHandler('cf_hourlies.log', mode='w') +disk = logging.FileHandler("cf_hourlies.log", mode="w") disk.setFormatter(formatter) log.addHandler(disk) root_log.addHandler(disk) log.propagate = False -with open(MAPPING_FILE, 'rt') as f: +with open(MAPPING_FILE, "rt") as f: mapping = yaml.safe_load(f.read()) @@ -67,41 +67,41 @@ def run(): for month in range(1, 13) ) request_params = { - 'timeframe': 1, # Daily - 'Prov': 'BC', - 'format': 'xml', - 'StationID': 889, # YVR - 'Day': 1, + "timeframe": 1, # Daily + "Prov": "BC", + "format": "xml", + "StationID": 889, # YVR + "Day": 1, } data = [] for data_month in data_months: ec_data = get_EC_data(data_month, request_params) - for record in ec_data.findall('stationdata'): - parts = [record.get(part) - for part in 'year month day hour'.split()] + for record in ec_data.findall("stationdata"): + parts = [record.get(part) for part in "year month day hour".split()] timestamp = datetime(*map(int, parts)) data.append((timestamp, read_cloud_fraction(timestamp, record))) patch_data(data) - hourly_file_name = ( - '{0}_{1}_{2}'.format(HOURLY_FILE_ROOT, START_YEAR, END_YEAR)) - with open(hourly_file_name, 'wt') as hourly_file: + hourly_file_name = "{0}_{1}_{2}".format(HOURLY_FILE_ROOT, START_YEAR, END_YEAR) + with open(hourly_file_name, "wt") as hourly_file: hourly_file.writelines(format_data(data)) def get_EC_data(data_month, request_params): - request_params.update({ - 'Year': data_month.year, - 'Month': data_month.month, - }) + request_params.update( + { + "Year": data_month.year, + "Month": data_month.month, + } + ) response = requests.get(EC_URL, params=request_params) - log.info('got meteo data for {0:%Y-%m}'.format(data_month)) + log.info("got meteo data for {0:%Y-%m}".format(data_month)) tree = ElementTree.parse(StringIO(response.content)) ec_data = tree.getroot() return ec_data def read_cloud_fraction(timestamp, record): - weather_desc = record.find('weather').text + weather_desc = record.find("weather").text try: cloud_fraction = mapping[weather_desc] except KeyError: @@ -110,9 +110,9 @@ def read_cloud_fraction(timestamp, record): cloud_fraction = [None] else: log.warning( - 'Unrecognized weather description: {0} at {1}; ' - 'cloud fraction set to 10' - .format(weather_desc, timestamp)) + "Unrecognized weather description: {0} at {1}; " + "cloud fraction set to 10".format(weather_desc, timestamp) + ) cloud_fraction = [10] if len(cloud_fraction) == 1: cloud_fraction = cloud_fraction[0] @@ -122,22 +122,20 @@ def read_cloud_fraction(timestamp, record): def patch_data(data): - """Patch missing data values by interpolation. - """ + """Patch missing data values by interpolation.""" gap_start = gap_end = None for i, value in enumerate(data): if value[1] is None: gap_start = i if gap_start is None else gap_start gap_end = i - log.debug('data patched for {0[0]}'.format(value)) + log.debug("data patched for {0[0]}".format(value)) elif gap_start is not None: interpolate_values(data, gap_start, gap_end) gap_start = gap_end = None def interpolate_values(data, gap_start, gap_end): - """Calculate values for missing data via linear interpolation. - """ + """Calculate values for missing data via linear interpolation.""" last_value = data[gap_start - 1][1] next_value = data[gap_end + 1][1] delta = (next_value - last_value) / (gap_end - gap_start + 2) @@ -163,14 +161,14 @@ def format_data(data): expressed as floats with 2 decimal place. """ for i in range(len(data) / 24): - item = data[i * 24:(i + 1) * 24] + item = data[i * 24 : (i + 1) * 24] timestamp = item[0][0] - line = '{0} {1:%Y %m %d} 42'.format(STATION_ID, timestamp) + line = "{0} {1:%Y %m %d} 42".format(STATION_ID, timestamp) for hour in item: - line += ' {0:.2f}'.format(hour[1]) - line += '\n' + line += " {0:.2f}".format(hour[1]) + line += "\n" yield line -if __name__ == '__main__': +if __name__ == "__main__": run() diff --git a/docs/Deployment.rst b/docs/Deployment.rst index d1a9dc5..9fe1850 100644 --- a/docs/Deployment.rst +++ b/docs/Deployment.rst @@ -123,4 +123,3 @@ Edit the :file:`run/config.yaml` file to point to the year's infile via the :kbd base_infile: 2015_bloomcast_infile.yaml ... - diff --git a/docs/DesignNotes.rst b/docs/DesignNotes.rst index ad3ba47..0b62173 100644 --- a/docs/DesignNotes.rst +++ b/docs/DesignNotes.rst @@ -183,4 +183,3 @@ This snippet of Python shows the transformation algorithm in detail:: # oceanography cross_wind = -cross_wind along_wind = -along_wind - diff --git a/docs/conf.py b/docs/conf.py index a22007a..b06e8d4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -16,194 +16,199 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx', 'sphinx.ext.viewcode'] +extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.viewcode"] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'SoG Bloomcast' -copyright = u'2021, Doug Latornell, Susan Allen' +project = "SoG Bloomcast" +copyright = "2021, Doug Latornell, Susan Allen" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.1' +version = "0.1" # The full version, including alpha/beta/rc tags. -release = '0.1dev' +release = "0.1dev" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'classic' +html_theme = "classic" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'SoGBloomcastdoc' +htmlhelp_basename = "SoGBloomcastdoc" # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). -#latex_paper_size = 'letter' +# latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). -#latex_font_size = '10pt' +# latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'SoGBloomcast.tex', u'SoG Bloomcast Documentation', - u'Doug Latornell, Susan Allen', 'manual'), + ( + "index", + "SoGBloomcast.tex", + "SoG Bloomcast Documentation", + "Doug Latornell, Susan Allen", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Additional stuff for the LaTeX preamble. -#latex_preamble = '' +# latex_preamble = '' # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -211,10 +216,15 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'sogbloomcast', u'SoG Bloomcast Documentation', - [u'Doug Latornell, Susan Allen'], 1) + ( + "index", + "sogbloomcast", + "SoG Bloomcast Documentation", + ["Doug Latornell, Susan Allen"], + 1, + ) ] # Example configuration for intersphinx: refer to the Python standard library. -#intersphinx_mapping = {'http://docs.python.org/': None} +# intersphinx_mapping = {'http://docs.python.org/': None} diff --git a/docs/index.rst b/docs/index.rst index c0d7d7a..1359f93 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -60,4 +60,3 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` - diff --git a/ensemble_analysis/ensemble_analysis.ipynb b/ensemble_analysis/ensemble_analysis.ipynb index c4700ee..9b5c787 100644 --- a/ensemble_analysis/ensemble_analysis.ipynb +++ b/ensemble_analysis/ensemble_analysis.ipynb @@ -839,4 +839,4 @@ "metadata": {} } ] -} \ No newline at end of file +} diff --git a/setup.py b/setup.py index 6c99fa4..4b0546f 100644 --- a/setup.py +++ b/setup.py @@ -21,30 +21,31 @@ python_classifiers = [ - 'Programming Language :: Python :: {0}'.format(py_version) - for py_version in ['3', '3.6', '3.7']] + "Programming Language :: Python :: {0}".format(py_version) + for py_version in ["3", "3.6", "3.7"] +] other_classifiers = [ - 'Development Status :: ' + __pkg_metadata__.DEV_STATUS, - 'License :: OSI Approved :: BSD License', - 'Programming Language :: Python :: Implementation :: CPython', - 'Operating System :: Unix', - 'Operating System :: MacOS :: MacOS X', - 'Environment :: Console', - 'Intended Audience :: Science/Research', - 'Intended Audience :: Education', + "Development Status :: " + __pkg_metadata__.DEV_STATUS, + "License :: OSI Approved :: BSD License", + "Programming Language :: Python :: Implementation :: CPython", + "Operating System :: Unix", + "Operating System :: MacOS :: MacOS X", + "Environment :: Console", + "Intended Audience :: Science/Research", + "Intended Audience :: Education", ] try: - long_description = open('README.rst', 'rt').read() + long_description = open("README.rst", "rt").read() except IOError: - long_description = '' + long_description = "" install_requires = [ - 'arrow', - 'BeautifulSoup4', - 'cliff', - 'matplotlib', - 'numpy', - 'PyYAML', - 'requests', + "arrow", + "BeautifulSoup4", + "cliff", + "matplotlib", + "numpy", + "PyYAML", + "requests", # Use `cd SOG; pip install -e .` to install SOG command processor # and its dependencies ] @@ -54,26 +55,27 @@ version=__pkg_metadata__.VERSION, description=__pkg_metadata__.DESCRIPTION, long_description=long_description, - author='Doug Latornell', - author_email='djl@douglatornell.ca', - url='http://eos.ubc.ca/~sallen/SoG-bloomcast/results.html', + author="Doug Latornell", + author_email="djl@douglatornell.ca", + url="http://eos.ubc.ca/~sallen/SoG-bloomcast/results.html", download_url=( - 'https://bitbucket.org/douglatornell/sog-bloomcast/get/default.tar.gz'), - license='Apache License, Version 2.0', + "https://bitbucket.org/douglatornell/sog-bloomcast/get/default.tar.gz" + ), + license="Apache License, Version 2.0", classifiers=python_classifiers + other_classifiers, - platforms=['MacOS X', 'Linux'], + platforms=["MacOS X", "Linux"], install_requires=install_requires, packages=setuptools.find_packages(), include_package_data=True, zip_safe=False, entry_points={ # The bloomcast command: - 'console_scripts': [ - 'bloomcast = bloomcast.main:main', + "console_scripts": [ + "bloomcast = bloomcast.main:main", ], # Sub-command plug-ins: - 'bloomcast.app': [ - 'ensemble = bloomcast.ensemble:Ensemble', + "bloomcast.app": [ + "ensemble = bloomcast.ensemble:Ensemble", ], }, ) diff --git a/tests/test_bloomcast.py b/tests/test_bloomcast.py index d3bfe9b..204a8a6 100644 --- a/tests/test_bloomcast.py +++ b/tests/test_bloomcast.py @@ -12,5 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for bloomcast modules. -""" +"""Unit tests for bloomcast modules.""" diff --git a/tests/test_ensemble.py b/tests/test_ensemble.py index ab083bd..2363092 100644 --- a/tests/test_ensemble.py +++ b/tests/test_ensemble.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for SoG-bloomcast ensemble module. -""" +"""Unit tests for SoG-bloomcast ensemble module.""" import datetime from unittest.mock import ( Mock, @@ -30,76 +29,77 @@ @pytest.fixture def ensemble(): import bloomcast.ensemble + return bloomcast.ensemble.Ensemble(Mock(spec=cliff.app.App), []) @pytest.fixture def ensemble_module(): import bloomcast.ensemble + return bloomcast.ensemble -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def ensemble_config(): config = Mock( ensemble=Mock( max_concurrent_jobs=32, - base_infile='foo.yaml', + base_infile="foo.yaml", start_year=1981, end_year=1981, forcing_data_file_roots={ - 'wind': 'wind_data', - 'air_temperature': 'AT_data', - 'cloud_fraction': 'CF_data', - 'relative_humidity': 'Hum_data', - 'major_river': 'major_river_data', - 'minor_river': 'minor_river_data', - } + "wind": "wind_data", + "air_temperature": "AT_data", + "cloud_fraction": "CF_data", + "relative_humidity": "Hum_data", + "major_river": "major_river_data", + "minor_river": "minor_river_data", + }, ), - std_phys_ts_outfile='std_phys_bloomcast.out', - user_phys_ts_outfile='user_phys_bloomcast.out', - std_bio_ts_outfile='std_bio_bloomcast.out', - user_bio_ts_outfile='user_bio_bloomcast.out', - std_chem_ts_outfile='std_chem_bloomcast.out', - user_chem_ts_outfile='user_chem_bloomcast.out', - profiles_outfile_base='profiles/bloomcast', - user_profiles_outfile_base='profiles/user_bloomcast', - halocline_outfile='profiles/halo_bloomcast.out', - Hoffmueller_profiles_outfile='hoff_bloomcast.out', - user_Hoffmueller_profiles_outfile='user_hoff_bloomcast.out', + std_phys_ts_outfile="std_phys_bloomcast.out", + user_phys_ts_outfile="user_phys_bloomcast.out", + std_bio_ts_outfile="std_bio_bloomcast.out", + user_bio_ts_outfile="user_bio_bloomcast.out", + std_chem_ts_outfile="std_chem_bloomcast.out", + user_chem_ts_outfile="user_chem_bloomcast.out", + profiles_outfile_base="profiles/bloomcast", + user_profiles_outfile_base="profiles/user_bloomcast", + halocline_outfile="profiles/halo_bloomcast.out", + Hoffmueller_profiles_outfile="hoff_bloomcast.out", + user_Hoffmueller_profiles_outfile="user_hoff_bloomcast.out", ) return config def test_get_parser(ensemble): - parser = ensemble.get_parser('bloomcast ensemble') - assert parser.prog == 'bloomcast ensemble' + parser = ensemble.get_parser("bloomcast ensemble") + assert parser.prog == "bloomcast ensemble" + +@pytest.mark.usefixtures("ensemble") +class TestEnsembleTakeAction: + """Unit tests for take_action method of Ensemble class.""" -@pytest.mark.usefixtures('ensemble') -class TestEnsembleTakeAction(): - """Unit tests for take_action method of Ensemble class. - """ - @patch('bloomcast.ensemble.utils.Config') + @patch("bloomcast.ensemble.utils.Config") def test_get_forcing_data_conflicts_w_data_date(self, m_config, ensemble): parsed_args = Mock( - config_file='config.yaml', + config_file="config.yaml", data_date=None, ) m_config.return_value = Mock(get_forcing_data=False) ensemble.log = Mock() - with patch('bloomcast.ensemble.configure_logging'): + with patch("bloomcast.ensemble.configure_logging"): ensemble.take_action(parsed_args) ensemble.log.debug.assert_called_once_with( - 'This will not end well: get_forcing_data=False ' - 'and data_date=None' + "This will not end well: get_forcing_data=False " "and data_date=None" ) - @patch('bloomcast.ensemble.utils.Config') - @patch('bloomcast.ensemble.arrow.now', return_value=arrow.get(2014, 3, 12)) + @patch("bloomcast.ensemble.utils.Config") + @patch("bloomcast.ensemble.arrow.now", return_value=arrow.get(2014, 3, 12)) def test_no_river_flow_data_by_date(self, m_now, m_config, ensemble): parsed_args = Mock( - config_file='config.yaml', + config_file="config.yaml", data_date=None, ) m_config.return_value = Mock( @@ -107,19 +107,19 @@ def test_no_river_flow_data_by_date(self, m_now, m_config, ensemble): run_start_date=datetime.datetime(2012, 9, 19), ) ensemble.log = Mock() - with patch('bloomcast.ensemble.configure_logging'): + with patch("bloomcast.ensemble.configure_logging"): ensemble.take_action(parsed_args) ensemble.log.error.assert_called_once_with( - 'A bloomcast run starting 2012-09-19 cannot be done today ' - 'because there are no river flow data available prior to ' - '2012-09-12' + "A bloomcast run starting 2012-09-19 cannot be done today " + "because there are no river flow data available prior to " + "2012-09-12" ) - @patch('bloomcast.ensemble.utils.Config') - @patch('bloomcast.ensemble.arrow.now', return_value=arrow.get(2014, 3, 12)) + @patch("bloomcast.ensemble.utils.Config") + @patch("bloomcast.ensemble.arrow.now", return_value=arrow.get(2014, 3, 12)) def test_no_new_wind_data(self, m_now, m_config, ensemble): parsed_args = Mock( - config_file='config.yaml', + config_file="config.yaml", data_date=None, ) m_config.return_value = Mock( @@ -127,192 +127,208 @@ def test_no_new_wind_data(self, m_now, m_config, ensemble): run_start_date=datetime.datetime(2013, 9, 19), ) ensemble.log = Mock() - p_config_logging = patch('bloomcast.ensemble.configure_logging') + p_config_logging = patch("bloomcast.ensemble.configure_logging") def get_forcing_data(config, log): config.data_date = arrow.get(2014, 3, 12) raise ValueError + p_get_forcing_data = patch( - 'bloomcast.ensemble.get_forcing_data', + "bloomcast.ensemble.get_forcing_data", side_effect=get_forcing_data, ) with p_config_logging, p_get_forcing_data: ensemble.take_action(parsed_args) ensemble.log.info.assert_called_once_with( - 'Wind data date 2014-03-12 is unchanged since last run' + "Wind data date 2014-03-12 is unchanged since last run" ) - @patch('bloomcast.ensemble.yaml') - @patch('bloomcast.ensemble.utils.Config') + @patch("bloomcast.ensemble.yaml") + @patch("bloomcast.ensemble.utils.Config") def test_create_infile_edits_forcing_data( - self, m_config, m_yaml, ensemble, ensemble_config, + self, + m_config, + m_yaml, + ensemble, + ensemble_config, ): ensemble.config = ensemble_config ensemble.log = Mock() - with patch('bloomcast.ensemble.open', mock_open(), create=True): + with patch("bloomcast.ensemble.open", mock_open(), create=True): ensemble._create_infile_edits() - result = m_yaml.safe_dump.call_args[0][0]['forcing_data'] - assert result['avg_historical_wind_file']['value'] == 'wind_data_8081' + result = m_yaml.safe_dump.call_args[0][0]["forcing_data"] + assert result["avg_historical_wind_file"]["value"] == "wind_data_8081" expected_keys = ( - 'avg_historical_wind_file avg_historical_air_temperature_file ' - 'avg_historical_cloud_file avg_historical_humidity_file ' - 'avg_historical_major_river_file avg_historical_minor_river_file' - .split()) + "avg_historical_wind_file avg_historical_air_temperature_file " + "avg_historical_cloud_file avg_historical_humidity_file " + "avg_historical_major_river_file avg_historical_minor_river_file".split() + ) for key in expected_keys: - assert result[key]['value'] is not None + assert result[key]["value"] is not None ensemble.log.debug.assert_called_once_with( - 'wrote infile edit file foo_8081.yaml' + "wrote infile edit file foo_8081.yaml" ) - @patch('bloomcast.ensemble.yaml') - @patch('bloomcast.ensemble.utils.Config') + @patch("bloomcast.ensemble.yaml") + @patch("bloomcast.ensemble.utils.Config") def test_create_infile_edits_timeseries_results( self, m_config, m_yaml, ensemble, ensemble_config ): ensemble.config = ensemble_config ensemble.log = Mock() - with patch('bloomcast.ensemble.open', mock_open(), create=True): + with patch("bloomcast.ensemble.open", mock_open(), create=True): ensemble._create_infile_edits() - result = m_yaml.safe_dump.call_args[0][0]['timeseries_results'] - assert result['std_physics']['value'] == 'std_phys_bloomcast.out_8081' + result = m_yaml.safe_dump.call_args[0][0]["timeseries_results"] + assert result["std_physics"]["value"] == "std_phys_bloomcast.out_8081" expected_keys = ( - 'std_physics user_physics ' - 'std_biology user_biology ' - 'std_chemistry user_chemistry' - .split()) + "std_physics user_physics " + "std_biology user_biology " + "std_chemistry user_chemistry".split() + ) for key in expected_keys: - assert result[key]['value'] is not None + assert result[key]["value"] is not None ensemble.log.debug.assert_called_once_with( - 'wrote infile edit file foo_8081.yaml' + "wrote infile edit file foo_8081.yaml" ) - @patch('bloomcast.ensemble.yaml') - @patch('bloomcast.ensemble.utils.Config') + @patch("bloomcast.ensemble.yaml") + @patch("bloomcast.ensemble.utils.Config") def test_create_infile_edits_profiles_results( self, m_config, m_yaml, ensemble, ensemble_config ): ensemble.config = ensemble_config ensemble.log = Mock() - with patch('bloomcast.ensemble.open', mock_open(), create=True): + with patch("bloomcast.ensemble.open", mock_open(), create=True): ensemble._create_infile_edits() - result = m_yaml.safe_dump.call_args[0][0]['profiles_results'] - expected = 'profiles/bloomcast_8081' - assert result['profile_file_base']['value'] == expected + result = m_yaml.safe_dump.call_args[0][0]["profiles_results"] + expected = "profiles/bloomcast_8081" + assert result["profile_file_base"]["value"] == expected expected_keys = ( - 'profile_file_base user_profile_file_base ' - 'halocline_file ' - 'hoffmueller_file user_hoffmueller_file' - .split()) + "profile_file_base user_profile_file_base " + "halocline_file " + "hoffmueller_file user_hoffmueller_file".split() + ) for key in expected_keys: - assert result[key]['value'] is not None + assert result[key]["value"] is not None ensemble.log.debug.assert_called_once_with( - 'wrote infile edit file foo_8081.yaml' + "wrote infile edit file foo_8081.yaml" ) - @patch('bloomcast.ensemble.yaml') - @patch('bloomcast.ensemble.utils.Config') + @patch("bloomcast.ensemble.yaml") + @patch("bloomcast.ensemble.utils.Config") def test_create_infile_edits_sets_edit_files_list_attr( self, m_config, m_yaml, ensemble, ensemble_config ): ensemble.config = ensemble_config ensemble.config.ensemble.end_year = 1982 ensemble.log = Mock() - with patch('bloomcast.ensemble.open', mock_open(), create=True): + with patch("bloomcast.ensemble.open", mock_open(), create=True): ensemble._create_infile_edits() assert ensemble.edit_files == [ - (1981, 'foo_8081.yaml', '_8081'), - (1982, 'foo_8182.yaml', '_8182'), + (1981, "foo_8081.yaml", "_8081"), + (1982, "foo_8182.yaml", "_8182"), ] - @patch('bloomcast.ensemble.yaml') + @patch("bloomcast.ensemble.yaml") def test_create_batch_description(self, m_yaml, ensemble, ensemble_config): ensemble.config = ensemble_config ensemble.config.ensemble.end_year = 1982 ensemble.log = Mock() ensemble.edit_files = [ - (1981, 'foo_8081.yaml', '_8081'), - (1982, 'foo_8182.yaml', '_8182'), + (1981, "foo_8081.yaml", "_8081"), + (1982, "foo_8182.yaml", "_8182"), ] - with patch('bloomcast.ensemble.open', mock_open(), create=True): + with patch("bloomcast.ensemble.open", mock_open(), create=True): ensemble._create_batch_description() result = m_yaml.safe_dump.call_args[0][0] expected = ensemble.config.ensemble.max_concurrent_jobs - assert result['max_concurrent_jobs'] == expected - assert result['SOG_executable'] == ensemble.config.SOG_executable - assert result['base_infile'] == ensemble.config.ensemble.base_infile - assert result['jobs'] == [ - {''.join(('bloomcast', suffix)): { - 'edit_files': [filename], - }} + assert result["max_concurrent_jobs"] == expected + assert result["SOG_executable"] == ensemble.config.SOG_executable + assert result["base_infile"] == ensemble.config.ensemble.base_infile + assert result["jobs"] == [ + { + "".join(("bloomcast", suffix)): { + "edit_files": [filename], + } + } for year, filename, suffix in ensemble.edit_files ] ensemble.log.debug.assert_called_once_with( - 'wrote ensemble batch description file: ' - 'bloomcast_ensemble_jobs.yaml' + "wrote ensemble batch description file: " "bloomcast_ensemble_jobs.yaml" ) - @patch('bloomcast.ensemble.SOGcommand') + @patch("bloomcast.ensemble.SOGcommand") def test_run_SOG_batch_skip(self, m_SOGcommand, ensemble, ensemble_config): ensemble.config = ensemble_config ensemble.config.run_SOG = False ensemble.log = Mock() ensemble._run_SOG_batch() - ensemble.log.info.assert_called_once_with('Skipped running SOG') + ensemble.log.info.assert_called_once_with("Skipped running SOG") assert not m_SOGcommand.api.batch.called - @patch('bloomcast.ensemble.SOGcommand') + @patch("bloomcast.ensemble.SOGcommand") def test_run_SOG_batch(self, m_SOGcommand, ensemble, ensemble_config): ensemble.config = ensemble_config ensemble.config.run_SOG = True ensemble.log = Mock() m_SOGcommand.api.batch.return_value = 0 ensemble._run_SOG_batch() - m_SOGcommand.api.batch.assert_called_once_with( - 'bloomcast_ensemble_jobs.yaml') + m_SOGcommand.api.batch.assert_called_once_with("bloomcast_ensemble_jobs.yaml") ensemble.log.info.assert_called_once_with( - 'ensemble batch SOG runs completed with return code 0') + "ensemble batch SOG runs completed with return code 0" + ) - @patch('bloomcast.utils.SOG_Timeseries') - def test_load_biology_timeseries_instances(self, m_SOG_ts, ensemble, ensemble_config): + @patch("bloomcast.utils.SOG_Timeseries") + def test_load_biology_timeseries_instances( + self, m_SOG_ts, ensemble, ensemble_config + ): ensemble.config = ensemble_config - ensemble.edit_files = [(1981, 'foo_8081.yaml', '_8081')] + ensemble.edit_files = [(1981, "foo_8081.yaml", "_8081")] ensemble._load_biology_timeseries() expected = [ - mock.call('std_bio_bloomcast.out_8081'), - mock.call('std_bio_bloomcast.out_8081'), + mock.call("std_bio_bloomcast.out_8081"), + mock.call("std_bio_bloomcast.out_8081"), ] assert m_SOG_ts.call_args_list == expected - @patch('bloomcast.utils.SOG_Timeseries') - def test_load_biology_timeseries_read_nitrate(self, m_SOG_ts, ensemble, ensemble_config): + @patch("bloomcast.utils.SOG_Timeseries") + def test_load_biology_timeseries_read_nitrate( + self, m_SOG_ts, ensemble, ensemble_config + ): ensemble.config = ensemble_config - ensemble.edit_files = [(1981, 'foo_8081.yaml', '_8081')] + ensemble.edit_files = [(1981, "foo_8081.yaml", "_8081")] ensemble._load_biology_timeseries() call = ensemble.nitrate_ts[1981].read_data.call_args_list[0] - assert call == mock.call('time', '3 m avg nitrate concentration') + assert call == mock.call("time", "3 m avg nitrate concentration") - @patch('bloomcast.utils.SOG_Timeseries') - def test_load_biology_timeseries_read_diatoms(self, m_SOG_ts, ensemble, ensemble_config): + @patch("bloomcast.utils.SOG_Timeseries") + def test_load_biology_timeseries_read_diatoms( + self, m_SOG_ts, ensemble, ensemble_config + ): ensemble.config = ensemble_config - ensemble.edit_files = [(1981, 'foo_8081.yaml', '_8081')] + ensemble.edit_files = [(1981, "foo_8081.yaml", "_8081")] ensemble._load_biology_timeseries() call = ensemble.diatoms_ts[1981].read_data.call_args_list[1] - assert call == mock.call('time', '3 m avg micro phytoplankton biomass') + assert call == mock.call("time", "3 m avg micro phytoplankton biomass") - @patch('bloomcast.utils.SOG_Timeseries') - def test_load_biology_timeseries_mpl_dates(self, m_SOG_ts, ensemble, ensemble_config): + @patch("bloomcast.utils.SOG_Timeseries") + def test_load_biology_timeseries_mpl_dates( + self, m_SOG_ts, ensemble, ensemble_config + ): ensemble.config = ensemble_config - ensemble.edit_files = [(1981, 'foo_8081.yaml', '_8081')] + ensemble.edit_files = [(1981, "foo_8081.yaml", "_8081")] ensemble._load_biology_timeseries() ensemble.nitrate_ts[1981].calc_mpl_dates.assert_called_with( - ensemble.config.run_start_date) + ensemble.config.run_start_date + ) ensemble.diatoms_ts[1981].calc_mpl_dates.assert_called_with( - ensemble.config.run_start_date) + ensemble.config.run_start_date + ) - @patch('bloomcast.utils.SOG_Timeseries') + @patch("bloomcast.utils.SOG_Timeseries") def test_load_biology_timeseries_copies(self, m_SOG_ts, ensemble, ensemble_config): ensemble.config = ensemble_config - ensemble.edit_files = [(1981, 'foo_8081.yaml', '_8081')] + ensemble.edit_files = [(1981, "foo_8081.yaml", "_8081")] ensemble._load_biology_timeseries() assert ensemble.nitrate == ensemble.nitrate_ts assert ensemble.nitrate is not ensemble.nitrate_ts @@ -322,7 +338,7 @@ def test_load_biology_timeseries_copies(self, m_SOG_ts, ensemble, ensemble_confi def test_two_yr_suffix(ensemble_module): suffix = ensemble_module.two_yr_suffix(1981) - assert suffix == '_8081' + assert suffix == "_8081" def test_find_member_single_year_day_match(ensemble_module): diff --git a/tests/test_meteo.py b/tests/test_meteo.py index 5766458..a0505ab 100644 --- a/tests/test_meteo.py +++ b/tests/test_meteo.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for SoG-bloomcast meteo module. -""" +"""Unit tests for SoG-bloomcast meteo module.""" import datetime from unittest.mock import Mock @@ -23,49 +22,57 @@ @pytest.fixture def meteo(): from bloomcast.meteo import MeteoProcessor - return MeteoProcessor(Mock(name='config')) + return MeteoProcessor(Mock(name="config")) + + +class TestMeteoProcessor: + """Unit tests for MeteoProcessor object.""" -class TestMeteoProcessor(): - """Unit tests for MeteoProcessor object. - """ def test_read_cloud_fraction_single_avg(self, meteo): - """read_cloud_fraction returns expected value for single avg CF list - """ + """read_cloud_fraction returns expected value for single avg CF list""" meteo.config.climate.meteo.cloud_fraction_mapping = { - 'Drizzle': [9.9675925925925934], + "Drizzle": [9.9675925925925934], } - record = Mock(name='record') - record.find().text = 'Drizzle' + record = Mock(name="record") + record.find().text = "Drizzle" cloud_faction = meteo.read_cloud_fraction(record) assert cloud_faction == 9.9675925925925934 def test_read_cloud_fraction_monthly_avg(self, meteo): - """read_cloud_fraction returns expected value for monthly avg CF list - """ + """read_cloud_fraction returns expected value for monthly avg CF list""" meteo.config.climate.meteo.cloud_fraction_mapping = { - 'Fog': [ - 9.6210045662100452, 9.3069767441860467, 9.5945945945945947, - 9.5, 9.931034482758621, 10.0, 9.7777777777777786, - 9.6999999999999993, 7.8518518518518521, 8.9701492537313428, - 9.2686980609418281, 9.0742358078602621] + "Fog": [ + 9.6210045662100452, + 9.3069767441860467, + 9.5945945945945947, + 9.5, + 9.931034482758621, + 10.0, + 9.7777777777777786, + 9.6999999999999993, + 7.8518518518518521, + 8.9701492537313428, + 9.2686980609418281, + 9.0742358078602621, + ] } - record = Mock(name='record') - record.find().text = 'Fog' + record = Mock(name="record") + record.find().text = "Fog" def mock_timestamp_data(part): - parts = {'year': 2012, 'month': 4, 'day': 1, 'hour': 12} + parts = {"year": 2012, "month": 4, "day": 1, "hour": 12} return parts[part] + record.get = mock_timestamp_data cloud_faction = meteo.read_cloud_fraction(record) assert cloud_faction == 9.5 def test_format_data(self, meteo): - """format_data generator returns formatted forcing data file line - """ - meteo.config.climate.meteo.station_id = '889' - meteo.data['air_temperature'] = [ - (datetime.datetime(2011, 9, 25, i, 0, 0), 215.0) - for i in range(24)] - line = next(meteo.format_data('air_temperature')) - assert line == '889 2011 09 25 42' + ' 215.00' * 24 + '\n' + """format_data generator returns formatted forcing data file line""" + meteo.config.climate.meteo.station_id = "889" + meteo.data["air_temperature"] = [ + (datetime.datetime(2011, 9, 25, i, 0, 0), 215.0) for i in range(24) + ] + line = next(meteo.format_data("air_temperature")) + assert line == "889 2011 09 25 42" + " 215.00" * 24 + "\n" diff --git a/tests/test_rivers.py b/tests/test_rivers.py index d17c1cd..3b70c3c 100644 --- a/tests/test_rivers.py +++ b/tests/test_rivers.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for SoG-bloomcast rivers module. -""" +"""Unit tests for SoG-bloomcast rivers module.""" import datetime from unittest.mock import ( Mock, @@ -28,38 +27,39 @@ @pytest.fixture def processor(): from bloomcast.rivers import RiversProcessor - return RiversProcessor(Mock(name='config')) + return RiversProcessor(Mock(name="config")) + + +class TestRiverProcessor: + """Uni tests for RiverProcessor object.""" -class TestRiverProcessor(): - """Uni tests for RiverProcessor object. - """ def test_date_params(self, processor): - """_date_params handles month-end rollover correctly - """ + """_date_params handles month-end rollover correctly""" processor.config.data_date = arrow.get(2011, 11, 30) expected = { - 'startDate': '2011-01-01', - 'endDate': '2011-12-01', + "startDate": "2011-01-01", + "endDate": "2011-12-01", } assert processor._date_params(2011) == expected def test_process_data_1_row(self, processor): - """process_data produces expected result for 1 row of data - """ + """process_data produces expected result for 1 row of data""" test_data = [ - '', - ' ', + "
", + " ", ' ', ' ', - ' ', - ' ', - ' ', - '
2022-03-16 15:40:004,200PRELIMINARYUNSPECIFIED
', + " PRELIMINARY", + " UNSPECIFIED", + " ", + "", ] - processor.raw_data = bs4.BeautifulSoup(''.join(test_data), features="html.parser") - processor.process_data('major') - assert processor.data['major'] == [(datetime.date(2022, 3, 16), 4200.0)] + processor.raw_data = bs4.BeautifulSoup( + "".join(test_data), features="html.parser" + ) + processor.process_data("major") + assert processor.data["major"] == [(datetime.date(2022, 3, 16), 4200.0)] def test_process_data_minor_river_scaling(self, processor): """process_data produces expected result for scaled minor river @@ -67,218 +67,225 @@ def test_process_data_minor_river_scaling(self, processor): with scaled Nanaimo River values for 2021 predictions """ test_data = [ - '', - ' ', + "
", + " ", ' ', ' ', - ' ', - ' ', - ' ', - '
2022-03-16 15:40:0010PRELIMINARYUNSPECIFIED
', + " PRELIMINARY", + " UNSPECIFIED", + " ", + "", ] - processor.raw_data = bs4.BeautifulSoup(''.join(test_data), features="html.parser") - processor.process_data('major', 0.351) - assert processor.data['major'] == [(datetime.date(2022, 3, 16), 3.51)] + processor.raw_data = bs4.BeautifulSoup( + "".join(test_data), features="html.parser" + ) + processor.process_data("major", 0.351) + assert processor.data["major"] == [(datetime.date(2022, 3, 16), 3.51)] def test_process_data_2_rows_1_day(self, processor): - """process_data produces result for 2 rows of data from same day - """ + """process_data produces result for 2 rows of data from same day""" test_data = [ - '', - ' ', + "
", + " ", ' ', ' ', - ' ', - ' ', - ' ', - ' ', + " ", + " ", + " ", + " ", ' ', ' ', - ' ', - ' ', - ' ', - '
2022-03-16 21:11:004,200PRELIMINARYUNSPECIFIED
PRELIMINARYUNSPECIFIED
2022-03-16 21:35:004,400PRELIMINARYUNSPECIFIED
', + " PRELIMINARY", + " UNSPECIFIED", + " ", + "", ] - processor.raw_data = bs4.BeautifulSoup(''.join(test_data), features="html.parser") - processor.process_data('major') - assert processor.data['major'] == [(datetime.date(2022, 3, 16), 4300.0)] + processor.raw_data = bs4.BeautifulSoup( + "".join(test_data), features="html.parser" + ) + processor.process_data("major") + assert processor.data["major"] == [(datetime.date(2022, 3, 16), 4300.0)] def test_process_data_2_rows_2_days(self, processor): - """process_data produces expected result for 2 rows of data from 2 days - """ + """process_data produces expected result for 2 rows of data from 2 days""" test_data = [ - '', - ' ', + "
", + " ", ' ', ' ', - ' ', - ' ', - ' ', - ' ', + " ", + " ", + " ", + " ", ' ', ' ', - ' ', - ' ', - ' ', - '
2022-03-15 21:11:004,200PRELIMINARYUNSPECIFIED
PRELIMINARYUNSPECIFIED
2022-03-16 21:35:004,400PRELIMINARYUNSPECIFIED
', + " PRELIMINARY", + " UNSPECIFIED", + " ", + "", ] - processor.raw_data = bs4.BeautifulSoup(''.join(test_data), features="html.parser") - processor.process_data('major') + processor.raw_data = bs4.BeautifulSoup( + "".join(test_data), features="html.parser" + ) + processor.process_data("major") expected = [ (datetime.date(2022, 3, 15), 4200.0), (datetime.date(2022, 3, 16), 4400.0), ] - assert processor.data['major'] == expected + assert processor.data["major"] == expected def test_process_data_4_rows_2_days(self, processor): - """process_data produces expected result for 4 rows of data from 2 days - """ + """process_data produces expected result for 4 rows of data from 2 days""" test_data = [ - '', - ' ', + "
", + " ", ' ', ' ', - ' ', - ' ', - ' ', - ' ', + " ", + " ", + " ", + " ", ' ', ' ', - ' ', - ' ', - ' ', + " ", + " ", + " ", ' ', ' ', - ' ', - ' ', - ' ', - ' ', + " ", + " ", + " ", + " ", ' ', ' ', - ' ', - ' ', - ' ', - '
2022-03-15 21:11:004,200PRELIMINARYUNSPECIFIED
PRELIMINARYUNSPECIFIED
2022-03-15 21:35:004,400PRELIMINARYUNSPECIFIED
PRELIMINARYUNSPECIFIED
2022-03-16 21:11:003,200PRELIMINARYUNSPECIFIED
PRELIMINARYUNSPECIFIED
2022-03-16 21:35:003,400PRELIMINARYUNSPECIFIED
', + " PRELIMINARY", + " UNSPECIFIED", + " ", + "", ] - processor.raw_data = bs4.BeautifulSoup(''.join(test_data), features="html.parser") - processor.process_data('major') + processor.raw_data = bs4.BeautifulSoup( + "".join(test_data), features="html.parser" + ) + processor.process_data("major") expected = [ (datetime.date(2022, 3, 15), 4300.0), (datetime.date(2022, 3, 16), 3300.0), ] - assert processor.data['major'] == expected + assert processor.data["major"] == expected def test_format_data(self, processor): - """format_data generator returns formatted forcing data file line - """ - processor.data['major'] = [ - (datetime.date(2011, 9, 27), 4200.0) - ] - line = next(processor.format_data('major')) - assert line == '2011 09 27 4.200000e+03\n' + """format_data generator returns formatted forcing data file line""" + processor.data["major"] = [(datetime.date(2011, 9, 27), 4200.0)] + line = next(processor.format_data("major")) + assert line == "2011 09 27 4.200000e+03\n" def test_patch_data_1_day_gap(self, processor): - """patch_data correctly flags 1 day gap in data for interpolation - """ - processor.data['major'] = [ + """patch_data correctly flags 1 day gap in data for interpolation""" + processor.data["major"] = [ (datetime.date(2011, 10, 23), 4300.0), (datetime.date(2011, 10, 25), 4500.0), ] - processor.interpolate_values = Mock(name='interpolate_values') - with patch('bloomcast.rivers.log') as mock_log: - processor.patch_data('major') + processor.interpolate_values = Mock(name="interpolate_values") + with patch("bloomcast.rivers.log") as mock_log: + processor.patch_data("major") expected = (datetime.date(2011, 10, 24), None) - assert processor.data['major'][1] == expected + assert processor.data["major"][1] == expected expected = [ - (('major river data patched for 2011-10-24',),), - (('1 major river data values patched; ' - 'see debug log on disk for details',),), + (("major river data patched for 2011-10-24",),), + ( + ( + "1 major river data values patched; " + "see debug log on disk for details", + ), + ), ] assert mock_log.debug.call_args_list == expected - processor.interpolate_values.assert_called_once_with( - 'major', 1, 1) + processor.interpolate_values.assert_called_once_with("major", 1, 1) def test_patch_data_2_day_gap(self, processor): - """patch_data correctly flags 2 day gap in data for interpolation - """ - processor.data['major'] = [ + """patch_data correctly flags 2 day gap in data for interpolation""" + processor.data["major"] = [ (datetime.date(2011, 10, 23), 4300.0), (datetime.date(2011, 10, 26), 4600.0), ] - processor.interpolate_values = Mock(name='interpolate_values') - with patch('bloomcast.rivers.log') as mock_log: - processor.patch_data('major') + processor.interpolate_values = Mock(name="interpolate_values") + with patch("bloomcast.rivers.log") as mock_log: + processor.patch_data("major") expected = [ (datetime.date(2011, 10, 24), None), (datetime.date(2011, 10, 25), None), ] - assert processor.data['major'][1:3] == expected + assert processor.data["major"][1:3] == expected expected = [ - (('major river data patched for 2011-10-24',),), - (('major river data patched for 2011-10-25',),), - (('2 major river data values patched; ' - 'see debug log on disk for details',),), + (("major river data patched for 2011-10-24",),), + (("major river data patched for 2011-10-25",),), + ( + ( + "2 major river data values patched; " + "see debug log on disk for details", + ), + ), ] assert mock_log.debug.call_args_list == expected - processor.interpolate_values.assert_called_once_with( - 'major', 1, 2) + processor.interpolate_values.assert_called_once_with("major", 1, 2) def test_patch_data_2_gaps(self, processor): - """patch_data correctly flags 2 gaps in data for interpolation - """ - processor.data['major'] = [ + """patch_data correctly flags 2 gaps in data for interpolation""" + processor.data["major"] = [ (datetime.date(2011, 10, 23), 4300.0), (datetime.date(2011, 10, 25), 4500.0), (datetime.date(2011, 10, 26), 4500.0), (datetime.date(2011, 10, 29), 4200.0), ] - processor.interpolate_values = Mock(name='interpolate_values') - with patch('bloomcast.rivers.log') as mock_log: - processor.patch_data('major') + processor.interpolate_values = Mock(name="interpolate_values") + with patch("bloomcast.rivers.log") as mock_log: + processor.patch_data("major") expected = (datetime.date(2011, 10, 24), None) - assert processor.data['major'][1] == expected + assert processor.data["major"][1] == expected expected = [ (datetime.date(2011, 10, 27), None), (datetime.date(2011, 10, 28), None), ] - assert processor.data['major'][4:6] == expected + assert processor.data["major"][4:6] == expected expected = [ - (('major river data patched for 2011-10-24',),), - (('major river data patched for 2011-10-27',),), - (('major river data patched for 2011-10-28',),), - (('3 major river data values patched; ' - 'see debug log on disk for details',),), + (("major river data patched for 2011-10-24",),), + (("major river data patched for 2011-10-27",),), + (("major river data patched for 2011-10-28",),), + ( + ( + "3 major river data values patched; " + "see debug log on disk for details", + ), + ), ] assert mock_log.debug.call_args_list == expected - expected = [(('major', 1, 1),), (('major', 4, 5),)] + expected = [(("major", 1, 1),), (("major", 4, 5),)] assert processor.interpolate_values.call_args_list == expected def test_interpolate_values_1_day_gap(self, processor): - """interpolate_values interpolates value for 1 day gap in data - """ + """interpolate_values interpolates value for 1 day gap in data""" processor.data = {} - processor.data['major'] = [ + processor.data["major"] = [ (datetime.date(2011, 10, 23), 4300.0), (datetime.date(2011, 10, 24), None), (datetime.date(2011, 10, 25), 4500.0), ] - processor.interpolate_values('major', 1, 1) + processor.interpolate_values("major", 1, 1) expected = (datetime.date(2011, 10, 24), 4400.0) - assert processor.data['major'][1] == expected + assert processor.data["major"][1] == expected def test_interpolate_values_2_day_gap(self, processor): - """interpolate_values interpolates value for 2 day gap in data - """ + """interpolate_values interpolates value for 2 day gap in data""" processor.data = {} - processor.data['major'] = [ + processor.data["major"] = [ (datetime.date(2011, 10, 23), 4300.0), (datetime.date(2011, 10, 24), None), (datetime.date(2011, 10, 25), None), (datetime.date(2011, 10, 26), 4600.0), ] - processor.interpolate_values('major', 1, 2) + processor.interpolate_values("major", 1, 2) expected = [ (datetime.date(2011, 10, 24), 4400.0), (datetime.date(2011, 10, 25), 4500.0), ] - assert processor.data['major'][1:3] == expected + assert processor.data["major"][1:3] == expected diff --git a/tests/test_utils.py b/tests/test_utils.py index b0045dd..2973dfc 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for SoG-bloomcast utils module. -""" +"""Unit tests for SoG-bloomcast utils module.""" import datetime from unittest.mock import ( DEFAULT, @@ -27,82 +26,78 @@ @pytest.fixture def config(): from bloomcast.utils import Config + return Config() -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def config_dict(): config_dict = { - 'get_forcing_data': None, - 'run_SOG': None, - 'SOG_executable': None, - 'html_results': None, - 'ensemble': { - 'base_infile': None, + "get_forcing_data": None, + "run_SOG": None, + "SOG_executable": None, + "html_results": None, + "ensemble": { + "base_infile": None, }, - 'climate': { - 'url': None, - 'params': None, - 'meteo': { - 'station_id': None, - 'quantities': [], - 'cloud_fraction_mapping': None, - }, - 'wind': { - 'station_id': None + "climate": { + "url": None, + "params": None, + "meteo": { + "station_id": None, + "quantities": [], + "cloud_fraction_mapping": None, }, + "wind": {"station_id": None}, }, - 'rivers': { - 'disclaimer_url': None, - 'accept_disclaimer': { - 'disclaimer_action': None, - }, - 'data_url': None, - 'params': { - 'mode': None, - 'prm1': None, + "rivers": { + "disclaimer_url": None, + "accept_disclaimer": { + "disclaimer_action": None, }, - 'major': { - 'station_id': None, + "data_url": None, + "params": { + "mode": None, + "prm1": None, }, - 'minor': { - 'station_id': None, - 'scale_factor': None + "major": { + "station_id": None, }, + "minor": {"station_id": None, "scale_factor": None}, }, - 'logging': { - 'debug': None, - 'toaddrs': [], - 'use_test_smtpd': None, + "logging": { + "debug": None, + "toaddrs": [], + "use_test_smtpd": None, }, - 'results': {}, + "results": {}, } return config_dict -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def infile_dict(): infile_dict = { - 'run_start_date': datetime.datetime(2011, 11, 11, 12, 33, 42), - 'SOG_timestep': '900', - 'std_phys_ts_outfile': None, - 'user_phys_ts_outfile': None, - 'std_bio_ts_outfile': None, - 'user_bio_ts_outfile': None, - 'std_chem_ts_outfile': None, - 'user_chem_ts_outfile': None, - 'profiles_outfile_base': None, - 'user_profiles_outfile_base': None, - 'halocline_outfile': None, - 'Hoffmueller_profiles_outfile': None, - 'user_Hoffmueller_profiles_outfile': None, - 'forcing_data_files': { - 'air_temperature': None, - 'relative_humidity': None, - 'cloud_fraction': None, - 'wind': None, - 'major_river': None, - 'minor_river': None, + "run_start_date": datetime.datetime(2011, 11, 11, 12, 33, 42), + "SOG_timestep": "900", + "std_phys_ts_outfile": None, + "user_phys_ts_outfile": None, + "std_bio_ts_outfile": None, + "user_bio_ts_outfile": None, + "std_chem_ts_outfile": None, + "user_chem_ts_outfile": None, + "profiles_outfile_base": None, + "user_profiles_outfile_base": None, + "halocline_outfile": None, + "Hoffmueller_profiles_outfile": None, + "user_Hoffmueller_profiles_outfile": None, + "forcing_data_files": { + "air_temperature": None, + "relative_humidity": None, + "cloud_fraction": None, + "wind": None, + "major_river": None, + "minor_river": None, }, } return infile_dict @@ -111,187 +106,225 @@ def infile_dict(): @pytest.fixture def forcing_processor(): from bloomcast.utils import ForcingDataProcessor - return ForcingDataProcessor(Mock(name='config')) + + return ForcingDataProcessor(Mock(name="config")) @pytest.fixture def climate_processor(): from bloomcast.utils import ClimateDataProcessor - mock_config = Mock(name='config') + + mock_config = Mock(name="config") mock_config.climate.params = {} mock_config.run_start_date = datetime.date(2011, 9, 19) - mock_data_readers = Mock(name='data_readers') + mock_data_readers = Mock(name="data_readers") return ClimateDataProcessor(mock_config, mock_data_readers) -class TestConfig(): - """Unit tests for Config object. - """ - def test_load_config_climate_url(self, config, config_dict, infile_dict, monkeypatch): - """load_config puts expected value in config.climate.url - """ - test_url = 'https://example.com/climateData/bulkdata_e.html' - monkeypatch.setitem(config_dict['climate'], 'url', test_url) +class TestConfig: + """Unit tests for Config object.""" + + def test_load_config_climate_url( + self, config, config_dict, infile_dict, monkeypatch + ): + """load_config puts expected value in config.climate.url""" + test_url = "https://example.com/climateData/bulkdata_e.html" + monkeypatch.setitem(config_dict["climate"], "url", test_url) config._read_yaml_file = Mock(return_value=config_dict) config._read_SOG_infile = Mock(return_value=infile_dict) - config.load_config('config_file') + config.load_config("config_file") assert config.climate.url == test_url - def test_load_config_climate_params(self, config, config_dict, infile_dict, monkeypatch): - """load_config puts expected value in config.climate.params - """ + def test_load_config_climate_params( + self, config, config_dict, infile_dict, monkeypatch + ): + """load_config puts expected value in config.climate.params""" test_params = { - 'timeframe': 1, - 'Prov': 'BC', - 'format': 'xml', + "timeframe": 1, + "Prov": "BC", + "format": "xml", } - monkeypatch.setitem(config_dict['climate'], 'params', test_params) + monkeypatch.setitem(config_dict["climate"], "params", test_params) config._read_yaml_file = Mock(return_value=config_dict) config._read_SOG_infile = Mock(return_value=infile_dict) - config.load_config('config_file') + config.load_config("config_file") assert config.climate.params == test_params - def test_load_meteo_config_station_id(self, config, config_dict, infile_dict, monkeypatch): - """_load_meteo_config puts exp value in config.climate.meteo.station_id - """ + def test_load_meteo_config_station_id( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_meteo_config puts exp value in config.climate.meteo.station_id""" test_station_id = 889 - monkeypatch.setitem(config_dict['climate']['meteo'], 'station_id', test_station_id) + monkeypatch.setitem( + config_dict["climate"]["meteo"], "station_id", test_station_id + ) config.climate = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_meteo_config(config_dict, infile_dict) assert config.climate.meteo.station_id == test_station_id - def test_load_meteo_config_cloud_fraction_mapping(self, config, config_dict, infile_dict, monkeypatch): - """_load_meteo_config puts expected value in cloud_fraction_mapping - """ - test_cloud_fraction_mapping_file = 'cloud_fraction_mapping.yaml' - monkeypatch.setitem(config_dict['climate']['meteo'], 'cloud_fraction_mapping', test_cloud_fraction_mapping_file) + def test_load_meteo_config_cloud_fraction_mapping( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_meteo_config puts expected value in cloud_fraction_mapping""" + test_cloud_fraction_mapping_file = "cloud_fraction_mapping.yaml" + monkeypatch.setitem( + config_dict["climate"]["meteo"], + "cloud_fraction_mapping", + test_cloud_fraction_mapping_file, + ) test_cloud_fraction_mapping = { - 'Drizzle': [9.9675925925925934], - 'Clear': [0.0] * 12, + "Drizzle": [9.9675925925925934], + "Clear": [0.0] * 12, } config.climate = Mock() - def side_effect(config_file): # NOQA - return (DEFAULT if config_file == 'config_file' - else test_cloud_fraction_mapping) - config._read_yaml_file = Mock( - return_value=config_dict, side_effect=side_effect) + def side_effect(config_file): # NOQA + return ( + DEFAULT if config_file == "config_file" else test_cloud_fraction_mapping + ) + + config._read_yaml_file = Mock(return_value=config_dict, side_effect=side_effect) config._load_meteo_config(config_dict, infile_dict) expected = test_cloud_fraction_mapping assert config.climate.meteo.cloud_fraction_mapping == expected - def test_load_wind_config_station_id(self, config, config_dict, infile_dict, monkeypatch): - """_load_wind_config puts value in config.climate.wind.station_id - """ + def test_load_wind_config_station_id( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_wind_config puts value in config.climate.wind.station_id""" test_station_id = 889 - monkeypatch.setitem(config_dict['climate']['wind'], 'station_id', test_station_id) + monkeypatch.setitem( + config_dict["climate"]["wind"], "station_id", test_station_id + ) config.climate = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_wind_config(config_dict, infile_dict) assert config.climate.wind.station_id == test_station_id - def test_load_rivers_config_major_station_id(self, config, config_dict, infile_dict, monkeypatch): - """_load_rivers_config puts value in config.rivers.major.station_id - """ - test_station_id = '08MF005' - monkeypatch.setitem(config_dict['rivers']['major'], 'station_id', test_station_id) + def test_load_rivers_config_major_station_id( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_rivers_config puts value in config.rivers.major.station_id""" + test_station_id = "08MF005" + monkeypatch.setitem( + config_dict["rivers"]["major"], "station_id", test_station_id + ) config.rivers = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_rivers_config(config_dict, infile_dict) assert config.rivers.major.station_id == test_station_id - def test_load_rivers_config_minor_station_id(self, config, config_dict, infile_dict, monkeypatch): - """_load_rivers_config puts value in config.rivers.minor.station_id - """ - test_station_id = '08HB002' - monkeypatch.setitem(config_dict['rivers']['minor'], 'station_id', test_station_id) + def test_load_rivers_config_minor_station_id( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_rivers_config puts value in config.rivers.minor.station_id""" + test_station_id = "08HB002" + monkeypatch.setitem( + config_dict["rivers"]["minor"], "station_id", test_station_id + ) config.rivers = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_rivers_config(config_dict, infile_dict) assert config.rivers.minor.station_id == test_station_id - def test_load_rivers_config_minor_scale_factor(self, config, config_dict, infile_dict, monkeypatch): - """_load_rivers_config puts value in config.rivers.minor.scale_factor - """ + def test_load_rivers_config_minor_scale_factor( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_rivers_config puts value in config.rivers.minor.scale_factor""" test_scale_factor = 0.351 - monkeypatch.setitem(config_dict['rivers']['minor'], 'scale_factor', test_scale_factor) + monkeypatch.setitem( + config_dict["rivers"]["minor"], "scale_factor", test_scale_factor + ) config.rivers = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_rivers_config(config_dict, infile_dict) assert config.rivers.minor.scale_factor == test_scale_factor - def test_load_rivers_config_major_forcing_data_file(self, config, config_dict, infile_dict, monkeypatch): - """_load_rivers_config puts value in config.rivers.output_file.major - """ - test_output_file = 'Fraser_flow' - monkeypatch.setitem(infile_dict['forcing_data_files'], 'major_river', test_output_file) + def test_load_rivers_config_major_forcing_data_file( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_rivers_config puts value in config.rivers.output_file.major""" + test_output_file = "Fraser_flow" + monkeypatch.setitem( + infile_dict["forcing_data_files"], "major_river", test_output_file + ) config.rivers = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_rivers_config(config_dict, infile_dict) assert config.rivers.output_files["major"] == test_output_file - def test_load_rivers_config_minor_forcing_data_file(self, config, config_dict, infile_dict, monkeypatch): - """_load_rivers_config puts value in config.rivers.output_file.minor - """ - test_output_file = 'Englishman_flow' - monkeypatch.setitem(infile_dict['forcing_data_files'], 'minor_river', test_output_file) + def test_load_rivers_config_minor_forcing_data_file( + self, config, config_dict, infile_dict, monkeypatch + ): + """_load_rivers_config puts value in config.rivers.output_file.minor""" + test_output_file = "Englishman_flow" + monkeypatch.setitem( + infile_dict["forcing_data_files"], "minor_river", test_output_file + ) config.rivers = Mock() config._read_yaml_file = Mock(return_value=config_dict) config._load_rivers_config(config_dict, infile_dict) assert config.rivers.output_files["minor"] == test_output_file -class TestForcingDataProcessor(): - """Unit tests for ForcingDataProcessor object. - """ +class TestForcingDataProcessor: + """Unit tests for ForcingDataProcessor object.""" + def test_patch_data_1_hour_gap(self, forcing_processor): - """patch_data correctly flags 1 hour gap in data for interpolation - """ - forcing_processor.data['air_temperature'] = [ + """patch_data correctly flags 1 hour gap in data for interpolation""" + forcing_processor.data["air_temperature"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), 215.0), (datetime.datetime(2011, 9, 25, 10, 0, 0), None), (datetime.datetime(2011, 9, 25, 11, 0, 0), 235.0), ] - forcing_processor.interpolate_values = Mock(name='interpolate_values') - with patch('bloomcast.utils.log') as mock_log: - forcing_processor.patch_data('air_temperature') + forcing_processor.interpolate_values = Mock(name="interpolate_values") + with patch("bloomcast.utils.log") as mock_log: + forcing_processor.patch_data("air_temperature") expected = [ - (('air_temperature data patched for 2011-09-25 10:00:00',),), - (('1 air_temperature data values patched; ' - 'see debug log on disk for details',),), + (("air_temperature data patched for 2011-09-25 10:00:00",),), + ( + ( + "1 air_temperature data values patched; " + "see debug log on disk for details", + ), + ), ] assert mock_log.debug.call_args_list == expected forcing_processor.interpolate_values.assert_called_once_with( - 'air_temperature', 1, 1) + "air_temperature", 1, 1 + ) def test_patch_data_2_hour_gap(self, forcing_processor): - """patch_data correctly flags 2 hour gap in data for interpolation - """ + """patch_data correctly flags 2 hour gap in data for interpolation""" forcing_processor.data = {} - forcing_processor.data['air_temperature'] = [ + forcing_processor.data["air_temperature"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), 215.0), (datetime.datetime(2011, 9, 25, 10, 0, 0), None), (datetime.datetime(2011, 9, 25, 11, 0, 0), None), (datetime.datetime(2011, 9, 25, 12, 0, 0), 230.0), ] forcing_processor.interpolate_values = Mock() - with patch('bloomcast.utils.log') as mock_log: - forcing_processor.patch_data('air_temperature') + with patch("bloomcast.utils.log") as mock_log: + forcing_processor.patch_data("air_temperature") expected = [ - (('air_temperature data patched for 2011-09-25 10:00:00',),), - (('air_temperature data patched for 2011-09-25 11:00:00',),), - (('2 air_temperature data values patched; ' - 'see debug log on disk for details',),), + (("air_temperature data patched for 2011-09-25 10:00:00",),), + (("air_temperature data patched for 2011-09-25 11:00:00",),), + ( + ( + "2 air_temperature data values patched; " + "see debug log on disk for details", + ), + ), ] assert mock_log.debug.call_args_list == expected forcing_processor.interpolate_values.assert_called_once_with( - 'air_temperature', 1, 2) + "air_temperature", 1, 2 + ) def test_patch_data_2_gaps(self, forcing_processor): - """patch_data correctly flags 2 gaps in data for interpolation - """ - forcing_processor.data['air_temperature'] = [ + """patch_data correctly flags 2 gaps in data for interpolation""" + forcing_processor.data["air_temperature"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), 215.0), (datetime.datetime(2011, 9, 25, 10, 0, 0), None), (datetime.datetime(2011, 9, 25, 11, 0, 0), None), @@ -300,76 +333,80 @@ def test_patch_data_2_gaps(self, forcing_processor): (datetime.datetime(2011, 9, 25, 14, 0, 0), 250.0), ] forcing_processor.interpolate_values = Mock() - with patch('bloomcast.utils.log') as mock_log: - forcing_processor.patch_data('air_temperature') + with patch("bloomcast.utils.log") as mock_log: + forcing_processor.patch_data("air_temperature") expected = [ - (('air_temperature data patched for 2011-09-25 10:00:00',),), - (('air_temperature data patched for 2011-09-25 11:00:00',),), - (('air_temperature data patched for 2011-09-25 13:00:00',),), - (('3 air_temperature data values patched; ' - 'see debug log on disk for details',),), + (("air_temperature data patched for 2011-09-25 10:00:00",),), + (("air_temperature data patched for 2011-09-25 11:00:00",),), + (("air_temperature data patched for 2011-09-25 13:00:00",),), + ( + ( + "3 air_temperature data values patched; " + "see debug log on disk for details", + ), + ), ] assert mock_log.debug.call_args_list == expected - expected = [(('air_temperature', 1, 2),), (('air_temperature', 4, 4),)] + expected = [(("air_temperature", 1, 2),), (("air_temperature", 4, 4),)] assert forcing_processor.interpolate_values.call_args_list == expected def test_interpolate_values_1_hour_gap(self, forcing_processor): - """interpolate_values interpolates value for 1 hour gap in data - """ + """interpolate_values interpolates value for 1 hour gap in data""" forcing_processor.data = {} - forcing_processor.data['air_temperature'] = [ + forcing_processor.data["air_temperature"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), 215.0), (datetime.datetime(2011, 9, 25, 10, 0, 0), None), (datetime.datetime(2011, 9, 25, 11, 0, 0), 235.0), ] - forcing_processor.interpolate_values('air_temperature', 1, 1) + forcing_processor.interpolate_values("air_temperature", 1, 1) expected = (datetime.datetime(2011, 9, 25, 10, 0, 0), 225.0) - assert forcing_processor.data['air_temperature'][1] == expected + assert forcing_processor.data["air_temperature"][1] == expected def test_interpolate_values_2_hour_gap(self, forcing_processor): - """interpolate_values interpolates value for 2 hour gap in data - """ + """interpolate_values interpolates value for 2 hour gap in data""" forcing_processor.data = {} - forcing_processor.data['air_temperature'] = [ + forcing_processor.data["air_temperature"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), 215.0), (datetime.datetime(2011, 9, 25, 10, 0, 0), None), (datetime.datetime(2011, 9, 25, 11, 0, 0), None), (datetime.datetime(2011, 9, 25, 12, 0, 0), 230.0), ] - forcing_processor.interpolate_values('air_temperature', 1, 2) + forcing_processor.interpolate_values("air_temperature", 1, 2) expected = (datetime.datetime(2011, 9, 25, 10, 0, 0), 220.0) - assert forcing_processor.data['air_temperature'][1] == expected + assert forcing_processor.data["air_temperature"][1] == expected expected = (datetime.datetime(2011, 9, 25, 11, 0, 0), 225.0) - assert forcing_processor.data['air_temperature'][2] == expected + assert forcing_processor.data["air_temperature"][2] == expected def test_interpolate_values_gap_gt_11_hr_logs_warning( - self, forcing_processor, + self, + forcing_processor, ): - """data gap >11 hr generates warning log message - """ - forcing_processor.data['air_temperature'] = [ + """data gap >11 hr generates warning log message""" + forcing_processor.data["air_temperature"] = [ (datetime.datetime(2014, 2, 11, 0, 0, 0), 15.0) ] - forcing_processor.data['air_temperature'].extend([ - (datetime.datetime(2014, 2, 11, 1 + i, 0, 0), None) - for i in range(15)]) - forcing_processor.data['air_temperature'].append( - (datetime.datetime(2014, 2, 11, 16, 0, 0), 30.0)) - with patch('bloomcast.utils.log', Mock()) as mock_log: + forcing_processor.data["air_temperature"].extend( + [(datetime.datetime(2014, 2, 11, 1 + i, 0, 0), None) for i in range(15)] + ) + forcing_processor.data["air_temperature"].append( + (datetime.datetime(2014, 2, 11, 16, 0, 0), 30.0) + ) + with patch("bloomcast.utils.log", Mock()) as mock_log: forcing_processor.interpolate_values( - 'air_temperature', gap_start=1, gap_end=15) + "air_temperature", gap_start=1, gap_end=15 + ) mock_log.warning.assert_called_once_with( - 'A air_temperature forcing data gap > 11 hr starting at ' - '2014-02-11 01:00 has been patched by linear interpolation') + "A air_temperature forcing data gap > 11 hr starting at " + "2014-02-11 01:00 has been patched by linear interpolation" + ) + +class TestClimateDataProcessor: + """Unit tests for ClimateDataProcessor object.""" -class TestClimateDataProcessor(): - """Unit tests for ClimateDataProcessor object. - """ def test_get_data_months_run_start_date_same_year(self, climate_processor): - """_get_data_months returns data months for run start date in same year - """ - with patch('bloomcast.utils.datetime') as mock_datetime: + """_get_data_months returns data months for run start date in same year""" + with patch("bloomcast.utils.datetime") as mock_datetime: mock_datetime.date.today.return_value = datetime.date(2011, 9, 1) mock_datetime.date.side_effect = datetime.date data_months = climate_processor._get_data_months() @@ -377,9 +414,8 @@ def test_get_data_months_run_start_date_same_year(self, climate_processor): assert data_months[-1] == datetime.date(2011, 9, 1) def test_get_data_months_run_start_date_prev_year(self, climate_processor): - """_get_data_months returns data months for run start date in prev yr - """ - with patch('bloomcast.utils.datetime') as mock_datetime: + """_get_data_months returns data months for run start date in prev yr""" + with patch("bloomcast.utils.datetime") as mock_datetime: mock_datetime.date.today.return_value = datetime.date(2012, 2, 1) mock_datetime.date.side_effect = datetime.date data_months = climate_processor._get_data_months() diff --git a/tests/test_wind.py b/tests/test_wind.py index dca33b0..fb5b88e 100644 --- a/tests/test_wind.py +++ b/tests/test_wind.py @@ -12,8 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Unit tests for SoG-bloomcast wind module. -""" +"""Unit tests for SoG-bloomcast wind module.""" import datetime from unittest.mock import ( Mock, @@ -26,61 +25,61 @@ @pytest.fixture def wind(): from bloomcast.wind import WindProcessor - return WindProcessor(Mock(name='config')) + return WindProcessor(Mock(name="config")) + + +class TestWindProcessor: + """Unit tests for WindProcessor object.""" -class TestWindProcessor(): - """Unit tests for WindProcessor object. - """ def test_interpolate_values_1_hour_gap(self, wind): - """interpolate_values interpolates value for 1 hour gap in data - """ - wind.data['wind'] = [ + """interpolate_values interpolates value for 1 hour gap in data""" + wind.data["wind"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), (1.0, -2.0)), (datetime.datetime(2011, 9, 25, 10, 0, 0), (None, None)), (datetime.datetime(2011, 9, 25, 11, 0, 0), (2.0, -1.0)), ] - wind.interpolate_values('wind', 1, 1) + wind.interpolate_values("wind", 1, 1) expected = (datetime.datetime(2011, 9, 25, 10, 0, 0), (1.5, -1.5)) - assert wind.data['wind'][1] == expected + assert wind.data["wind"][1] == expected def test_interpolate_values_2_hour_gap(self, wind): - """interpolate_values interpolates value for 2 hour gap in data - """ - wind.data['wind'] = [ + """interpolate_values interpolates value for 2 hour gap in data""" + wind.data["wind"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), (1.0, -2.0)), (datetime.datetime(2011, 9, 25, 10, 0, 0), (None, None)), (datetime.datetime(2011, 9, 25, 11, 0, 0), (None, None)), (datetime.datetime(2011, 9, 25, 12, 0, 0), (2.5, -0.5)), ] - wind.interpolate_values('wind', 1, 2) + wind.interpolate_values("wind", 1, 2) expected = (datetime.datetime(2011, 9, 25, 10, 0, 0), (1.5, -1.5)) - assert wind.data['wind'][1] == expected + assert wind.data["wind"][1] == expected expected = (datetime.datetime(2011, 9, 25, 11, 0, 0), (2.0, -1.0)) - assert wind.data['wind'][2] == expected + assert wind.data["wind"][2] == expected def test_interpolate_values_gap_gt_11_hr_logs_warning(self, wind): - """wind data gap >11 hr generates warning log message - """ - wind.data['wind'] = [ - (datetime.datetime(2011, 9, 25, 0, 0, 0), (1.0, -2.0)) - ] - wind.data['wind'].extend([ - (datetime.datetime(2011, 9, 25, 1 + i, 0, 0), (None, None)) - for i in range(15)]) - wind.data['wind'].append( - (datetime.datetime(2011, 9, 25, 16, 0, 0), (1.0, -2.0))) - with patch('bloomcast.wind.log', Mock()) as mock_log: - wind.interpolate_values('wind', gap_start=1, gap_end=15) + """wind data gap >11 hr generates warning log message""" + wind.data["wind"] = [(datetime.datetime(2011, 9, 25, 0, 0, 0), (1.0, -2.0))] + wind.data["wind"].extend( + [ + (datetime.datetime(2011, 9, 25, 1 + i, 0, 0), (None, None)) + for i in range(15) + ] + ) + wind.data["wind"].append( + (datetime.datetime(2011, 9, 25, 16, 0, 0), (1.0, -2.0)) + ) + with patch("bloomcast.wind.log", Mock()) as mock_log: + wind.interpolate_values("wind", gap_start=1, gap_end=15) mock_log.warning.assert_called_once_with( - 'A wind forcing data gap > 11 hr starting at 2011-09-25 01:00 ' - 'has been patched by linear interpolation') + "A wind forcing data gap > 11 hr starting at 2011-09-25 01:00 " + "has been patched by linear interpolation" + ) def test_format_data(self, wind): - """format_data generator returns formatted forcing data file line - """ - wind.data['wind'] = [ + """format_data generator returns formatted forcing data file line""" + wind.data["wind"] = [ (datetime.datetime(2011, 9, 25, 9, 0, 0), (1.0, 2.0)), ] line = next(wind.format_data()) - assert line == '25 09 2011 9.0 1.000000 2.000000\n' + assert line == "25 09 2011 9.0 1.000000 2.000000\n" From 66974b122d573aa2e7beea9e83db745055b73fc0 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:48:58 -0800 Subject: [PATCH 12/20] Add pytest-cov and pytest-randomly to dev env Updated environment-dev.yaml to include pytest-cov for test coverage monitoring and pytest-randomly for randomized test execution. These additions aim to improve testing comprehensiveness and robustness. --- envs/environment-dev.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/envs/environment-dev.yaml b/envs/environment-dev.yaml index 68cf403..2bcffad 100644 --- a/envs/environment-dev.yaml +++ b/envs/environment-dev.yaml @@ -38,8 +38,10 @@ dependencies: - black - pre-commit - # For unit tests + # For unit tests and coverage monitoring - pytest + - pytest-cov + - pytest-randomly # For documentation - sphinx From d23a11c94f4200623d4002d6c5239f2bb2971fd0 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 12:51:33 -0800 Subject: [PATCH 13/20] Pin versions of sphinx and optional extensions As recommended for reproducible builds on readthedocs: https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html and to ensure that our docs build works with readthedocs changes to default project dependencies: https://blog.readthedocs.com/defaulting-latest-build-tools/ --- envs/environment-dev.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/envs/environment-dev.yaml b/envs/environment-dev.yaml index 2bcffad..ee45407 100644 --- a/envs/environment-dev.yaml +++ b/envs/environment-dev.yaml @@ -44,5 +44,5 @@ dependencies: - pytest-randomly # For documentation - - sphinx - - sphinx_rtd_theme + - sphinx=8.1.3 + - sphinx-rtd-theme=3.0.0 From 840f8a03675bab83a31b234bfe9537b153c13107 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 14:12:20 -0800 Subject: [PATCH 14/20] Switch to pyproject.toml for build configuration Replaced setup.py with pyproject.toml to modernize the project's build system and comply with PEP 518 standards. Updated references in the codebase to use the new configuration format. This change simplifies dependency management and aligns the project with current Python packaging practices. --- bloomcast/main.py | 4 +-- pyproject.toml | 60 +++++++++++++++++++++++++++++++++++ setup.py | 81 ----------------------------------------------- 3 files changed, 62 insertions(+), 83 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.py diff --git a/bloomcast/main.py b/bloomcast/main.py index dc4bdc0..ed6a526 100644 --- a/bloomcast/main.py +++ b/bloomcast/main.py @@ -16,8 +16,8 @@ Operational prediction of the Strait of Georgia spring phytoplankton bloom -This module is connected to the `bloomcast` command via a console_scripts -entry point in setup.py. +This module is connected to the :command:`bloomcast ensemble` command via the scripts and +entry-points configuration elements in the :file:`pyproject.toml` file. """ import sys diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9061bac --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +# Copyright 2011– present by Doug Latornell and The University of British Columbia +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# SPDX-License-Identifier: Apache-2.0 + + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "SoG-bloomcast" +#dynamic = [ "version" ] +version = "3.1" +description = "Strait of Georgia spring diatom bloom predictor" +readme = "README.rst" +requires-python = ">=3.12" +license = {text = "Apache License, Version 2.0"} +#license-files = { paths = ["LICENSE"] } +authors = [ + {name = "Doug Latornell", email = "dlatornell@eoas.ubc.ca"} +] +dependencies = [ + # see envs/environment-dev.yaml for conda environment dev installation, + # see envs/requirements.txt for package versions used during recent development "arrow", + "BeautifulSoup4", + "cliff", + "matplotlib", + "numpy", + "PyYAML", + "requests" + # "SOGcommand" # use python -m pip install --editable SOG/ +] + +[project.urls] +"Homepage" = "https://salishsea.eos.ubc.ca/bloomcast/" +"Issue Tracker" = "https://github.com/SalishSeaCast/SOG-Bloomcast-Ensemble/issues" +"Source Code" = "https://github.com/SalishSeaCast/SOG-Bloomcast-Ensemble" + +[project.scripts] +bloomcast = "bloomcast.main:main" + +[tool.setuptools.entry-points."bloomcast.app"] +ensemble = "bloomcast.ensemble:Ensemble" + + +[tool.setuptools.packages.find] +include = ["."] +include_package_data = true diff --git a/setup.py b/setup.py deleted file mode 100644 index 4b0546f..0000000 --- a/setup.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2011-2021 Doug Latornell and The University of British Columbia - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""SoG-bloomcast -- Operational Prediction of the Strait of Georgia -Spring Phytoplankton Bloom -""" -import setuptools - -import __pkg_metadata__ - - -python_classifiers = [ - "Programming Language :: Python :: {0}".format(py_version) - for py_version in ["3", "3.6", "3.7"] -] -other_classifiers = [ - "Development Status :: " + __pkg_metadata__.DEV_STATUS, - "License :: OSI Approved :: BSD License", - "Programming Language :: Python :: Implementation :: CPython", - "Operating System :: Unix", - "Operating System :: MacOS :: MacOS X", - "Environment :: Console", - "Intended Audience :: Science/Research", - "Intended Audience :: Education", -] -try: - long_description = open("README.rst", "rt").read() -except IOError: - long_description = "" -install_requires = [ - "arrow", - "BeautifulSoup4", - "cliff", - "matplotlib", - "numpy", - "PyYAML", - "requests", - # Use `cd SOG; pip install -e .` to install SOG command processor - # and its dependencies -] - -setuptools.setup( - name=__pkg_metadata__.PROJECT, - version=__pkg_metadata__.VERSION, - description=__pkg_metadata__.DESCRIPTION, - long_description=long_description, - author="Doug Latornell", - author_email="djl@douglatornell.ca", - url="http://eos.ubc.ca/~sallen/SoG-bloomcast/results.html", - download_url=( - "https://bitbucket.org/douglatornell/sog-bloomcast/get/default.tar.gz" - ), - license="Apache License, Version 2.0", - classifiers=python_classifiers + other_classifiers, - platforms=["MacOS X", "Linux"], - install_requires=install_requires, - packages=setuptools.find_packages(), - include_package_data=True, - zip_safe=False, - entry_points={ - # The bloomcast command: - "console_scripts": [ - "bloomcast = bloomcast.main:main", - ], - # Sub-command plug-ins: - "bloomcast.app": [ - "ensemble = bloomcast.ensemble:Ensemble", - ], - }, -) From 574415791d0cc42a16852c751b63355bc2db0c6d Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 14:18:43 -0800 Subject: [PATCH 15/20] Chg to Hatch for project build system and pkg management Updated `pyproject.toml` to replace Setuptools with Hatchling as the build system. Added Hatch to the dev environment dependencies in `environment-dev.yaml` for package management and consistency. Removed Setuptools-specific configurations and aligned the project with Hatch's requirements. --- envs/environment-dev.yaml | 3 ++- pyproject.toml | 12 +++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/envs/environment-dev.yaml b/envs/environment-dev.yaml index ee45407..9792188 100644 --- a/envs/environment-dev.yaml +++ b/envs/environment-dev.yaml @@ -34,8 +34,9 @@ dependencies: - pyyaml - requests - # For coding style and repo QA + # For coding style, repo QA, and package management - black + - hatch - pre-commit # For unit tests and coverage monitoring diff --git a/pyproject.toml b/pyproject.toml index 9061bac..342cda6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,8 +16,8 @@ [build-system] -requires = ["setuptools", "wheel"] -build-backend = "setuptools.build_meta" +requires = ["hatchling"] +build-backend = "hatchling.build" [project] name = "SoG-bloomcast" @@ -26,8 +26,7 @@ version = "3.1" description = "Strait of Georgia spring diatom bloom predictor" readme = "README.rst" requires-python = ">=3.12" -license = {text = "Apache License, Version 2.0"} -#license-files = { paths = ["LICENSE"] } +license-files = { paths = ["LICENSE"] } authors = [ {name = "Doug Latornell", email = "dlatornell@eoas.ubc.ca"} ] @@ -55,6 +54,5 @@ bloomcast = "bloomcast.main:main" ensemble = "bloomcast.ensemble:Ensemble" -[tool.setuptools.packages.find] -include = ["."] -include_package_data = true +[tool.hatch.build.targets.wheel] +packages = ["SoG-bloomcast"] From b92645a654695a7c567f74e37a2746ed50bf4292 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 14:34:47 -0800 Subject: [PATCH 16/20] Refactor metadata handling and update project config Centralize package metadata in `pyproject.toml` and `bloomcast/__about__.py`, removing legacy `__pkg_metadata__.py`. Updated Sphinx configuration to dynamically fetch metadata, modernizing the documentation setup. Adjusted authors list and introduced version automation with Hatch. --- __pkg_metadata__.py => bloomcast/__about__.py | 25 +++----- bloomcast/__pkg_metadata__.py | 1 - docs/conf.py | 60 ++++++++----------- pyproject.toml | 9 ++- 4 files changed, 38 insertions(+), 57 deletions(-) rename __pkg_metadata__.py => bloomcast/__about__.py (55%) delete mode 120000 bloomcast/__pkg_metadata__.py diff --git a/__pkg_metadata__.py b/bloomcast/__about__.py similarity index 55% rename from __pkg_metadata__.py rename to bloomcast/__about__.py index a0a1242..ccff6a1 100644 --- a/__pkg_metadata__.py +++ b/bloomcast/__about__.py @@ -1,29 +1,18 @@ -# Copyright 2011-2021 Doug Latornell and The University of British Columbia - +# Copyright 2011– present by Doug Latornell, Susan Allen, and The University of British Columbia +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - +# +# https://www.apache.org/licenses/LICENSE-2.0 +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Python packaging metadata for SoG-bloomcast.""" - - -__all__ = [ - "PROJECT", - "DESCRIPTION", - "VERSION", - "DEV_STATUS", -] +# SPDX-License-Identifier: Apache-2.0 -PROJECT = "SoG-bloomcast" -DESCRIPTION = "Strait of Georgia spring diatom bloom predictor" -VERSION = "3.1" -DEV_STATUS = "5 - Production" +__version__ = "3.1" # pragma: no cover diff --git a/bloomcast/__pkg_metadata__.py b/bloomcast/__pkg_metadata__.py deleted file mode 120000 index eb861a0..0000000 --- a/bloomcast/__pkg_metadata__.py +++ /dev/null @@ -1 +0,0 @@ -../__pkg_metadata__.py \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index b06e8d4..060bf53 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- -# # SoG Bloomcast documentation build configuration file, created by # sphinx-quickstart on Mon Aug 15 17:45:54 2011. # @@ -11,47 +9,44 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import importlib.metadata +import os +import sys +import tomllib +from pathlib import Path + +sys.path.insert(0, os.path.abspath("..")) + -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -# sys.path.insert(0, os.path.abspath('.')) +# -- Project information ----------------------------------------------------- -# -- General configuration ----------------------------------------------------- -# If your documentation needs a minimal Sphinx version, state it here. -# needs_sphinx = '1.0' +with Path("../pyproject.toml").open("rb") as f: + pkg_info = tomllib.load(f) +project = pkg_info["project"]["name"] +author = "Doug Latornell, Susan Allen, and The University of British Columbia" +pkg_creation_year = 2011 +copyright = f"{pkg_creation_year} – present by {author}" + +# The short X.Y version +version = importlib.metadata.version(project) +# The full version, including alpha/beta/rc tags +release = version # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.viewcode"] +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.intersphinx", + "sphinx.ext.viewcode", +] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] -# The suffix of source filenames. -source_suffix = ".rst" - -# The encoding of source files. -# source_encoding = 'utf-8-sig' - # The master toctree document. master_doc = "index" -# General information about the project. -project = "SoG Bloomcast" -copyright = "2021, Doug Latornell, Susan Allen" - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -version = "0.1" -# The full version, including alpha/beta/rc tags. -release = "0.1dev" - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None @@ -117,11 +112,6 @@ # pixels large. # html_favicon = None -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ["_static"] - # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' diff --git a/pyproject.toml b/pyproject.toml index 342cda6..9154c3c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,14 +21,14 @@ build-backend = "hatchling.build" [project] name = "SoG-bloomcast" -#dynamic = [ "version" ] -version = "3.1" +dynamic = [ "version" ] description = "Strait of Georgia spring diatom bloom predictor" readme = "README.rst" requires-python = ">=3.12" license-files = { paths = ["LICENSE"] } authors = [ - {name = "Doug Latornell", email = "dlatornell@eoas.ubc.ca"} + {name = "Doug Latornell", email = "dlatornell@eoas.ubc.ca"}, + {name = "Susan Allen", email = "sallen@eoas.ubc.ca"}, ] dependencies = [ # see envs/environment-dev.yaml for conda environment dev installation, @@ -56,3 +56,6 @@ ensemble = "bloomcast.ensemble:Ensemble" [tool.hatch.build.targets.wheel] packages = ["SoG-bloomcast"] + +[tool.hatch.version] +path = "bloomcast/__about__.py" From 973d674ed20affb1fedaf3cf533a5d1c032ceaa1 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 14:36:24 -0800 Subject: [PATCH 17/20] Update pkgs & versions used in recent dev env --- envs/requirements.txt | 215 +++++++++++++++++++++++++----------------- 1 file changed, 126 insertions(+), 89 deletions(-) diff --git a/envs/requirements.txt b/envs/requirements.txt index d189653..abdf873 100644 --- a/envs/requirements.txt +++ b/envs/requirements.txt @@ -8,152 +8,189 @@ # # python -m pip list --format=freeze >> envs/requirements.txt -alabaster==0.7.16 -anyio==4.4.0 +alabaster==1.0.0 +anyio==4.8.0 argon2-cffi==23.1.0 argon2-cffi-bindings==21.2.0 arrow==1.3.0 -asttokens==2.4.1 +asttokens==3.0.0 async-lru==2.0.4 -attrs==23.2.0 +attrs==25.1.0 autopage==0.5.2 -Babel==2.14.0 -beautifulsoup4==4.12.3 -bleach==6.1.0 +babel==2.17.0 +backports.tarfile==1.2.0 +beautifulsoup4==4.13.3 +black==25.1.0 +bleach==6.2.0 Brotli==1.1.0 cached-property==1.5.2 -certifi==2024.7.4 -cffi==1.16.0 -charset-normalizer==3.3.2 -cliff==4.7.0 -cmd2==2.4.3 +certifi==2025.1.31 +cffi==1.17.1 +cfgv==3.3.1 +charset-normalizer==3.4.1 +click==8.1.8 +cliff==4.8.0 +cmd2==2.5.8 colander==1.8.3 colorama==0.4.6 comm==0.2.2 -contourpy==1.2.1 +contourpy==1.3.1 +coverage==7.6.12 +cryptography==44.0.1 cycler==0.12.1 -debugpy==1.8.1 +debugpy==1.8.12 decorator==5.1.1 defusedxml==0.7.1 -docutils==0.20.1 -entrypoints==0.4 -exceptiongroup==1.2.0 -executing==2.0.1 -fastjsonschema==2.20.0 -fonttools==4.53.0 +distlib==0.3.9 +docutils==0.21.2 +editables==0.5 +exceptiongroup==1.2.2 +executing==2.1.0 +fastjsonschema==2.21.1 +filelock==3.17.0 +fonttools==4.56.0 fqdn==1.5.1 h11==0.14.0 -h2==4.1.0 -hpack==4.0.0 -httpcore==1.0.5 -httpx==0.27.0 -hyperframe==6.0.1 -idna==3.7 +h2==4.2.0 +hatch==1.14.0 +hatchling==1.27.0 +hpack==4.1.0 +httpcore==1.0.7 +httpx==0.28.1 +hyperframe==6.1.0 +hyperlink==21.0.0 +identify==2.6.7 +idna==3.10 imagesize==1.4.1 -importlib_metadata==7.1.0 -importlib_resources==6.4.0 +importlib_metadata==8.6.1 +importlib_resources==6.5.2 iniconfig==2.0.0 -ipykernel==6.29.4 -ipython==8.25.0 +ipykernel==6.29.5 +ipython==8.32.0 iso8601==2.1.0 isoduration==20.11.0 -jedi==0.19.1 +jaraco.classes==3.4.0 +jaraco.context==6.0.1 +jaraco.functools==4.1.0 +jedi==0.19.2 +jeepney==0.8.0 Jinja2==3.1.5 -json5==0.9.25 +json5==0.10.0 jsonpointer==3.0.0 -jsonschema==4.22.0 -jsonschema-specifications==2023.12.1 -jupyter_client==8.6.2 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +jupyter_client==8.6.3 jupyter_core==5.7.2 -jupyter-events==0.10.0 +jupyter-events==0.12.0 jupyter-lsp==2.2.5 -jupyter_server==2.14.1 +jupyter_server==2.15.0 jupyter_server_terminals==0.5.3 -jupyterlab==4.2.5 +jupyterlab==4.3.5 jupyterlab_pygments==0.3.0 -jupyterlab_server==2.27.2 -kiwisolver==1.4.5 -MarkupSafe==2.1.5 -matplotlib==3.8.4 +jupyterlab_server==2.27.3 +keyring==25.6.0 +kiwisolver==1.4.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.0 matplotlib-inline==0.1.7 -mistune==3.0.2 +mdurl==0.1.2 +mistune==3.1.1 +more-itertools==10.6.0 munkres==1.1.4 -nbclient==0.10.0 -nbconvert==7.16.4 +mypy_extensions==1.0.0 +nbclient==0.10.2 +nbconvert==7.16.6 nbformat==5.10.4 nest_asyncio==1.6.0 +nodeenv==1.9.1 notebook_shim==0.2.4 -numpy==2.0.0 +numpy==2.2.3 overrides==7.7.0 -packaging==24.1 +packaging==24.2 pandocfilters==1.5.0 parso==0.8.4 -pbr==6.0.0 +pathspec==0.12.1 +pbr==6.1.1 pexpect==4.9.0 pickleshare==0.7.5 -pillow==10.3.0 -pip==24.0 +pillow==11.1.0 +pip==25.0.1 pkgutil_resolve_name==1.3.10 -platformdirs==4.2.2 +platformdirs==4.3.6 pluggy==1.5.0 -ply==3.11 -prettytable==3.10.0 -prometheus_client==0.20.0 -prompt_toolkit==3.0.47 -psutil==5.9.8 +pre_commit==4.1.0 +prettytable==3.14.0 +prometheus_client==0.21.1 +prompt_toolkit==3.0.50 +psutil==6.1.1 ptyprocess==0.7.0 -pure-eval==0.2.2 +pure_eval==0.2.3 pycparser==2.22 -Pygments==2.18.0 -pyparsing==3.1.2 -pyperclip==1.8.2 -PyQt5==5.15.9 -PyQt5-sip==12.12.2 +Pygments==2.19.1 +pyparsing==3.2.1 +pyperclip==1.9.0 +PySide6==6.8.2 PySocks==1.7.1 -pytest==8.2.2 -python-dateutil==2.9.0 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-randomly==3.15.0 +python-dateutil==2.9.0.post0 python-json-logger==2.0.7 -pytz==2024.1 -PyYAML==6.0.1 -pyzmq==26.0.3 -referencing==0.35.1 +pytz==2025.1 +PyYAML==6.0.2 +pyzmq==26.2.1 +referencing==0.36.2 requests==2.32.3 -rfc3339-validator==0.1.4 +rfc3339_validator==0.1.4 rfc3986-validator==0.1.1 -rpds-py==0.18.1 +rich==13.9.4 +rpds-py==0.22.3 +SecretStorage==3.3.3 Send2Trash==1.8.3 -setuptools==70.0.0 -sip==6.7.12 -six==1.16.0 +setuptools==75.8.0 +shellingham==1.5.4 +shiboken6==6.8.2 +six==1.17.0 sniffio==1.3.1 snowballstemmer==2.2.0 +SoG-bloomcast==3.1 +SOGcommand==1.3.3 soupsieve==2.5 -Sphinx==7.3.7 -sphinx-rtd-theme==2.0.0 -sphinxcontrib-applehelp==1.0.8 -sphinxcontrib-devhelp==1.0.6 -sphinxcontrib-htmlhelp==2.0.5 +Sphinx==8.1.3 +sphinx_rtd_theme==3.0.0 +sphinxcontrib-applehelp==2.0.0 +sphinxcontrib-devhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-qthelp==1.0.7 +sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==1.1.10 -stack-data==0.6.2 -stevedore==5.2.0 +stack_data==0.6.3 +stevedore==5.4.0 terminado==0.18.1 -tinycss2==1.3.0 +tinycss2==1.4.0 toml==0.10.2 -tomli==2.0.1 +tomli==2.2.1 +tomli_w==1.2.0 +tomlkit==0.13.2 tornado==6.4.2 traitlets==5.14.3 translationstring==1.4 -types-python-dateutil==2.9.0.20240316 +trove-classifiers==2025.1.15.22 +types-python-dateutil==2.9.0.20241206 typing_extensions==4.12.2 -typing-utils==0.1.0 +typing_utils==0.1.0 +ukkonen==1.0.1 +unicodedata2==16.0.0 uri-template==1.3.0 -urllib3==2.2.2 +urllib3==2.3.0 +userpath==1.9.2 +virtualenv==20.29.2 wcwidth==0.2.13 -webcolors==24.6.0 +webcolors==24.11.1 webencodings==0.5.1 websocket-client==1.8.0 -wheel==0.43.0 -zipp==3.19.2 +wheel==0.45.1 +zipp==3.21.0 +zstandard==0.23.0 From c7490c875768e33af5506712853e1937106ffae8 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 14:56:54 -0800 Subject: [PATCH 18/20] Rename project and package to "bloomcast" Updated the project name and package references in pyproject.toml to address CI test import failures. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9154c3c..c367637 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ requires = ["hatchling"] build-backend = "hatchling.build" [project] -name = "SoG-bloomcast" +name = "bloomcast" dynamic = [ "version" ] description = "Strait of Georgia spring diatom bloom predictor" readme = "README.rst" @@ -55,7 +55,7 @@ ensemble = "bloomcast.ensemble:Ensemble" [tool.hatch.build.targets.wheel] -packages = ["SoG-bloomcast"] +packages = ["bloomcast"] [tool.hatch.version] path = "bloomcast/__about__.py" From 4bb12c89817dea3b27e82a99b80d25a0181b19d1 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 15:02:55 -0800 Subject: [PATCH 19/20] Move coverage config from .coveragerc to pyproject.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Transitioned the coverage settings to pyproject.toml for a cleaner and centralized configuration. Removed the redundant .coveragerc file as its contents are now fully integrated into the project’s primary configuration file. --- .coveragerc | 9 --------- pyproject.toml | 8 ++++++++ 2 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7858d40..0000000 --- a/.coveragerc +++ /dev/null @@ -1,9 +0,0 @@ -[run] -branch = True -source = - bloomcast - tests - -[report] -show_missing = True -omit = __pkg_metadata__.py diff --git a/pyproject.toml b/pyproject.toml index c367637..54d3364 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,14 @@ bloomcast = "bloomcast.main:main" ensemble = "bloomcast.ensemble:Ensemble" +[tool.coverage.run] +branch = true +source = [ "bloomcast", "tests"] + +[tool.coverage.report] +show_missing = true + + [tool.hatch.build.targets.wheel] packages = ["bloomcast"] From 8475968634eb0b4229efeff59d71eeffadae9a73 Mon Sep 17 00:00:00 2001 From: Doug Latornell Date: Sat, 15 Feb 2025 15:18:03 -0800 Subject: [PATCH 20/20] Add pytest configuration to pyproject.toml Include pytest minimum version and the `tests/` path in pyproject.toml. This is done to avoid collection of the dependency `env/src/sogcommand/SOGcommand/tests/` directory in the GHA pytest-with-coverage workflow. --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 54d3364..5b6029c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,13 @@ bloomcast = "bloomcast.main:main" ensemble = "bloomcast.ensemble:Ensemble" +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = [ + "tests", +] + + [tool.coverage.run] branch = true source = [ "bloomcast", "tests"]