From 8cb7040117fcfbfe2470e2007826cce8ea89a522 Mon Sep 17 00:00:00 2001 From: Benjamin Thomas Schwertfeger Date: Fri, 2 Feb 2024 15:59:17 +0100 Subject: [PATCH 1/3] update documentation; update workflows; update readme and pre-commit hooks --- .dockerignore | 8 + .github/workflows/cicd.yaml | 36 ++- .github/workflows/release.yaml | 65 ---- .github/workflows/release.yml | 54 ---- .pre-commit-config.yaml | 2 +- .readthedocs.yaml | 4 +- README.rst | 177 ++++++----- {docs => doc}/Makefile | 0 {docs => doc}/_static/arguments.csv | 0 {docs => doc}/_static/images/biasCdiagram.png | Bin {docs => doc}/_static/images/dm-doy-plot.png | Bin .../_static/images/qm-cdf-plot-1.png | Bin .../_static/images/qm-cdf-plot-2.png | Bin {docs => doc}/conf.py | 0 {docs => doc}/index.rst | 0 {docs => doc}/links.rst | 0 {docs => doc}/make.bat | 0 {docs => doc}/requirements.txt | 0 doc/src/arguments.rst | 34 +++ {docs => doc}/src/examples.rst | 13 +- doc/src/introduction.rst | 119 ++++++++ {docs => doc}/src/license.rst | 0 {docs => doc}/src/methods.rst | 288 ++++++++++-------- {docs => doc}/src/quickstart.rst | 22 +- {docs => doc}/src/references.rst | 0 docs/src/arguments.rst | 32 -- docs/src/introduction.rst | 113 ------- src/CMethods.cxx | 10 +- 28 files changed, 479 insertions(+), 498 deletions(-) create mode 100644 .dockerignore delete mode 100644 .github/workflows/release.yaml delete mode 100644 .github/workflows/release.yml rename {docs => doc}/Makefile (100%) rename {docs => doc}/_static/arguments.csv (100%) rename {docs => doc}/_static/images/biasCdiagram.png (100%) rename {docs => doc}/_static/images/dm-doy-plot.png (100%) rename {docs => doc}/_static/images/qm-cdf-plot-1.png (100%) rename {docs => doc}/_static/images/qm-cdf-plot-2.png (100%) rename {docs => doc}/conf.py (100%) rename {docs => doc}/index.rst (100%) rename {docs => doc}/links.rst (100%) rename {docs => doc}/make.bat (100%) rename {docs => doc}/requirements.txt (100%) create mode 100644 doc/src/arguments.rst rename {docs => doc}/src/examples.rst (90%) create mode 100644 doc/src/introduction.rst rename {docs => doc}/src/license.rst (100%) rename {docs => doc}/src/methods.rst (58%) rename {docs => doc}/src/quickstart.rst (75%) rename {docs => doc}/src/references.rst (100%) delete mode 100644 docs/src/arguments.rst delete mode 100644 docs/src/introduction.rst diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4bb00eb --- /dev/null +++ b/.dockerignore @@ -0,0 +1,8 @@ +* +!cmake +!src +!include +!Makefile +!tests +!validation +!CMakeLists.txt diff --git a/.github/workflows/cicd.yaml b/.github/workflows/cicd.yaml index 3bc4c6d..16ba19a 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/cicd.yaml @@ -12,6 +12,10 @@ on: push: branches: - "**" + schedule: + - cron: "20 16 * * 0" + release: + types: [created] concurrency: group: CICD-${{ github.ref }} @@ -46,7 +50,7 @@ jobs: ## Create and upload a docker image ## Docker: - if: success() && github.ref == 'refs/heads/dev' + if: success() && github.event_name == 'push' needs: [Build, Test] uses: ./.github/workflows/_build_docker.yaml with: @@ -54,3 +58,33 @@ jobs: secrets: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + ## Create and upload the tagged docker image + ## + Docker-tag: + if: | + success() && + github.actor == 'btschwertfeger' && + github.event_name == 'release' + needs: [Build, Test] + uses: ./.github/workflows/_build_docker.yaml + with: + TAG: ${{ github.ref_name }} + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + ## Create and upload the latest docker image + ## + Docker-latest: + if: | + success() && + github.actor == 'btschwertfeger' && + github.event_name == 'release' + needs: [Build, Test] + uses: ./.github/workflows/_build_docker.yaml + with: + TAG: latest + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml deleted file mode 100644 index 991d54e..0000000 --- a/.github/workflows/release.yaml +++ /dev/null @@ -1,65 +0,0 @@ -## Checks the code logic, style and more -# -*- coding: utf-8 -*- -# Copyright (C) 2023 Benjamin Thomas Schwertfeger -# GitHub: https://github.com/btschwertfeger -# -# Workflow to apply pre-commit, build, test and upload the docker -# image. This job is only triggered during a release. - -name: Release - -on: - release: - types: [created] - -concurrency: - group: Release-${{ github.ref }} - cancel-in-progress: true - -jobs: - ## Run the pre-commit content - Pre-Commit: - uses: ./.github/workflows/_pre_commit.yaml - - ## Build the BiasAdjustCXX command-line tool - ## - Build: - needs: [Pre-Commit] - uses: ./.github/workflows/_build.yaml - - ## Build the documentation - ## - Build-Doc: - needs: [Pre-Commit] - uses: ./.github/workflows/_build_doc.yaml - with: - os: "ubuntu-latest" - python-version: "3.11" - - ## Build the test suite and run the unit tests - ## - Test: - needs: [Pre-Commit] - uses: ./.github/workflows/_test.yaml - - ## Create and upload the tagged docker image - ## - Docker-tag: - needs: [Build, Test] - uses: ./.github/workflows/_build_docker.yaml - with: - TAG: ${{ github.ref_name }} - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} - - ## Create and upload the latest docker image - ## - Docker-latest: - needs: [Build, Test] - uses: ./.github/workflows/_build_docker.yaml - with: - TAG: latest - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 74a8de3..0000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,54 +0,0 @@ -## Checks the code logic, style and more -# -*- coding: utf-8 -*- -# Copyright (C) 2023 Benjamin Thomas Schwertfeger -# GitHub: https://github.com/btschwertfeger -# -# Workflow to apply pre-commit, build, test and upload the docker -# image. This job is only triggered during a release. - -name: CI/CD - -on: - release: - types: [created] - -concurrency: - group: CICD-${{ github.ref }} - cancel-in-progress: true - -jobs: - ## Run the pre-commit content - Pre-Commit: - uses: ./.github/workflows/_pre_commit.yml - - ## Build the BiasAdjustCXX command-line tool - ## - Build: - needs: [Pre-Commit] - uses: ./.github/workflows/_build.yml - - ## Build the documentation - ## - Build-Doc: - needs: [Pre-Commit] - uses: ./.github/workflows/_build_doc.yml - with: - os: "ubuntu-latest" - python-version: "3.11" - - ## Build the test suite and run the unit tests - ## - Test: - needs: [Pre-Commit] - uses: ./.github/workflows/_test.yml - - ## Create and upload the latest docker image - ## - Docker-latest: - needs: [Build, Test] - uses: ./.github/workflows/_build_docker.yml - with: - TAG: ${{ github.ref_name }} - secrets: - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3c99988..1a55e04 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0a13fe3..7c5be70 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,7 +13,7 @@ build: # Build documentation in the docs/ directory with Sphinx sphinx: - configuration: docs/conf.py + configuration: doc/conf.py # If using Sphinx, optionally build your docs in additional formats such as PDF # formats: @@ -22,4 +22,4 @@ sphinx: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: docs/requirements.txt + - requirements: doc/requirements.txt diff --git a/README.rst b/README.rst index 9f29617..4c53e80 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,6 @@ BiasAdjustCXX command-line tool for the application of fast and efficient bias corrections in climatic research =============================================================================================================== - .. BADGES .. |GitHub badge| image:: https://badgen.net/badge/icon/github?icon=github&label :target: https://github.com/btschwertfeger/BiasAdjustCXX @@ -86,23 +85,25 @@ BiasAdjustCXX command-line tool for the application of fast and efficient bias c **The documentation can be found here: https://biasadjustcxx.readthedocs.io/en/latest.** -This software is tested using Google's testing framework *googletest* (https://github.com/google/googletest). +This software is tested using Google's testing framework *googletest* +(https://github.com/google/googletest). .. contents:: :local: 1. About -------- -The command-line tool `BiasAdjustCXX`_ is the subject of a publication that provides an -insight into the architecture, possible applications and new scientific questions. This publication referencing -`BiasAdjustCXX v1.8.1`_ was published in the journal SoftwareX in March 2023 and is available -at `https://doi.org/10.1016/j.softx.2023.101379`_. +The command-line tool `BiasAdjustCXX`_ is the subject of a publication that +provides an insight into the architecture, possible applications and new +scientific questions. This publication referencing `BiasAdjustCXX v1.8.1`_ was +published in the journal SoftwareX in March 2023 and is available at +`https://doi.org/10.1016/j.softx.2023.101379`_. -This tool and the provided data structures are designed -to help minimize discrepancies between modeled and observed climate data of different -time periods. Data from past periods are used to adjust variables -from current and future time series so that their distributional -properties approximate possible actual values. +This tool and the provided data structures are designed to help minimize +discrepancies between modeled and observed climate data of different time +periods. Data from past periods are used to adjust variables from current and +future time series so that their distributional properties approximate possible +actual values. .. figure:: docs/_static/images/biasCdiagram.png @@ -113,12 +114,12 @@ properties approximate possible actual values. Fig 1: Schematic representation of a bias adjustment procedure -In this way, for example, modeled data, which on average represent values -that are too cold, can be bias-corrected by applying an adjustment procedure. -The following figure shows the observed, the modeled, and the bias-corrected values. +In this way, for example, modeled data, which on average represent values that +are too cold, can be bias-corrected by applying an adjustment procedure. The +following figure shows the observed, the modeled, and the bias-corrected values. It is directly visible that the delta adjusted time series -(:math:`T^{*DM}_{sim,p}`) are much more similar to the observed data (:math:`T_{obs,p}`) -than the raw modeled data (:math:`T_{sim,p}`). +(:math:`T^{*DM}_{sim,p}`) are much more similar to the observed data +(:math:`T_{obs,p}`) than the raw modeled data (:math:`T_{sim,p}`). .. figure:: docs/_static/images/dm-doy-plot.png :width: 800 @@ -127,12 +128,13 @@ than the raw modeled data (:math:`T_{sim,p}`). Fig 2: Temperature per day of year in modeled, observed and bias-adjusted climate data -In addition - most of these methods available here have also been implemented in Python. -This can be found in the `python-cmethods`_ package. +In addition - all of these methods available here have also been implemented in +Python. This can be found in the `python-cmethods`_ package. -If you have any inquiries, remarks, requests for assistance, ideas, or potential collaborations, -you can always create an issue on `BiasAdjustCXX/issues`_, utilize the discussion area on -`BiasAdjustCXX/discussions`_, or directly contact me at contact@b-schwertfeger.de. +If you have any inquiries, remarks, requests for assistance, ideas, or potential +collaborations, you can always create an issue on `BiasAdjustCXX/issues`_, +utilize the discussion area on `BiasAdjustCXX/discussions`_, or directly contact +me at contact@b-schwertfeger.de. 2. Available bias correction methods @@ -148,50 +150,54 @@ The following bias correction techniques are available: * Quantile Mapping * Quantile Delta Mapping -All of these mathematical methods are intended to be applied on 1-dimensional time-series climate data. -This module also provides the possibility that enables -the application of the desired bias correction method on 3-dimensinoal data sets. +All of these mathematical methods are intended to be applied on 1-dimensional +time-series climate data. This module also provides the possibility that enables +the application of the desired bias correction method on 3-dimensional data +sets. General Notes ~~~~~~~~~~~~~ -- Except for the variance scaling, all methods can be applied on stochastic and non-stochastic - climate variables. Variance scaling can only be applied on non-stochastic climate variables. - - - Non-stochastic climate variables are those that can be predicted with relative certainty based - on factors such as location, elevation, and season. Examples of non-stochastic climate variables - include air temperature, air pressure, and solar radiation. +- Except for the variance scaling, all methods can be applied on stochastic and + non-stochastic climate variables. Variance scaling can only be applied on + non-stochastic climate variables. - - Stochastic climate variables, on the other hand, are those that exhibit a high degree of - variability and unpredictability, making them difficult to forecast accurately. - Precipitation is an example of a stochastic climate variable because it can vary greatly in timing, - intensity, and location due to complex atmospheric and meteorological processes. + - Non-stochastic climate variables are those that can be predicted with + relative certainty based on factors such as location, elevation, and season. + Examples of non-stochastic climate variables include air temperature, air + pressure, and solar radiation. -- The Delta Method requires that the time series of the control period have the same length - as the time series to be adjusted. + - Stochastic climate variables, on the other hand, are those that exhibit a + high degree of variability and unpredictability, making them difficult to + forecast accurately. Precipitation is an example of a stochastic climate + variable because it can vary greatly in timing, intensity, and location due + to complex atmospheric and meteorological processes. -- Examples can be found in the `BiasAdjustCXX`_ repository and of course - within the `Documentation`_. +- The Delta Method requires that the time series of the control period have the + same length as the time series to be adjusted. -- Speed/Performance tests and comparison to other tools can be found here: `tool comparison`_ +- Examples can be found in the `BiasAdjustCXX`_ repository and of course within + the `Documentation`_. +- Speed/Performance tests and comparison to other tools can be found here: `tool + comparison`_ Notes regarding the scaling-based techniques ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- All data sets must exclude the 29th February and every year must have 365 entries. - This is not required when using the ``--no-group`` flag which can be used to apply - the scaling techniques in such a way that the scaling factors are based on the whole - time series at once. This enables the possibility to apply the BiasAdjustCXX tool to data - sets with custom time scales for example to adjust monthly separated time series individually to - match the techniques described by `Teutschbein et al. (2012)`_ and `Beyer et al. (2020)`_. On the other hand the - long-term 31-day interval procedures are customized variations and prevent - disproportionately high differences in the long-term mean values at the monthly transitions. - That's why the long-term 31-day interval variant is the preferred method and is enabled by - default for all scaling-based techniques. - - +- All data sets must exclude the 29th February and every year must have 365 + entries. This is not required when using the ``--no-group`` flag which can be + used to apply the scaling techniques in such a way that the scaling factors + are based on the whole time series at once. This enables the possibility to + apply the BiasAdjustCXX tool to data sets with custom time scales for example + to adjust monthly separated time series individually to match the techniques + described by `Teutschbein et al. (2012)`_ and `Beyer et al. (2020)`_. On the + other hand the long-term 31-day interval procedures are customized variations + and prevent disproportionately high differences in the long-term mean values + at the monthly transitions. That's why the long-term 31-day interval variant + is the preferred method and is enabled by default for all scaling-based + techniques. 3. Compilation and Installation @@ -200,17 +206,17 @@ Notes regarding the scaling-based techniques Build from source ~~~~~~~~~~~~~~~~~ -Since this tool is written in C++ it must be compiled and installed, before it can be used. -The following libraries and tools must be installed to successfully compile and install -the BiasAdjustCXX command-line tool. +Since this tool is written in C++ it must be compiled and installed, before it +can be used. The following libraries and tools must be installed to successfully +compile and install the BiasAdjustCXX command-line tool. - NetCDF-4 C++ library (`How to install NetCDF-4 C++`_) - CMake v3.10+ (`How to install CMake`_) - GNU Make (see https://www.gnu.org/software/make/) - [optional] Climate Data Operators (`How to install cdo`_) -Please have a look at the following code blocks that demonstrate how to download, build and install -the BiasAdjustCXX tool from source: +Please have a look at the following code blocks that demonstrate how to +download, build and install the BiasAdjustCXX tool from source: .. code:: bash @@ -221,18 +227,21 @@ the BiasAdjustCXX tool from source: make build make install -The tool can be uninstalled using the following command within the project directory: +The tool can be uninstalled using the following command within the project +directory: .. code:: bash make uninstall -After the installation, the tool can be executed using the command listed below. This repository -also serves example data to test this. See the documentation for more information (https://biasadjustcxx.readthedocs.io/en/latest). +After the installation, the tool can be executed using the command listed below. +This repository also serves example data to test this. See the documentation for +more information (https://biasadjustcxx.readthedocs.io/en/latest). -If the netcdf-cxx4 libraries cannot be found - make sure that ``ncxx4-config`` is globally executable, since this tool is used -to determine the location of these libraries. +If the netcdf-cxx4 libraries cannot be found - make sure that ``ncxx4-config`` +is globally executable, since this tool is used to determine the location of +these libraries. .. code:: bash @@ -250,10 +259,11 @@ to determine the location of these libraries. Docker 🐳 ~~~~~~~~~ -The execution of BiasAdjustCXX is also possible within a Docker container. -This is the preferred option when the installation of `NetCDF-4 C++`_, `CMake`_ or `BiasAdjustCXX`_ -on the local system is not desired. It also makes easier to access this tool since Docker -container can run on nearly every operating system. +The execution of BiasAdjustCXX is also possible within a Docker container. This +is the preferred option when the installation of `NetCDF-4 C++`_, `CMake`_ or +`BiasAdjustCXX`_ on the local system is not desired. It also makes easier to +access this tool since Docker container can run on nearly every operating +system. .. code:: bash @@ -269,15 +279,15 @@ container can run on nearly every operating system. --processes 4 # number of threads -See the Dockerhub registry to access the dev, pinned and older versions: `Dockerhub`_ - - +See the Dockerhub registry to access the dev, pinned and older versions: +`Dockerhub`_ 4. Arguments and Parameters --------------------------- -The following table lists the available command-line arguments that can be passed -to the `BiasAdjustCXX`_ tool. Please also have a look at the requirements section below. +The following table lists the available command-line arguments that can be +passed to the `BiasAdjustCXX`_ tool. Please also have a look at the requirements +section below. ``--ref``, ``--reference`` @@ -314,23 +324,26 @@ to the `BiasAdjustCXX`_ tool. Please also have a look at the requirements sectio [optional] display usage example, arguments, hints, and exits the program - Requirements ~~~~~~~~~~~~ -See the documentation for more information (https://biasadjustcxx.readthedocs.io/en/latest). +See the documentation for more information +(https://biasadjustcxx.readthedocs.io/en/latest). - The variable of interest must have the same name in all data sets. -- The dimensions must be named "time", "lat" and "lon" (i.e., time, latitudes and longitudes) - in exactly this order - in case the data sets have more than one dimension. -- Executed scaling-based techniques without the ``--no-group`` flag require that the data - sets exclude the 29th February and every year has exactly 365 entries. -- For adjusting data using the linear scaling, variance scaling or delta method and - the ``--no-group`` flag: You have to separate the input files by month and then apply - the correction for each month individually e.g., for 30 years of data to correct, - you need to prepare the three input data sets so that they first contain all time series for - all Januaries and then apply the adjustment for this data set. After that you have to - do the same for the rest of the months (see ``/examples/example_all_methods.run.sh`` in +- The dimensions must be named "time", "lat" and "lon" (i.e., time, latitudes + and longitudes) in exactly this order - in case the data sets have more than + one dimension. +- Executed scaling-based techniques without the ``--no-group`` flag require that + the data sets exclude the 29th February and every year has exactly 365 + entries. +- For adjusting data using the linear scaling, variance scaling or delta method + and the ``--no-group`` flag: You have to separate the input files by month and + then apply the correction for each month individually e.g., for 30 years of + data to correct, you need to prepare the three input data sets so that they + first contain all time series for all Januaries and then apply the adjustment + for this data set. After that you have to do the same for the rest of the + months (see ``/examples/example_all_methods.run.sh`` in the repository). 5. References diff --git a/docs/Makefile b/doc/Makefile similarity index 100% rename from docs/Makefile rename to doc/Makefile diff --git a/docs/_static/arguments.csv b/doc/_static/arguments.csv similarity index 100% rename from docs/_static/arguments.csv rename to doc/_static/arguments.csv diff --git a/docs/_static/images/biasCdiagram.png b/doc/_static/images/biasCdiagram.png similarity index 100% rename from docs/_static/images/biasCdiagram.png rename to doc/_static/images/biasCdiagram.png diff --git a/docs/_static/images/dm-doy-plot.png b/doc/_static/images/dm-doy-plot.png similarity index 100% rename from docs/_static/images/dm-doy-plot.png rename to doc/_static/images/dm-doy-plot.png diff --git a/docs/_static/images/qm-cdf-plot-1.png b/doc/_static/images/qm-cdf-plot-1.png similarity index 100% rename from docs/_static/images/qm-cdf-plot-1.png rename to doc/_static/images/qm-cdf-plot-1.png diff --git a/docs/_static/images/qm-cdf-plot-2.png b/doc/_static/images/qm-cdf-plot-2.png similarity index 100% rename from docs/_static/images/qm-cdf-plot-2.png rename to doc/_static/images/qm-cdf-plot-2.png diff --git a/docs/conf.py b/doc/conf.py similarity index 100% rename from docs/conf.py rename to doc/conf.py diff --git a/docs/index.rst b/doc/index.rst similarity index 100% rename from docs/index.rst rename to doc/index.rst diff --git a/docs/links.rst b/doc/links.rst similarity index 100% rename from docs/links.rst rename to doc/links.rst diff --git a/docs/make.bat b/doc/make.bat similarity index 100% rename from docs/make.bat rename to doc/make.bat diff --git a/docs/requirements.txt b/doc/requirements.txt similarity index 100% rename from docs/requirements.txt rename to doc/requirements.txt diff --git a/doc/src/arguments.rst b/doc/src/arguments.rst new file mode 100644 index 0000000..ce2d80e --- /dev/null +++ b/doc/src/arguments.rst @@ -0,0 +1,34 @@ +Arguments and Requirements +=========================== + +Arguments +------------- + +The following table lists the available command-line arguments that can be +passed to the `BiasAdjustCXX`_ tool. Please also have a look at the requirements +section below. + +.. csv-table:: Command-line arguments + :file: ../_static/arguments.csv + :delim: ; + :widths: 30, 70 + :header-rows: 1 + + +Requirements +------------- + +- The variable of interest must have the same name in all data sets. +- The dimensions must be named "time", "lat" and "lon" (i.e., time, latitudes + and longitudes) in exactly this order - in case the data sets have more than + one dimension. +- Executed scaling-based techniques without the ``--no-group`` flag require that + the data sets exclude the 29th February and every year has exactly 365 entries + (see :ref:`section-notes-scaling`). +- For adjusting data using the linear scaling, variance scaling or delta method + and the ``--no-group`` flag: You have to separate the input files by month and + then apply the correction for each month individually e.g., for 30 years of + data to correct, you need to prepare the three input data sets so that they + first contain all time series for all Januaries and then apply the adjustment + for this data set. After that you have to do the same for the rest of the + months (see `/examples/example_all_methods.run.sh`_ in the repository). diff --git a/docs/src/examples.rst b/doc/src/examples.rst similarity index 90% rename from docs/src/examples.rst rename to doc/src/examples.rst index 7cc070a..b222631 100644 --- a/docs/src/examples.rst +++ b/doc/src/examples.rst @@ -41,10 +41,11 @@ non-stochastic variable like temperatures ("tas"): -p 4 # use 4 threads (only if the input data is 3-dimensional) -Note/alternative: The regular linear scaling procedure as described by `Teutschbein et al. (2012)`_ -needs to be applied on monthly separated data sets. To do so, you have to separate the input data sets into -individual long-term months and apply the tool on each of these long-term months using -the ``--no-group`` flag. This is shown in `/examples/example_all_methods.run.sh`_. +Note/alternative: The regular linear scaling procedure as described by +`Teutschbein et al. (2012)`_ needs to be applied on monthly separated data sets. +To do so, you have to separate the input data sets into individual long-term +months and apply the tool on each of these long-term months using the +``--no-group`` flag. This is shown in `/examples/example_all_methods.run.sh`_. Multiplicative Delta Method ------------------------------------------------ @@ -65,8 +66,8 @@ stochastic variable like precipitation ("pr"): --max-scaling-factor 3 # set custom max-scaling factor to avoid unrealistic results (default: 10) -Note: The multiplicative variant is only preferred when adjusting non-stochastic variables like precipitaiton, -air pressure or wind speed. +Note: The multiplicative variant is only preferred when adjusting non-stochastic +variables like precipitation, air pressure or wind speed. Additive Quantile Delta Mapping ------------------------------------ diff --git a/doc/src/introduction.rst b/doc/src/introduction.rst new file mode 100644 index 0000000..6679724 --- /dev/null +++ b/doc/src/introduction.rst @@ -0,0 +1,119 @@ +.. This is the introduction + +Introduction +============= + +|GitHub badge| |License badge| |C++ badge| |CICD badge| +|Docker pulls badge| |GCC badge| |CMake badge| +|Release date badge| |Release tag badge| |DOI badge| |Publication bage| + + +About +----- + +The command-line tool `BiasAdjustCXX`_ is the subject of a publication that +provides an insight into the architecture, possible applications and new +scientific questions. This publication referencing `BiasAdjustCXX v1.8.1`_ was +published in the journal SoftwareX in March 2023 and is available at +`https://doi.org/10.1016/j.softx.2023.101379`_. + +This tool and the provided data structures are designed to help minimize +discrepancies between modeled and observed climate data of different time +periods. Data from past periods are used to adjust variables from current and +future time series so that their distributional properties approximate possible +actual values. + + +.. figure:: ../_static/images/biasCdiagram.png + :width: 800 + :align: center + :alt: Schematic representation of a bias adjustment procedure + + Fig 1: Schematic representation of a bias adjustment procedure + + +In this way, for example, modeled data, which on average represent values that +are too cold, can be bias-corrected by applying an adjustment procedure. The +following figure shows the observed, the modeled, and the bias-corrected values. +It is directly visible that the delta adjusted time series +(:math:`T^{*DM}_{sim,p}`) are much more similar to the observed data +(:math:`T_{obs,p}`) than the raw modeled data (:math:`T_{sim,p}`). + +.. figure:: ../_static/images/dm-doy-plot.png + :width: 800 + :align: center + :alt: Temperature per day of year in modeled, observed and bias-adjusted climate data + + Fig 2: Temperature per day of year in modeled, observed and bias-adjusted climate data + +In addition - most of these methods available here have also been implemented in +Python. This can be found in the `python-cmethods`_ package. + +If you have any inquiries, remarks, requests for assistance, ideas, or potential +collaborations, you can always create an issue on `BiasAdjustCXX/issues`_, +utilize the discussion area on `BiasAdjustCXX/discussions`_, or directly contact +me at contact@b-schwertfeger.de. + + +Available bias correction methods +--------------------------------- + +The following bias correction techniques are available: + Scaling-based techniques: + * :ref:`Linear Scaling` + * :ref:`Variance Scaling` + * :ref:`Delta Method` + + Distribution-based techniques: + * :ref:`Quantile Mapping` + * :ref:`Quantile Delta Mapping` + +All of these mathematical methods are intended to be applied on 1-dimensional +time-series climate data. This module also provides the possibility that enables +the application of the desired bias correction method on 3-dimensional data +sets. + +General Notes +~~~~~~~~~~~~~ + +- Except for the variance scaling, all methods can be applied on stochastic and + non-stochastic climate variables. Variance scaling can only be applied on + non-stochastic climate variables. + + - Non-stochastic climate variables are those that can be predicted with + relative certainty based on factors such as location, elevation, and season. + Examples of non-stochastic climate variables include air temperature, air + pressure, and solar radiation. + + - Stochastic climate variables, on the other hand, are those that exhibit a + high degree of variability and unpredictability, making them difficult to + forecast accurately. Precipitation is an example of a stochastic climate + variable because it can vary greatly in timing, intensity, and location due + to complex atmospheric and meteorological processes. + +- The Delta Method requires that the time series of the control period have the +- same length as the time series to be adjusted. +- Examples can be found in the `BiasAdjustCXX`_ repository and of course + within this documentation. +- Speed/Performance tests and comparison to other tools can be found here: `tool + comparison`_ +- References can be found in the :ref:`References` section. + + +.. _section-notes-scaling: + +Notes regarding the scaling-based techniques +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +- All data sets must exclude the 29th February and every year must have 365 + entries. This is not required when using the ``--no-group`` flag which can be + used to apply the scaling techniques in such a way that the scaling factors + are based on the whole time series at once. This enables the possibility to + apply the BiasAdjustCXX tool to data sets with custom time scales for example + to adjust monthly separated time series individually to match the techniques + described by `Teutschbein et al. (2012)`_ and `Beyer et al. (2020)`_. On the + other hand the long-term 31-day interval procedures are customized variations + and prevent disproportionately high differences in the long-term mean values + at the monthly transitions. Thats why the long-term 31-day interval variant is + the preferred method and is enabled by default for all scaling-based + techniques. diff --git a/docs/src/license.rst b/doc/src/license.rst similarity index 100% rename from docs/src/license.rst rename to doc/src/license.rst diff --git a/docs/src/methods.rst b/doc/src/methods.rst similarity index 58% rename from docs/src/methods.rst rename to doc/src/methods.rst index 5af4343..b9fd96d 100644 --- a/docs/src/methods.rst +++ b/doc/src/methods.rst @@ -3,45 +3,54 @@ Available Methods ================= -The implemented bias correction techniques can be devided into scaling-based and distribution-based -methods. While scaling-based methods try to minimize the deviations in the mean values by adding -or multiplying values so called "scaling factors", distribution-based techniques apply distributional -transformations often called "CDF transformations". - -The mathematical basis of the scaling-based bias correction techniques Linear Scaling (LS), -Variance Scaling (VS) and the Delta Method (DM) are described in `Teutschbein et al. (2012)`_ and `Beyer et al. (2020)`_. -During the development of the BiasAdjustCXX command-line tool a weak point of these techniques -was found - *the unrealistic mean values in the monthly transitions*. Since the scaling-based -techniques described in the articles are scaling/adjusting the time-series in the long-term monthly -mean values the scaling factors of the individual months are completly differnt, so that for example -all Januaries are scaled by 1.5, but Februaries by 1.2 which can lead to high and unrealistic deviations -in the long-term monthly mean. - -Since these weak point was detected, a new approach was developed: *the scaling based on long-term 31-day -moving windows*. This technque ensures that values are scaled based on the long-term values of the -surrounding entries. This technique is the default in BiasAdjustCXX. To disable this behaviour and -apply the scaling on the whole time-seroes at once the ``--no-group``-flag can be used. - -The month-dependent scaling described in the mentioned articles is not implemented, but can be -applied as demonstrated in the provided example script `/examples/example_all_methods.run.sh`_ -within the reposiory. - -The distribution-based bias correction techniques are implemented based on the mathematical -formlas described by `Cannon et al. (2015)`_ and `Tong et al. (2021)`_. - -Except for the Variance Scaling all methods can be applied on both, stochastic and non-stochastic -variables. The Variance Scaling can only be applied on stochastic climate variables. - - - Stochastic climate variables are those that are subject to random fluctuations - and are not predictable. They have no predictable trend or pattern. Examples of - stochastic climate variables include precipitation, air temperature, and humidity. - - - Non-stochastic climate variables, on the other hand, have clear trend and pattern histories - and can be readily predicted. They are often referred to as climate elements and include - variables such as water temperature and air pressure. - -Of course all examples shown at the methods can be executed using the provided Docker image. See -section :ref:`section-installation` for more details. +The implemented bias correction techniques can be divided into scaling-based and +distribution-based methods. While scaling-based methods try to minimize the +deviations in the mean values by adding or multiplying values so called "scaling +factors", distribution-based techniques apply distributional transformations +often called "CDF transformations". + +The mathematical basis of the scaling-based bias correction techniques Linear +Scaling (LS), Variance Scaling (VS) and the Delta Method (DM) are described in +`Teutschbein et al. (2012)`_ and `Beyer et al. (2020)`_. During the development +of the BiasAdjustCXX command-line tool a weak point of these techniques was +found - *the unrealistic mean values in the monthly transitions*. Since the +scaling-based techniques described in the articles are scaling/adjusting the +time-series in the long-term monthly mean values the scaling factors of the +individual months are completely different, so that for example all Januaries +are scaled by 1.5, but Februaries by 1.2 which can lead to high and unrealistic +deviations in the long-term monthly mean. + +Since these weak point was detected, a new approach was developed: *the scaling +based on long-term 31-day moving windows*. This technique ensures that values +are scaled based on the long-term values of the surrounding entries. This +technique is the default in BiasAdjustCXX. To disable this behaviour and apply +the scaling on the whole time-series at once the ``--no-group``-flag can be +used. + +The month-dependent scaling described in the mentioned articles is not +implemented, but can be applied as demonstrated in the provided example script +`/examples/example_all_methods.run.sh`_ within the repository. + +The distribution-based bias correction techniques are implemented based on the +mathematical formulas described by `Cannon et al. (2015)`_ and `Tong et al. +(2021)`_. + +Except for the Variance Scaling all methods can be applied on both, stochastic +and non-stochastic variables. The Variance Scaling can only be applied on +stochastic climate variables. + + - Stochastic climate variables are those that are subject to random + fluctuations and are not predictable. They have no predictable trend or + pattern. Examples of stochastic climate variables include precipitation, air + temperature, and humidity. + + - Non-stochastic climate variables, on the other hand, have clear trend and + pattern histories and can be readily predicted. They are often referred to + as climate elements and include variables such as water temperature and air + pressure. + +Of course all examples shown at the methods can be executed using the provided +Docker image. See section :ref:`section-installation` for more details. Linear Scaling ------------------------ @@ -54,19 +63,23 @@ a maximum scaling factor of 10 is set. This can be changed by passing another value to the optional ``--max-scaling-factor`` argument. The Linear Scaling bias correction technique implemented here is based on the -method described in the equations of `Teutschbein et al. (2012)`_ -*"Bias correction of regional climate model simulations for hydrological climate-change -impact studies: Review and evaluation of different methods"* but using long-term 31-day moving windows instead -of long-term monthly means (because of the weak point mentioned in section :ref:`section-available-methods`). +method described in the equations of `Teutschbein et al. (2012)`_ *"Bias +correction of regional climate model simulations for hydrological climate-change +impact studies: Review and evaluation of different methods"* but using long-term +31-day moving windows instead of long-term monthly means (because of the weak +point mentioned in section :ref:`section-available-methods`). -In the following the equations for both additive and multiplicative Linear Scaling are shown: +In the following the equations for both additive and multiplicative Linear +Scaling are shown: **Additive**: -In linear scaling, the mean of the long-term 31-day moving window (:math:`\mu_m`) of the modeled -data :math:`X_{sim,h}` is subtracted from mean of the the long-term 31-day moving window -of the reference data :math:`X_{obs,h}` at time step :math:`i`. This difference in the mean is than added -to the value of time step :math:`i` in the time-series that is to be adjusted (:math:`X_{sim,p}`). +In linear scaling, the mean of the long-term 31-day moving window +(:math:`\mu_m`) of the modeled data :math:`X_{sim,h}` is subtracted from mean of +the the long-term 31-day moving window of the reference data :math:`X_{obs,h}` +at time step :math:`i`. This difference in the mean is than added to the value +of time step :math:`i` in the time-series that is to be adjusted +(:math:`X_{sim,p}`). .. math:: @@ -75,8 +88,8 @@ to the value of time step :math:`i` in the time-series that is to be adjusted (: **Multiplicative**: - The multiplicative linear scaling differs from the additive variant in such way, that the changes are not computed - in absolute but in relative values. + The multiplicative linear scaling differs from the additive variant in such + way, that the changes are not computed in absolute but in relative values. .. math:: @@ -85,8 +98,8 @@ to the value of time step :math:`i` in the time-series that is to be adjusted (: **Example**: -The following example shows how to apply the additive linear scaling technique on a -3-dimensional data set containing the variable "tas" (i.e., temperatures). +The following example shows how to apply the additive linear scaling technique +on a 3-dimensional data set containing the variable "tas" (i.e., temperatures). .. code-block:: bash :linenos: @@ -105,19 +118,21 @@ The following example shows how to apply the additive linear scaling technique o Variance Scaling ------------------------ -The Variance Scaling bias correction technique can be applied only on non-stochastic -climate variables to minimize deviations in the mean and variance +The Variance Scaling bias correction technique can be applied only on +non-stochastic climate variables to minimize deviations in the mean and variance between predicted and observed time-series of past and future time periods. The Variance Scaling bias correction technique implemented here is based on the -method described by `Teutschbein et al. (2012)`_ *"Bias correction of regional climate model -simulations for hydrological climate-change impact studies: Review and evaluation of different methods"* -but using long-term 31-day moving windows instead of long-term monthly means -(because of the weak point mentioned in section :ref:`section-available-methods`). -In the following the equations of the variance scaling approach are shown: +method described by `Teutschbein et al. (2012)`_ *"Bias correction of regional +climate model simulations for hydrological climate-change impact studies: Review +and evaluation of different methods"* but using long-term 31-day moving windows +instead of long-term monthly means (because of the weak point mentioned in +section :ref:`section-available-methods`). In the following the equations of the +variance scaling approach are shown: -**(1)** First, the modeled data of the control and scenario period must be bias-corrected using -the additive linear scaling technique. This adjusts the deviation in the mean. +**(1)** First, the modeled data of the control and scenario period must be +bias-corrected using the additive linear scaling technique. This adjusts the +deviation in the mean. .. math:: @@ -126,8 +141,8 @@ the additive linear scaling technique. This adjusts the deviation in the mean. X^{*LS}_{sim,p}(i) = X_{sim,p}(i) + \mu_{m}(X_{obs,h}(i)) - \mu_{m}(X_{sim,h}(i)) -**(2)** In the second step, the time-series are shifted to a zero mean. This enables the adjustment -of the standard deviation in the following step. +**(2)** In the second step, the time-series are shifted to a zero mean. This +enables the adjustment of the standard deviation in the following step. .. math:: @@ -143,7 +158,7 @@ of the standard deviation in the following step. X^{VS(2)}_{sim,p}(i) = X^{VS(1)}_{sim,p}(i) \cdot \left[\frac{\sigma_{m}(X_{obs,h}(i))}{\sigma_{m}(X^{VS(1)}_{sim,h}(i))}\right] -**(4)** Finally the prevously removed mean is shifted back +**(4)** Finally the previously removed mean is shifted back .. math:: @@ -153,8 +168,9 @@ of the standard deviation in the following step. **Example**: -The following example shows how to apply the (additive) variance scaling technique on a -3-dimensional data set containing the variable "tas" (i.e., temperatures). +The following example shows how to apply the (additive) variance scaling +technique on a 3-dimensional data set containing the variable "tas" (i.e., +temperatures). .. code-block:: bash :linenos: @@ -181,19 +197,25 @@ a maximum scaling factor of 10 is set. This can be changed by passing another value to the optional ``--max-scaling-factor`` argument. The Delta Method bias correction technique implemented here is based on the -method described in the equations of `Beyer et al. (2020)`_ *"An empirical evaluation of bias -correction methods for palaeoclimate simulations"* but using long-term 31-day moving windows -instead of long-term monthly means (because of the weak point mentioned in section :ref:`section-available-methods`). -In the following the equations for both additive and multiplicative Delta Method are shown: +method described in the equations of `Beyer et al. (2020)`_ *"An empirical +evaluation of bias correction methods for palaeoclimate simulations"* but using +long-term 31-day moving windows instead of long-term monthly means (because of +the weak point mentioned in section :ref:`section-available-methods`). In the +following the equations for both additive and multiplicative Delta Method are +shown: **Additive**: - The Delta Method looks like the Linear Scaling method but the important difference is, that the Delta method - uses the change between the modeled data instead of the difference between the modeled and reference data of the control - period. This means that the long-term monthly mean (:math:`\mu_m`) of the modeled data of the control period :math:`T_{sim,h}` - is subtracted from the long-term monthly mean of the modeled data from the scenario period :math:`T_{sim,p}` at time step :math:`i`. - This change in month-dependent long-term mean is than added to the long-term monthly mean for time step :math:`i`, - in the time-series that represents the reference data of the control period (:math:`T_{obs,h}`). + The Delta Method looks like the Linear Scaling method but the important + difference is, that the Delta method uses the change between the modeled + data instead of the difference between the modeled and reference data of the + control period. This means that the long-term monthly mean (:math:`\mu_m`) + of the modeled data of the control period :math:`T_{sim,h}` is subtracted + from the long-term monthly mean of the modeled data from the scenario period + :math:`T_{sim,p}` at time step :math:`i`. This change in month-dependent + long-term mean is than added to the long-term monthly mean for time step + :math:`i`, in the time-series that represents the reference data of the + control period (:math:`T_{obs,h}`). .. math:: @@ -202,8 +224,9 @@ In the following the equations for both additive and multiplicative Delta Method **Multiplicative**: - The multiplicative variant behaves like the additive, but with the difference that the change is computed using the relative change - instead of the absolute change. + The multiplicative variant behaves like the additive, but with the + difference that the change is computed using the relative change instead of + the absolute change. .. math:: @@ -212,8 +235,9 @@ In the following the equations for both additive and multiplicative Delta Method **Example**: -The following example shows how to apply the multiplicative delta method technique on a -3-dimensional data set containing the variable "pr" (i.e., precipitaiton). +The following example shows how to apply the multiplicative delta method +technique on a 3-dimensional data set containing the variable "pr" (i.e., +precipitation). .. code-block:: bash :linenos: @@ -233,18 +257,18 @@ The following example shows how to apply the multiplicative delta method techniq Quantile Mapping ------------------------ -The Quantile Mapping bias correction technique can be used to minimize distributional -biases between modeled and observed time-series climate data. Its interval-independant -behaviour ensures that the whole time series is taken into account to redistribute -its values, based on the distributions of the modeled and observed/reference data of the -control period. +The Quantile Mapping bias correction technique can be used to minimize +distributional biases between modeled and observed time-series climate data. Its +interval-independent behaviour ensures that the whole time series is taken into +account to redistribute its values, based on the distributions of the modeled +and observed/reference data of the control period. The Quantile Mapping technique implemented here is based on the equations of -`Cannon et al. (2015)`_ *"Bias Correction of GCM Precipitation by Quantile Mapping: -How Well Do Methods Preserve Changes in Quantiles and Extremes?"*. +`Cannon et al. (2015)`_ *"Bias Correction of GCM Precipitation by Quantile +Mapping: How Well Do Methods Preserve Changes in Quantiles and Extremes?"*. -A weak point of the regular Quantile Mapping is, that the values are bounded to the -value range of the modeled data of the control period. +A weak point of the regular Quantile Mapping is, that the values are bounded to +the value range of the modeled data of the control period. In the following the equations of `Cannon et al. (2015)`_ are shown and explained: @@ -255,22 +279,25 @@ In the following the equations of `Cannon et al. (2015)`_ are shown and explaine X^{*QM}_{sim,p}(i) = F^{-1}_{obs,h} \left\{F_{sim,h}\left[X_{sim,p}(i)\right]\right\} - The additive quantile mapping procedure consists of inserting the value to be - adjusted (:math:`X_{sim,p}(i)`) into the cumulative distribution function - of the modeled data of the control period (:math:`F_{sim,h}`). This determines, - in which quantile the value to be adjusted can be found in the modeled data of the control period - The following images show this by using :math:`T` for temperatures. + The additive quantile mapping procedure consists of inserting the value to + be adjusted (:math:`X_{sim,p}(i)`) into the cumulative distribution function + of the modeled data of the control period (:math:`F_{sim,h}`). This + determines, in which quantile the value to be adjusted can be found in the + modeled data of the control period The following images show this by using + :math:`T` for temperatures. .. figure:: ../_static/images/qm-cdf-plot-1.png :width: 600 :align: center :alt: Determination of the quantile value - Fig 1: Inserting :math:`X_{sim,p}(i)` into :math:`F_{sim,h}` to determine the quantile value + Fig 1: Inserting :math:`X_{sim,p}(i)` into :math:`F_{sim,h}` to + determine the quantile value This value, which of course lies between 0 and 1, is subsequently inserted - into the inverse cumulative distribution function of the reference data of the control period to - determine the bias-corrected value at time step :math:`i`. + into the inverse cumulative distribution function of the reference data of + the control period to determine the bias-corrected value at time step + :math:`i`. .. figure:: ../_static/images/qm-cdf-plot-2.png :width: 600 @@ -281,14 +308,14 @@ In the following the equations of `Cannon et al. (2015)`_ are shown and explaine **Multiplicative**: - .. math:: - - X^{*QM}_{sim,p}(i) = F^{-1}_{obs,h}\Biggl\{F_{sim,h}\left[\frac{\mu{X_{sim,h}} \cdot \mu{X_{sim,p}(i)}}{\mu{X_{sim,p}(i)}}\right]\Biggr\}\frac{\mu{X_{sim,p}(i)}}{\mu{X_{sim,h}}} + The formula is the same as for the additive variant, but the values are + bound to the lower level of zero. **Example**: -The following example shows how to apply the multiplicative quantile mapping technique on a -3-dimensional data set containing the variable "pr" (i.e., precipitaiton). +The following example shows how to apply the multiplicative quantile mapping +technique on a 3-dimensional data set containing the variable "pr" (i.e., +precipitation). .. code-block:: bash :linenos: @@ -304,21 +331,21 @@ The following example shows how to apply the multiplicative quantile mapping tec --variable pr # variable to adjust - Quantile Delta Mapping ------------------------ -The Quantile Delta Mapping bias correction technique can be used to minimize distributional -biases between modeled and observed time-series climate data. Its interval-independant -behaviour ensures that the whole time series is taken into account to redistribute -its values, based on the distributions of the modeled and observed/reference data of the -control period. In contrast to the regular Quantile Mapping the Quantile Delta Mapping -also takes the change between the modeled data of the control and scenario period into account. +The Quantile Delta Mapping bias correction technique can be used to minimize +distributional biases between modeled and observed time-series climate data. Its +interval-independent behaviour ensures that the whole time series is taken into +account to redistribute its values, based on the distributions of the modeled +and observed/reference data of the control period. In contrast to the regular +Quantile Mapping the Quantile Delta Mapping also takes the change between the +modeled data of the control and scenario period into account. -The Quantile Delta Mapping technique implemented here is based on the equations by -`Tong et al. (2021)`_ *"Bias correction of temperature and precipitation -over China for RCM simulations using the QM and QDM methods"*. In the following the formulas of the -additive and multiplicative variant are shown. +The Quantile Delta Mapping technique implemented here is based on the equations +by `Tong et al. (2021)`_ *"Bias correction of temperature and precipitation over +China for RCM simulations using the QM and QDM methods"*. In the following the +formulas of the additive and multiplicative variant are shown. **Additive**: @@ -330,18 +357,19 @@ additive and multiplicative variant are shown. \varepsilon(i) = F_{sim,p}\left[X_{sim,p}(i)\right], \hspace{1em} \varepsilon(i)\in\{0,1\} - **(1.2)** The bias corrected value at time step :math:`i` is now determined by inserting the - quantile value into the inverse cummulative distribution function of the reference data of the control - period. This results in a bias corrected value for time step :math:`i` but still without taking the - change in modeled data into account. + **(1.2)** The bias corrected value at time step :math:`i` is now determined + by inserting the quantile value into the inverse cumulative distribution + function of the reference data of the control period. This results in a bias + corrected value for time step :math:`i` but still without taking the change + in modeled data into account. .. math:: X^{QDM(1)}_{sim,p}(i) = F^{-1}_{obs,h}\left[\varepsilon(i)\right] - **(1.3)** The :math:`\Delta(i)` represents the absolute change in quantiles between the modeled value - in the control and scenario period. + **(1.3)** The :math:`\Delta(i)` represents the absolute change in quantiles + between the modeled value in the control and scenario period. .. math:: @@ -349,7 +377,8 @@ additive and multiplicative variant are shown. & = X_{sim,p}(i) - F^{-1}_{sim,h}\left\{F^{}_{sim,p}\left[X_{sim,p}(i)\right]\right\} - **(1.4)** Finally the previously calculated change can be added to the bias-corrected value. + **(1.4)** Finally the previously calculated change can be added to the + bias-corrected value. .. math:: @@ -358,11 +387,12 @@ additive and multiplicative variant are shown. **Multiplicative**: - The first two steps of the multiplicative Quantile Delta Mapping bias correction technique are the - same as for the additive variant. + The first two steps of the multiplicative Quantile Delta Mapping bias + correction technique are the same as for the additive variant. - **(2.3)** The :math:`\Delta(i)` in the multiplicative Quantile Delta Mapping is calulated like the - additive variant, but using the relative than the absolute change. + **(2.3)** The :math:`\Delta(i)` in the multiplicative Quantile Delta Mapping + is calculated like the additive variant, but using the relative than the + absolute change. .. math:: @@ -370,8 +400,9 @@ additive and multiplicative variant are shown. & = \frac{ X_{sim,p}(i) }{ F^{-1}_{sim,h}\left\{F_{sim,p}\left[X_{sim,p}(i)\right]\right\} } - **(2.4)** The relative change between the modeled data of the control and scenario period is than - multiplicated with the bias-corrected value (see **1.2**). + **(2.4)** The relative change between the modeled data of the control and + scenario period is than multiplied with the bias-corrected value (see + **1.2**). .. math:: @@ -380,8 +411,9 @@ additive and multiplicative variant are shown. **Example**: -The following example shows how to apply the additive quantile delta mapping technique on a -3-dimensional data set containing the variable "tas" (i.e., temperatures). +The following example shows how to apply the additive quantile delta mapping +technique on a 3-dimensional data set containing the variable "tas" (i.e., +temperatures). .. code-block:: bash :linenos: diff --git a/docs/src/quickstart.rst b/doc/src/quickstart.rst similarity index 75% rename from docs/src/quickstart.rst rename to doc/src/quickstart.rst index 07d0e37..cc3acfe 100644 --- a/docs/src/quickstart.rst +++ b/doc/src/quickstart.rst @@ -8,16 +8,16 @@ Compilation and Installation Build from source ---------------------------- -Since this tool is written in C++ it must be compiled and installed, before it can be used. -The following libraries and tools must be installed to successfully compile and install -the BiasAdjustCXX command-line tool. +Since this tool is written in C++ it must be compiled and installed, before it +can be used. The following libraries and tools must be installed to successfully +compile and install the BiasAdjustCXX command-line tool. - NetCDF-4 C++ library (`How to install NetCDF-4 C++`_) - CMake v3.10+ (`How to install CMake`_) - [optional] Climate Data Operators (`How to install cdo`_) -Please have a look at the following code blocks that demonstrate how to download, build and install -the BiasAdjustCXX tool from source: +Please have a look at the following code blocks that demonstrate how to +download, build and install the BiasAdjustCXX tool from source: .. code-block:: bash @@ -42,10 +42,11 @@ The tool can be uninstalled using the following command within the project direc Docker 🐳 -------------- -The execution of BiasAdjustCXX is also possiblie within a Docker container. -This is the preferred option when the installation of `NetCDF-4 C++`_, `CMake`_ or `BiasAdjustCXX`_ -on the local system is not desired. It also makes easier to access this tool since Docker -container can run on nearly every operating system. +The execution of BiasAdjustCXX is also possible within a Docker container. This +is the preferred option when the installation of `NetCDF-4 C++`_, `CMake`_ or +`BiasAdjustCXX`_ on the local system is not desired. It also makes easier to +access this tool since Docker container can run on nearly every operating +system. .. code-block:: bash :caption: Run the BiasAdjustCXX tool using the provided Docker image @@ -62,4 +63,5 @@ container can run on nearly every operating system. -p 4 # number of threads -See the Dockerhub registry to access the dev, pinned and older versions: `Dockerhub`_ +See the Dockerhub registry to access the dev, pinned and older versions: +`Dockerhub`_ diff --git a/docs/src/references.rst b/doc/src/references.rst similarity index 100% rename from docs/src/references.rst rename to doc/src/references.rst diff --git a/docs/src/arguments.rst b/docs/src/arguments.rst deleted file mode 100644 index b8d630c..0000000 --- a/docs/src/arguments.rst +++ /dev/null @@ -1,32 +0,0 @@ -Arguments and Requirements -=========================== - -Arguments -------------- - -The following table lists the available command-line arguments that can be passed -to the `BiasAdjustCXX`_ tool. Please also have a look at the requirements section below. - -.. csv-table:: Command-line arguments - :file: ../_static/arguments.csv - :delim: ; - :widths: 30, 70 - :header-rows: 1 - - -Requirements -------------- - -- The variable of interest must have the same name in all data sets. -- The dimensions must be named "time", "lat" and "lon" (i.e., time, latitudes and longitudes) - in exactly this order - in case the data sets have more than one dimension. -- Executed scaling-based techniques without the ``--no-group`` flag require that the data - sets exclude the 29th February and every - year has exactly 365 entries (see :ref:`section-notes-scaling`). -- For adjusting data using the linear scaling, variance scaling or delta method and - the ``--no-group`` flag: You have to separate the input files by month and then apply - the correction for each month individually e.g., for 30 years of data to correct, - you need to prepare the three input data sets so that they first contain all time series for - all Januaries and then apply the adjustment for this data set. After that you have to - do the same for the rest of the months (see `/examples/example_all_methods.run.sh`_ in - the repository). diff --git a/docs/src/introduction.rst b/docs/src/introduction.rst deleted file mode 100644 index 0ad3ba6..0000000 --- a/docs/src/introduction.rst +++ /dev/null @@ -1,113 +0,0 @@ -.. This is the introduction - -Introduction -============= - -|GitHub badge| |License badge| |C++ badge| |CICD badge| -|Docker pulls badge| |GCC badge| |CMake badge| -|Release date badge| |Release tag badge| |DOI badge| |Publication bage| - - -About ------ - -The command-line tool `BiasAdjustCXX`_ is the subject of a publication that provides an -insight into the architecture, possible applications and new scientific questions. This publication referencing -`BiasAdjustCXX v1.8.1`_ was published in the journal SoftwareX in March 2023 and is available -at `https://doi.org/10.1016/j.softx.2023.101379`_. - -This tool and the provided data structures are designed -to help minimize discrepancies between modeled and observed climate data of different -time periods. Data from past periods are used to adjust variables -from current and future time series so that their distributional -properties approximate possible actual values. - - -.. figure:: ../_static/images/biasCdiagram.png - :width: 800 - :align: center - :alt: Schematic representation of a bias adjustment procedure - - Fig 1: Schematic representation of a bias adjustment procedure - - -In this way, for example, modeled data, which on average represent values -that are too cold, can be bias-corrected by applying an adjustment procedure. -The following figure shows the observed, the modeled, and the bias-corrected values. -It is directly visible that the delta adjusted time series -(:math:`T^{*DM}_{sim,p}`) are much more similar to the observed data (:math:`T_{obs,p}`) -than the raw modeled data (:math:`T_{sim,p}`). - -.. figure:: ../_static/images/dm-doy-plot.png - :width: 800 - :align: center - :alt: Temperature per day of year in modeled, observed and bias-adjusted climate data - - Fig 2: Temperature per day of year in modeled, observed and bias-adjusted climate data - -In addition - most of these methods available here have also been implemented in Python. -This can be found in the `python-cmethods`_ package. - -If you have any inquiries, remarks, requests for assistance, ideas, or potential collaborations, -you can always create an issue on `BiasAdjustCXX/issues`_, utilize the discussion area on -`BiasAdjustCXX/discussions`_, or directly contact me at contact@b-schwertfeger.de. - - -Available bias correction methods ---------------------------------- - -The following bias correction techniques are available: - Scaling-based techniques: - * :ref:`Linear Scaling` - * :ref:`Variance Scaling` - * :ref:`Delta Method` - - Distribution-based techniques: - * :ref:`Quantile Mapping` - * :ref:`Quantile Delta Mapping` - -All of these mathematical methods are intended to be applied on 1-dimensional time-series climate data. -This module also provides the possibility that enables -the application of the desired bias correction method on 3-dimensinoal data sets. - -General Notes -~~~~~~~~~~~~~ - -- Except for the variance scaling, all methods can be applied on stochastic and non-stochastic - climate variables. Variance scaling can only be applied on non-stochastic climate variables. - - - Non-stochastic climate variables are those that can be predicted with relative certainty based - on factors such as location, elevation, and season. Examples of non-stochastic climate variables - include air temperature, air pressure, and solar radiation. - - - Stochastic climate variables, on the other hand, are those that exhibit a high degree of - variability and unpredictability, making them difficult to forecast accurately. - Precipitation is an example of a stochastic climate variable because it can vary greatly in timing, - intensity, and location due to complex atmospheric and meteorological processes. - -- The Delta Method requires that the time series of the control period have the same length - as the time series to be adjusted. - -- Examples can be found in the `BiasAdjustCXX`_ repository and of course - within this documentation. - -- Speed/Performance tests and comparison to other tools can be found here: `tool comparison`_ - -- References can be found in the :ref:`References` section. - - -.. _section-notes-scaling: - -Notes regarding the scaling-based techniques -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -- All data sets must exclude the 29th February and every year must have 365 entries. - This is not required when using the ``--no-group`` flag which can be used to apply - the scaling techniques in such a way that the scaling factors are based on the whole - time series at once. This enables the possibility to apply the BiasAdjustCXX tool to data - sets with custom time scales for example to adjust monthly separated time series individually to - match the techniques described by `Teutschbein et al. (2012)`_ and `Beyer et al. (2020)`_. On the other hand the - long-term 31-day interval procedures are customized variations and prevent - disproportionately high differences in the long-term mean values at the monthly transitions. - Thats why the long-term 31-day interval variant is the preferred method and is enabled by - default for all scaling-based techniques. diff --git a/src/CMethods.cxx b/src/CMethods.cxx index d746aed..8d04347 100644 --- a/src/CMethods.cxx +++ b/src/CMethods.cxx @@ -54,9 +54,11 @@ */ std::vector CMethods::scaling_method_names = { - "linear_scaling", "variance_scaling", "delta_method"}; + "linear_scaling", "variance_scaling", "delta_method" +}; std::vector CMethods::distribution_method_names = { - "quantile_mapping", "quantile_delta_mapping"}; + "quantile_mapping", "quantile_delta_mapping" +}; /** * * ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- @@ -624,13 +626,13 @@ void CMethods::Quantile_Mapping( } else { for (unsigned ts = 0; ts < v_scenario.size(); ts++) { - double y = MathUtils::interpolate(v_xbins, contr_cdf, (double)v_scenario[ts], true); // Eq. 1 + double y = MathUtils::interpolate(v_xbins, contr_cdf, (double)v_scenario[ts], true); // Eq. 2 cdf_values.push_back((y >= 0) ? y : 0); } // ? Invert in invers CDF and return for (unsigned ts = 0; ts < v_scenario.size(); ts++) { - float y = (float)MathUtils::interpolate(ref_cdf, v_xbins, cdf_values[ts], true); // Eq. 1 + float y = (float)MathUtils::interpolate(ref_cdf, v_xbins, cdf_values[ts], true); // Eq. 2 v_output[ts] = (y >= 0) ? y : 0; } } From b53b8bc237b8304e6bb861943fdd9a7640b3002c Mon Sep 17 00:00:00 2001 From: Benjamin Thomas Schwertfeger Date: Fri, 2 Feb 2024 16:06:11 +0100 Subject: [PATCH 2/3] adjust workflow --- .github/workflows/_build_doc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_build_doc.yaml b/.github/workflows/_build_doc.yaml index 973c019..3afd7c2 100644 --- a/.github/workflows/_build_doc.yaml +++ b/.github/workflows/_build_doc.yaml @@ -32,7 +32,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install -r docs/requirements.txt + python -m pip install -r doc/requirements.txt - name: Build the documentation run: make doc From ea1771a97880a9395c8deb3fd20c887d865aae68 Mon Sep 17 00:00:00 2001 From: Benjamin Thomas Schwertfeger Date: Fri, 2 Feb 2024 16:10:26 +0100 Subject: [PATCH 3/3] update github action versions --- .github/workflows/_build.yaml | 2 +- .github/workflows/_build_doc.yaml | 4 ++-- .github/workflows/_pre_commit.yaml | 2 +- .github/workflows/_test.yaml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index dc3bcf0..53bdafb 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -17,7 +17,7 @@ jobs: image: alpine:3.17 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: | diff --git a/.github/workflows/_build_doc.yaml b/.github/workflows/_build_doc.yaml index 3afd7c2..9b4a44c 100644 --- a/.github/workflows/_build_doc.yaml +++ b/.github/workflows/_build_doc.yaml @@ -22,10 +22,10 @@ jobs: runs-on: ${{ inputs.os }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python ${{ inputs.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} diff --git a/.github/workflows/_pre_commit.yaml b/.github/workflows/_pre_commit.yaml index 68e6781..2c8a4f8 100644 --- a/.github/workflows/_pre_commit.yaml +++ b/.github/workflows/_pre_commit.yaml @@ -15,5 +15,5 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - uses: pre-commit/action@v3.0.0 diff --git a/.github/workflows/_test.yaml b/.github/workflows/_test.yaml index 1ab7acf..b12bf0d 100644 --- a/.github/workflows/_test.yaml +++ b/.github/workflows/_test.yaml @@ -17,7 +17,7 @@ jobs: image: alpine:3.17 steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: |