Merge pull request #216 from mala-project/develop

Create v0.2.0

.bumpversion.cfg

@@ -0,0 +1,11 @@
+[bumpversion]
+current_version = 0.1.0
+commit = True
+tag = True
+sign_tags = True
+
+[bumpversion:file:mala/version.py]
+
+[bumpversion:file:CITATION.cff]
+
+[bumpversion:file:Copyright.txt]

.gitattributes

@@ -0,0 +1 @@
+*.cff   linguist-language=YAML

.github/workflows/cpu-tests.yml

@@ -1,6 +1,8 @@
 name: CPU
 
 on:
+  pull_request_review:
+    types: [submitted,edited]
   pull_request:
     branches:
       - master
@@ -43,6 +45,7 @@ jobs:
         run: docker build . --file Dockerfile --tag $IMAGE_NAME --cache-from=$IMAGE_ID --build-arg DEVICE=cpu --label "runnumber=${GITHUB_RUN_ID}"
 
       - name: Push image
+        if: ${{ github.event.review.state == 'approved' || github.event_name == 'push' }}
         run: |
           # Strip git ref prefix from version
           VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
@@ -72,15 +75,15 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Install mala package
-        run: pip --no-cache-dir install -e .
-
+        run: |
+          pip --no-cache-dir install -e .
+          pip install pytest
       - name: Check out repository (data)
         uses: actions/checkout@v2
         with:
           repository: mala-project/test-data
-          token: ${{ secrets.ACCESS_TOKEN }}
           path: mala_data
-          ref: v0.1.0
+          ref: v1.1.0
           lfs: false
 
       - name: Data setup
@@ -90,8 +93,5 @@ jobs:
           chmod +x ../install/data_repo_link/link_data_repo.sh
           ../install/data_repo_link/link_data_repo.sh `pwd`
 
-      - name: Test basic functions
-        run: python test/mala_tests.py
-
-      - name: Test workflow
-        run: python examples/ex99_verify_all_examples.py
+      - name: Test MALA
+        run: pytest --disable-warnings

.github/workflows/gh-pages.yml

@@ -1,6 +1,8 @@
 name: docs
 
 on:
+  pull_request_review:
+    types: [submitted,edited]
   pull_request:
     branches:
       - master
@@ -30,7 +32,7 @@ jobs:
       - name: Check docstrings
         run: pydocstyle --convention=numpy mala
 
-  deploy-pages:
+  build-and-deploy-pages:
     needs: test-docstrings
     runs-on: ubuntu-18.04
     steps:
@@ -58,7 +60,8 @@ jobs:
           mv -v docs/_build/html public
 
       - name: Deploy
+        if: ${{ github.event.review.state == 'approved' || github.event_name == 'push' }}
         uses: peaceiris/actions-gh-pages@v3
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
+          deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
           publish_dir: ./public

.github/workflows/mirror-to-casus.yml

@@ -0,0 +1,16 @@
+name: mirror
+
+on: [push, delete]
+
+jobs:
+  mirror-to-CASUS:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v1
+    - name: mirror-repository
+      uses: spyoungtech/mirror-action@v0.4.0
+      with:
+        REMOTE: git@github.com:casus/mala.git
+        GIT_SSH_PRIVATE_KEY: ${{ secrets.GIT_SSH_KEY }}
+        GIT_SSH_NO_VERIFY_HOST: "true"
+        DEBUG: "true"

.gitignore

@@ -161,3 +161,9 @@ cython_debug/
 # JupyterNotebooks
 .ipynb_checkpoints
 */.ipynb_checkpoints/*
+
+# SQLite
+*.db
+
+# LAMMPS
+*.tmp

CITATION.cff

@@ -0,0 +1,50 @@
+# YAML 1.2
+cff-version: 1.1.0
+message: "If you use this software, please cite it using these metadata."
+authors:
+  - affiliation: "Center for Advanced Systems Understanding (CASUS), Helmholtz-Zentrum Dresden-Rossendorf e.V. (HZDR)"
+    family-names: Cangi
+    given-names: Attila
+  - affiliation: "Oak Ridge National Laboratory (ORNL)"
+    family-names: Ellis
+    given-names: J. Austin
+  - affiliation: "Center for Advanced Systems Understanding (CASUS), Helmholtz-Zentrum Dresden-Rossendorf e.V. (HZDR)"
+    family-names: Fiedler
+    given-names: Lenz
+    orcid: https://orcid.org/0000-0002-8311-0613
+  - affiliation: "Center for Advanced Systems Understanding (CASUS), Helmholtz-Zentrum Dresden-Rossendorf e.V. (HZDR)"
+    family-names: Kotik
+    given-names: Daniel
+    orcid: https://orcid.org/0000-0001-8735-3199
+  - affiliation: "Sandia National Laboratories (SNL)"
+    family-names: Modine
+    given-names: Normand A.
+  - affiliation: "Oak Ridge National Laboratory (ORNL)"
+    family-names: Oles
+    given-names: Vladyslav
+  - affiliation: "Sandia National Laboratories (SNL)"
+    family-names: Popoola
+    given-names: Gabriel A.
+  - affiliation: "Sandia National Laboratories (SNL)"
+    family-names: Rajamanickam
+    given-names: Siva
+  - affiliation: "Helmholtz-Zentrum Dresden-Rossendorf e.V. (HZDR)"
+    family-names: Schmerler
+    given-names: Steve
+  - affiliation: "Sandia National Laboratories (SNL)"
+    family-names: Stephens
+    given-names: J. Adam
+  - affiliation: "Sandia National Laboratories (SNL)"
+    family-names: Thompson
+    given-names: Aidan P.
+
+
+date-released: 2021-07-07
+keywords:
+  - "machine-learning"
+  - "dft"
+license: "BSD-3-Clause"
+repository-code: "https://github.com/mala-project/mala"
+title: mala
+# doi:
+version: 0.1.0

Dockerfile

@@ -4,7 +4,7 @@ FROM continuumio/miniconda3:4.9.2
 ARG DEVICE=cpu
 
 # Update the image to the latest packages
-RUN apt-get update && apt-get upgrade -y
+RUN apt-get --allow-releaseinfo-change update && apt-get upgrade -y
 
 RUN apt-get install --no-install-recommends -y build-essential libz-dev swig git-lfs
 RUN apt-get clean && rm -rf /var/lib/apt/lists/*

README.md

@@ -35,29 +35,42 @@ the examples in the `example/` folder.
 - [Sandia National Laboratories](https://www.sandia.gov/) (SNL), USA.
 - [Center for Advanced Systems Understanding](https://www.casus.science/) (CASUS), Germany.
 
+### Contributing Institutions
+
+- [Oak Ridge National Laboratory](https://www.ornl.gov/) (ORNL), USA
+
 ## Developers
 ### Scientific Supervision
 - Attila Cangi (CASUS)
 - Siva Rajamanickam (SNL)
 
 ### Core Developers
+
+- Austin Ellis (ORNL)
 - Lenz Fiedler (CASUS)
-- Austin Ellis (ORNL*)
-- Normand Modine (SNL)
-- Steve Schmerler (HZDR)
 - Daniel Kotik (CASUS)
+- Normand Modine (SNL)
+- Vladyslav Oles (ORNL)
 - Gabriel Popoola (SNL)
 - Aidan Thompson (SNL)
+- Steve Schmerler (HZDR)
 - Adam Stephens (SNL)
 
-\* Work done as part of postdoctoral research at Sandia National Laboratories
+### Contributors
 
+- Sneha Verma (CASUS)
+- Parvez Mohammed (CASUS)
+- Nils Hoffmann (CASUS)
+- Omar Faruk (CASUS)
+- Somashekhar Kulkarni (CASUS)
 
 ## Citing MALA
 
 If you publish work which uses or mentions MALA, please cite the following paper:
 
 J. A. Ellis, L. Fiedler, G. A. Popoola, N. A. Modine, J. A. Stephens, A. P. Thompson,
-A. Cangi, S. Rajamanickam (2020). Accelerating Finite-temperature
+A. Cangi, S. Rajamanickam (2021). Accelerating Finite-temperature
 Kohn-Sham Density Functional Theory with Deep Neural Networks.
-[arXiv:2010.04905](https://arxiv.org/abs/2010.04905).
+[Phys. Rev. B 104, 035120 (2021)](https://doi.org/10.1103/PhysRevB.104.035120)
+
+alongside this repository.

docs/source/CONTRIBUTE.md

@@ -19,6 +19,9 @@ the form `MAJOR.MINOR.FIX`:
 * `MINOR`: new features have beend added to the code.
 * `FIX`: A bug in a feature has been fixed.
 
+Every new version should be accompanied by a changelog. Please include the 
+version of the test data repository with which this version is supposed to be
+used in this changelog. 
 
 ## Branching strategy
 

docs/source/conf.py

@@ -93,14 +93,20 @@
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = []
 
+html_context = {
+  'display_github': True,
+  'github_repo': 'mala-project/mala',
+  'github_version': 'develop',
+  'conf_py_path': '/docs/source/',
+}
+
 # The name of an image file (relative to this directory) to place at the top
 # of the sidebar.
 #html_logo = "./img/logos/mala_vertical.png"
 
 # The name of an image file (relative to this directory) to use as a favicon of
 # the docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
 # pixels large.
-#html_favicon = "../resources/theme/favicon.ico"
 html_favicon = "./img/logos/mala_favicon.png"
 
 # The suffix of source file names.

docs/source/features/background.rst

@@ -1,22 +1,5 @@
-Background
-===========
-
-Workflow
-*********
-
-The goal of MALA is to build surrogate models for electronic structure theory.
-These surrogate models are based on neural networks. After training such
-a model, it allows the fast evaluation of the total energy and atomic forces.
-MALA is build around Density Functional Theory, but can in
-principle be used with all electronic structure methods that calculate the
-total energy and atomic forces given atomic positions as input.
-Building these surrogate models requires preprocessing
-:doc:`preprocessing <preprocessing>` of the data, training of a
-:doc:`neural network <neuralnetworks>` and
-:doc:`postprocessing <postprocessing>` of the results.
-MALA is designed for the investigation of systems at non-zero temperatures and
-operates in a "per-grid-point" manner, meaning that every grid point of a
-simulation cell is passed through the network individually.
+Theoretical Background
+======================
 
 Density Functional Theory
 *************************

docs/source/features/neuralnetworks.rst

@@ -3,4 +3,37 @@ Neural Networks
 
 Neural networks are powerful machine learning tools in principle capable of
 approximating any function. In MALA, neural networks are built using PyTorch.
-Hyperparameter optimization can be done using optuna and custom routines.
+Hyperparameter optimization can be done using optuna and custom routines.
+
+Data Handling
+#############
+
+MALA provides a ``DataHandler`` class that takes care of all the necessary data
+operations. It loads, scales and performs inference on data, as well as
+unit conversion. Raw simulation data has to be processed into data that can
+be used with a ``DataHandler`` using the ``DataConverter`` class. In MALA,
+data handling works on a per-snapshot basis; i.e. paths to individual snapshots are
+added to the data handler via ``add_snapshot``.
+After all snapshots of interests are given to
+the data handler the necessary routines outlined above can be started with
+``prepare_data``.
+Large datasets can be lazily-loaded into the RAM, i.e. only a subset of the
+entire data will be present in the RAM simultaneously.
+
+Training and inference
+######################
+
+Neural networks are built using ``pytorch`` and the ``Network`` class. They
+can be trained using the ``Trainer`` class and can be tested using the
+``Tester`` class. Currently, only feed-forward neural-networks are implemented.
+
+Hyperparameter optimization
+###########################
+
+Hyperparameter optimization can be done using optuna. Two experimental methods
+are supported as well, orthogonal array tuning and neural architecture search
+without training. In order to perform a hyperparameter optimization, a
+``HyperOptOptuna`` object (for optuna) is created. Hyperparameters can be
+added to the study with the ``add_hyperparameter`` function. Afterwards a
+hyperparameter study can be executed.
+Optuna can be used in a distributed fashion, using e.g. PostgreSQL.

docs/source/features/overview.rst

@@ -0,0 +1,60 @@
+Overview
+========
+
+List of features
+****************
+
+* General features
+
+  * Central parameters class that holds all necessary parameters and allows for saving and loading for later use
+
+* Preprocesing of simulation data
+
+  * Calculation of atomic descriptors from simulation data
+
+  * Parsing of LDOS data files
+
+  * Scaling and Conversion of data
+
+
+* Training of Surrogate models
+
+  * Creation, training and evaluation of feed-forward neural networks
+
+  * Training progress can be checkpointed, neural networks can be saved for later use
+
+  * Distributed training with horovod (experimental)
+
+  * (Distributed) hyperparameter optimization with optuna
+
+  * Hyperparameter optimization with orthogonal array tuning and neural architecure search without training (both experimental)
+
+* Postprocessing of surogate model output
+
+  * LDOS can be used to calculate DOS and/or density
+
+  * Different quantities of interest can be calculated
+
+    * Number of electrons (from LDOS, DOS or density)
+
+    * Band energy (from LDOS or DOS)
+
+    * Total energy (requires QE; from LDOS or DOS + density)
+
+
+Workflow
+********
+
+The goal of MALA is to build surrogate models for electronic structure theory.
+These surrogate models are based on neural networks. After training such
+a model, it allows the fast evaluation of the total energy and atomic forces.
+MALA is build around Density Functional Theory, but can in
+principle be used with all electronic structure methods that calculate the
+total energy and atomic forces given atomic positions as input.
+Building these surrogate models requires preprocessing
+:doc:`preprocessing <preprocessing>` of the data, training of a
+:doc:`neural network <neuralnetworks>` and
+:doc:`postprocessing <postprocessing>` of the results.
+MALA is designed for the investigation of systems at non-zero temperatures and
+operates in a "per-grid-point" manner, meaning that every grid point of a
+simulation cell is passed through the network individually.