OpenCOMPES · rettigl · Mar 2, 2024 · Feb 29, 2024 · Feb 29, 2024 · Feb 29, 2024
diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -0,0 +1,98 @@
+name: documentation
+on:
+  # Triggers the workflow on push but only for the main branch
+  push:
+#    branches: [ main ]
+
+    paths:
+      - specsanalyzer/**/*
+      - specsscan/**/*
+      - tutorial/**
+      - .github/workflows/documentation.yml
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+
+  # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
+# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
+concurrency:
+  group: "pages"
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      #  Check out repo and set up Python
+      - name: Check out the repository
+        uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      # see https://stackoverflow.com/questions/57612428/cloning-private-github-repository-within-organisation-in-actions and https://docs.github.com/en/authentication/connecting-to-github-with-ssh/generating-a-new-ssh-key-and-adding-it-to-the-ssh-agent#generating-a-new-ssh-key
+      - name: checkout test data
+        run: |
+          eval `ssh-agent -s`
+          ssh-add - <<< '${{ secrets.TEST_DATA_ACCESS_KEY }}'
+          git submodule sync --recursive
+          git submodule update --init --recursive
+
+      # Use cached python and dependencies, install poetry
+      - name: "Setup Python, Poetry and Dependencies"
+        uses: packetcoders/action-setup-cache-python-poetry@main
+        with:
+          python-version: 3.8
+          poetry-version: 1.2.2
+
+      - name: Install notebook dependencies
+        run: poetry install -E notebook --with docs
+
+      - name: Install pandoc
+        run: |
+          sudo wget https://github.com/jgm/pandoc/releases/download/3.1.8/pandoc-3.1.8-1-amd64.deb
+          sudo dpkg -i pandoc-3.1.8-1-amd64.deb
+
+      # rm because hextof_workflow notebook can not run outside maxwell
+      - name: copy tutorial files to docs
+        run: |
+          cp -r $GITHUB_WORKSPACE/tutorial $GITHUB_WORKSPACE/docs/
+          cp -r $GITHUB_WORKSPACE/specsscan/config $GITHUB_WORKSPACE/docs/specsscan/
+          mkdir $GITHUB_WORKSPACE/docs/tests
+          cp -r $GITHUB_WORKSPACE/tests/data $GITHUB_WORKSPACE/docs/tests/
+
+      # To be included later
+      # - name: Cache docs build
+      #   id: cache-docs
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: $GITHUB_WORKSPACE/_build
+      #     key: ${{ runner.os }}-docs
+
+      - name: build Sphinx docs
+        run: poetry run sphinx-build -b html $GITHUB_WORKSPACE/docs $GITHUB_WORKSPACE/_build
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v3
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: '_build'
+
+  # Deployment job
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2
diff --git a/.github/workflows/testing_coverage.yml b/.github/workflows/testing_coverage.yml
@@ -40,8 +40,8 @@ jobs:
           poetry run pytest --cov --cov-report xml:cobertura.xml --full-trace --show-capture=no -sv -n auto tests/
 
       # Take report and upload to coveralls
-#      - name: Coveralls
-#        uses: coverallsapp/github-action@v2
-#        with:
-#          github-token: ${{ secrets.GITHUB_TOKEN }}
-#          file: ./cobertura.xml
+      - name: Coveralls
+        uses: coverallsapp/github-action@v2
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          file: ./cobertura.xml
diff --git a/.gitignore b/.gitignore
@@ -133,3 +133,8 @@ dmypy.json
 
 # SPECS config files
 *.calib2d
+
+# result files
+*.h5
+*.tiff
+*.nxs
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -13,7 +13,7 @@ repos:
 
 -   repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.1.7
+    rev: v0.2.2
     hooks:
       # Run the formatter.
       - id: ruff-format

diff --git a/README.md b/README.md
@@ -1,3 +1,4 @@
+[![Documentation Status](https://github.com/OpenCOMPES/specsanalyzer/actions/workflows/documentation.yml/badge.svg)](https://opencompes.github.io/specsanalyzer/)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 ![](https://github.com/OpenCOMPES/specsanalyzer/actions/workflows/linting.yml/badge.svg)
 ![](https://github.com/OpenCOMPES/specsanalyzer/actions/workflows/testing_multiversion.yml/badge.svg?branch=main)
@@ -7,10 +8,14 @@
 [![Coverage Status](https://coveralls.io/repos/github/OpenCOMPES/specsanalyzer/badge.svg?branch=main&kill_cache=1)](https://coveralls.io/github/OpenCOMPES/specsanalyzer?branch=main)
 
 # specsanalyzer
+This is the package `specsanalyzer` for conversion and handling of SPECS Phoibos analyzer data.
+
 This package contains two modules:
 `specsanalyzer` is a package to import and convert MCP analyzer images from SPECS Phoibos analyzers into energy and emission angle/physical coordinates.
 `specsscan` is a Python package for loading Specs Phoibos scans accquired with the labview software developed at FHI/EPFL
 
+Tutorials for usage and the API documentation can be found in the [Documentation](https://opencompes.github.io/specsanalyzer/)
+
 ## Installation
 
 ### Pip (for users)
@@ -40,6 +45,12 @@ pip install specsanalyzer
 ```bash
 python -m ipykernel install --user --name=specs_kernel
 ```
+
+#### Configuration and calib2d file
+The conversion procedures require to set up several configuration parameters in a config file. An example config file is provided as part of the package (see documentation). Configuration files can either be passed to the class constructures, or are read from system-wide or user-defined locations (see documentation).
+
+Most importantly, conversion of analyzer data to energy/angular coordinates requires detector calibration data provided by the manufacturer. The corresponding *.calib2d file (e.g. phoibos150.calbid2d) are provided together with the spectrometer software, and need to be set in the config file.
+
 ### For Contributors
 
 To contribute to the development of `specsanalyzer`, you can follow these steps:

diff --git a/docs/conf.py b/docs/conf.py
@@ -1,11 +1,3 @@
-import os
-import sys
-
-import specsanalyzer
-
-
-sys.path.insert(0, os.path.abspath(".."))
-
 # Configuration file for the Sphinx documentation builder.
 #
 # This file only contains a selection of the most common options. For a full
@@ -16,30 +8,33 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
+import os
+import sys
+
+import tomlkit
+
+
+sys.path.insert(0, os.path.abspath(".."))
 
 # -- Project information -----------------------------------------------------
 
-# The suffix of source filenames.
-source_suffix = ".rst"
 
-# The encoding of source files.
-# source_encoding = 'utf-8-sig'
+def _get_project_meta():
+    with open("../pyproject.toml") as pyproject:
+        file_contents = pyproject.read()
 
-# The master toctree document.
-master_doc = "index"
+    return tomlkit.parse(file_contents)["tool"]["poetry"]
 
-# General information about the project.
-project = "specsanalyzer"
-copyright = "2022, Laurenz Rettig, Michele Puppin, Abeer Arora"
 
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-version = specsanalyzer.__version__
-# The full version, including alpha/beta/rc tags.
-release = specsanalyzer.__version__
+pkg_meta = _get_project_meta()
+project = str(pkg_meta["name"])
+copyright = "2024, OpenCOMPES team"
+author = "OpenCOMPES team"
+
+# The short X.Y version
+version = str(pkg_meta["version"])
+# The full version, including alpha/beta/rc tags
+release = version
 
 # -- General configuration ---------------------------------------------------
 
@@ -55,8 +50,13 @@
     "sphinx.ext.autosummary",
     "sphinx.ext.coverage",
     "sphinx_autodoc_typehints",
+    # "bokeh.sphinxext.bokeh_autodoc",
+    # "bokeh.sphinxext.bokeh_plot",
+    "nbsphinx",
+    "myst_parser",
 ]
 
+
 autoclass_content = "class"
 autodoc_member_order = "bysource"
 

diff --git a/docs/examples/example.rst b/docs/examples/example.rst
diff --git a/docs/getting_started.rst b/docs/getting_started.rst
@@ -0,0 +1,2 @@
+..  include:: ../README.md
+    :parser: myst_parser.sphinx_
diff --git a/docs/index.rst b/docs/index.rst
@@ -8,23 +8,33 @@ Welcome to specsanalyzer's documentation!
 Concept description. TODO
 
 .. toctree::
-   :maxdepth: 3
+   :maxdepth: 1
+   :caption: Getting Started
+
+   getting_started
+   specsanalyzer/config
+   tutorial/1_specsanalyzer_conversion_examples
+   tutorial/2_specsscan_example
+   tutorial/3_specsscan_conversion_to_NeXus
+   tutorial/4_specsscan_load_sweep_scan
+
+.. toctree::
+   :maxdepth: 1
    :numbered:
    :caption: SpecsAnalyzer Core Modules
 
    specsanalyzer/core
    specsanalyzer/convert
    specsanalyzer/img_tools
    specsanalyzer/io
-   specsanalyzer/metadata
-   specsanalyzer/settings
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
    :numbered:
-   :caption: Examples
+   :caption: SpecsScan Core Modules
 
-   examples/example
+   specsscan/core
+   specsscan/helpers
 
 .. toctree::
    :maxdepth: 2

diff --git a/docs/specsanalyzer/config.rst b/docs/specsanalyzer/config.rst
@@ -0,0 +1,39 @@
+Config
+===================================================
+The config module contains a mechanis to collect configuration parameters from various sources and configuration files, and to combine them in a hierachical manner into a single, consistent configuration dictionary.
+It will load an (optional) provided config file, or alternatively use a passed python dictionary as initial config dictionary, and subsequently look for the following additional config files to load:
+
+* ``folder_config``: A config file of name :file:`specs_config.yaml` in the current working directory. This is mostly intended to pass calibration parameters of the workflow between different notebook instances.
+* ``user_config``: A config file provided by the user, stored as :file:`.specsanalyzer/config.yaml` in the current user's home directly. This is intended to give a user the option for individual configuration modifications of system settings.
+* ``system_config``: A config file provided by the system administrator, stored as :file:`/etc/specsanalyzer/config.yaml` on Linux-based systems, and :file:`%ALLUSERSPROFILE%/specsanalyzer/config.yaml` on Windows. This should provide all necessary default parameters for using the specsanalyzer processor with a given setup. For an example for the setup at the Fritz Haber Institute setup, see :ref:`example_config`
+* ``default_config``: The default configuration shipped with the package. Typically, all parameters here should be overwritten by any of the other configuration files.
+
+The config mechanism returns the combined dictionary, and reports the loaded configuration files. In order to disable or overwrite any of the configuration files, they can be also given as optional parameters (path to a file, or python dictionary).
+
+
+API
+***************************************************
+.. automodule:: specsanalyzer.config
+   :members:
+   :undoc-members:
+
+
+.. _example_config:
+
+Default specsanalyzer configuration settings
+***************************************************
+
+.. literalinclude:: ../../specsanalyzer/config/default.yaml
+   :language: yaml
+
+Default specsscan configuration settings
+***************************************************
+
+.. literalinclude:: ../../specsscan/config/default.yaml
+   :language: yaml
+
+Example configuration file for the trARPES setup at FHI-Berlin
+*********************************************************************************
+
+.. literalinclude:: ../../specsscan/config/example_config_FHI.yaml
+   :language: yaml
diff --git a/docs/specsanalyzer/metadata.rst b/docs/specsanalyzer/metadata.rst
diff --git a/docs/specsanalyzer/settings.rst b/docs/specsanalyzer/settings.rst
diff --git a/docs/specsscan/settings.rst → docs/specsscan/helpers.rst b/docs/specsscan/settings.rst → docs/specsscan/helpers.rst
@@ -1,5 +1,5 @@
-Settings
+Helpers
 ===================================================
-.. automodule:: specsscan.settings
+.. automodule:: specsscan.helpers
    :members:
    :undoc-members:
diff --git a/docs/specsscan/io.rst b/docs/specsscan/io.rst
diff --git a/docs/specsscan/metadata.rst b/docs/specsscan/metadata.rst
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		.. include:: ../README.md
		:parser: myst_parser.sphinx_