Merge pull request #144 from AICONSlab/dev

Release 2.4.0

.circleci/config.yml

@@ -54,7 +54,7 @@ jobs:
           no_output_timeout: 2h
           command: |
             # Log in to cloud.sylabs.io
-            echo $SYLABS_TOKEN_MAR_08_2024 > tokenfile && \
+            echo $SYLABS_TOKEN_JUL_10_2024 > tokenfile && \
             singularity remote login -u aiconslab --tokenfile tokenfile && \
             # Create Singularity container remotely and push to cloud
             singularity build miracl.sif docker://mgoubran/miracl:latest

docs/installation/installation.rst

@@ -165,6 +165,11 @@ either of the following methods:
                - '/home/mgoubran/.Xauthority:/home/mgoubran/.Xauthority'
                - '/home/mgoubran/mydata:/home/mgoubran/mydata'
 
+      The format of mounting volumes is ``<host_path>:<container_path>``.
+      In the above example, the host path ``/home/mgoubran/mydata`` is mounted
+      to the container path ``/home/mgoubran/mydata``. Note that in general
+      these paths can be different.
+
       .. raw:: html
 
          <h2>Stopping the container</h2>

docs/tutorials/stats/ace_cluster/ace_cluster.rst

@@ -21,15 +21,23 @@ use the following command on the CLI:
    $ miracl stats ace -h
 
 
+Main outputs
+============
 
-
-
-
-
-
-
-
-
+.. code-block::
+   
+   ./
+   |-- clust_final/
+      |-- f_obs.nii.gz
+      |-- p_values.nii.gz
+      |-- pvalue_heatmap_mean_plot.tiff
+   |-- corr_final/
+
+- ``clust_final``: Contains the cluster-wise TFCE permutation statistics at the atlas space
+  (``f_obs.nii.gz``), the p-value image of the F-statistics (``p_values.nii.gz``), and the p-value heatmap
+  projected onto the Allen atlas space (``pvalue_heatmap_mean_plot.tiff``). All p-values are expressed
+  as ``-log10(p-value)``.
+- ``corr_final``: Contains the correlation analysis output including correlation maps and p_value maps.
 
 
 Example usage (`link to sample data <https://huggingface.co/datasets/AICONSlab/MIRACL/resolve/dev/sample_data/ace/ace_sample_data_stats.zip>`_):

docs/tutorials/stats/index.rst

@@ -9,5 +9,5 @@ This module consists of multiple sub-modules to apply statistical and correlatio
    :maxdepth: 2
    :caption: Table of contents:
 
-   ACE CLuster only **new** <ace_cluster/ace_cluster>
+   ACE Cluster only **new** <ace_cluster/ace_cluster>
 

docs/tutorials/workflows/ace_flow/ace_flow.rst

@@ -18,11 +18,24 @@ ACE Workflow
 Installation
 ============
 
-To install the ACE workflow, refer to the MIRACL installation guide and video tutorials:
+To install the ACE workflow, refer to the MIRACL installation guide:
 
 - :doc:`Installation guide <../../../installation/installation>`
 
-- `Video tutorial <https://www.google.ca/>`_
+Video Tutorial
+==============
+
+`Video tutorial <https://www.youtube.com/playlist?list=PLZeAd6YsEkyhWsHuym5dTV2wjQ299ekm8>`_
+
+- This video tutorial covers the following topics:
+   - MIRACL installation validation
+   - Download sample data
+   - Run ACE on a single subject (Mode 2) including deep learning
+     segmentation of cFos+ cells, registration, voxelization, and warping
+   - Analyze the results of the above step
+   - Run ACE cluster-wise statistical algorithm between two groups to map
+     local cell activation
+   - Analyze the results of the above step
 
 .. TODO: update the tutorial link
 
@@ -231,17 +244,70 @@ Main outputs
 
 .. code-block::
 
-   seg_final # segmentation output including model(s) outputs and uncertainty estimates
-   conv_final # conversion (tiff to nifti) output
-   clar_allen_reg # registration output / preliminary files  
-   reg_final  # main registration output 
-   vox_final 
-   warp_final
+   final_ctn_down_<CONVERSION DOWNSAMPLE RATIO>_rca_voxel_size_<REGISTRATION VOXEL SIZE>/ # main output folder
+   |-- seg_final/
+      |-- ...
+   |-- conv_final/
+      |-- <CONVERSION NAME>.nii.gz
+   |-- clar_allen_reg/
+      |-- ...
+   |-- reg_final/
+      |-- annotation_*_tiff_clar/
+      |-- annotation_*_clar_space_downsample.nii.gz
+      |-- annotation_*_clar_downsample.nii.gz
+      |-- clar_downsample*.nii.gz
+   |-- vox_final/
+      |-- voxelized_seg_seg.nii.gz
+   |-- warp_final/
+      |-- voxelized_seg_seg_allen_space.nii.gz
    
    # the following outputs are generated only in Mode 1
-   heatmap_final
-   cluster_final # cluster-wise analysis output including p_value and f_stats maps
-   corr_final # correlation analysis output including correlation maps and p_value maps
+   |-- heatmap_final/
+      |-- group_1_mean_plot.tiff
+      |-- group_2_mean_plot.tiff
+      |-- group_difference_mean_plot.tiff
+   |-- clust_final/
+      |-- f_obs.nii.gz
+      |-- p_values.nii.gz
+      |-- pvalue_heatmap_mean_plot.tiff
+   |-- corr_final/
+
+- ``seg_final``: Contains the segmentation output (binary) including model(s) outputs (and
+  uncertainty estimates) in slice format that match with the raw data naming.
+- ``conv_final``: Contains the conversion (tiff to nifti) output. The name of this file depends
+  on the parameters used in conversion. This will be the only file in this directory.
+- ``clar_allen_reg``: Contains the registration outputs / preliminary files.
+- ``reg_final``: Contains the main registration outputs.
+
+   - ``annotation_*_tiff_clar``: Contains the atlas annotations in native space.
+     These are saved in slice format, with the same naming as the raw input
+   - ``annotation_*_clar_space_downsample.nii.gz``: Contains the atlas annotations
+     in native space in nifti format. Can be overlaid on the conversion output.
+   - ``annotation_*_clar_downsample.nii.gz``: Contains the atlas annotations in
+     atlas space in nifti format. Can be overlaid on ``clar_downsample*.nii.gz``.
+   - ``clar_downsample*.nii.gz``: Contains the downsampled conversion output 
+     warped to atlas space.
+
+.. note::
+
+   The ``annotation_*_clar_downsample.nii.gz`` file is the most important file in this directory.
+   Please overlay this file on the ``clar_downsample*.nii.gz`` file to visualize and check the
+   registration output in native space and make sure it is correct.
+
+- ``vox_final``: Contains the voxelized segmentation output.
+- ``warp_final``: Contains the voxelized + warped segmentation output. This file
+  is in atlas space.
+- ``heatmap_final``: Contains the group-wise heatmaps of cell density using the average
+  of voxelized and warped segmentation maps in each group. Also contains the group difference
+  heatmap. ``group_1_mean_plot.tiff`` is for the control group, ``group_2_mean_plot.tiff``
+  is for the treated group.
+- ``clust_final``: Contains the cluster-wise TFCE permutation statistics at the atlas space
+  (``f_obs.nii.gz``), the p-value image of the F-statistics (``p_values.nii.gz``), and the p-value heatmap
+  projected onto the Allen atlas space (``pvalue_heatmap_mean_plot.tiff``). All p-values are expressed
+  as ``-log10(p-value)``.
+- ``corr_final``: Contains the correlation analysis output including correlation maps and p_value maps.
+
+
 
 .. _example_anchor:
 
@@ -254,7 +320,8 @@ Example of running ACE flow on multiple subjects (Mode 1):
       --control ./non_walking/ ./non_walking/Newton_HC1/cells/ \
       --treated ./walking/ ./walking/Newton_UI1/cells/ \
       --sa_output_folder ./output_dir \
-      --sa_model_type unet
+      --sa_model_type unet \
+      --sa_resolution 1.4 1.4 5.0
 
 
 Example of running ACE on single subject (Mode 2) (`link to sample data <https://huggingface.co/datasets/AICONSlab/MIRACL/resolve/dev/sample_data/ace/ace_sample_data_mode_2.zip>`__):
@@ -344,11 +411,13 @@ Example of running only ACE cluster wise analysis on voxelized and warped segmen
    you want to download. For this tutorial, you will need to
    download option ``2``.
 
-   $ miracl stats ace \
-      --control ./ctrl/ \
-      --treated ./treated/ \
-      --sa_output_folder ./output_dir \
-      --rwc_voxel_size 25
+   .. code-block::
+
+      $ miracl stats ace \
+         --control ./ctrl/ \
+         --treated ./treated/ \
+         --sa_output_folder ./output_dir \
+         --rwc_voxel_size 25
 
 More information on the ``miracl stats ace`` function can be found
 :doc:`here <../../stats/ace_cluster/ace_cluster>`.

install.sh

@@ -39,9 +39,9 @@ miracl_version_file=$(cat ./miracl/version.txt)
 
 # Set container memory size limit
 if [[ "$os" == "Linux" ]]; then
-    mem=$(grep MemTotal /proc/meminfo | awk '{printf "%dmb", int($2/1024*0.85)}')
+    shm_mem=$(grep MemTotal /proc/meminfo | awk '{printf "%dmb", int($2/1024*0.85)}')
 elif [[ "$os" == "Darwin" ]]; then
-    mem=$(sysctl hw.memsize | awk '{printf "%dmb", int($2/1024/1024*0.85)}')
+    shm_mem=$(sysctl hw.memsize | awk '{printf "%dmb", int($2/1024/1024*0.85)}')
 else
     echo "Unsupported operating system: $os"
     exit 1
@@ -75,7 +75,7 @@ function usage()
    -t, set when using specific MIRACL tag/version. Use 'auto' to parse from 'miracl/version.txt' or specify version as floating point value in format 'x.x.x' (default: 'latest')
    -g, enable Nvidia GPU passthrough mode for Docker container which is required for some of MIRACL's scripts e.g. ACE segmentation (default: false)
    -e, disable mounting MIRACL's script directory into Docker container. Mounting is useful if you want host changes to propagate to the container directly (default: enabled)
-   -d, manually set limit on max amount of memory the container can use ('--memory'). E.g. '1024mb', '16gb' or '512gb'. This is good practice for memory intensive applications like ACE (default: int(MemTotal/1024)*0.85 of host machine)
+   -d, set shared memory (shm) size (e.g. '1024mb', '16gb' or '512gb') which is important for e.g ACE (default: int(MemTotal/1024)*0.85 of host machine)
    -v, mount volumes for MIRACL in docker-compose.yml, using a separate flag for each additional volume (format: '/path/on/host:/path/in/container'; default: none)
    -l, write logfile of build process to 'build.log' in MIRACL root directory (default: false)
    -s, print version of build script and exit
@@ -131,8 +131,8 @@ while getopts ":n:i:c:t:ged:v:lsmh" opt; do
       ;;
 
     d)
-      if [ "${OPTARG}" != "$mem" ]; then
-      mem=${OPTARG}
+      if [ "${OPTARG}" != "$shm_mem" ]; then
+      shm_mem=${OPTARG}
       fi
       ;;
 
@@ -191,7 +191,7 @@ services:
     stdin_open: true
     network_mode: host
     container_name: $container_name
-    mem_limit: $mem
+    shm_size: ${shm_mem}
 EOF
 
 if [[ $gpu ]]; then
@@ -285,7 +285,7 @@ if [ -x "$(command -v docker)" ]; then
       printf " User: %s\n" "$HOST_USER"
       printf " pid: %s\n" "$(id -u)"
       printf " gid: %s\n" "$(id -g)"
-      printf " Max memory: %s\n" "$mem"
+      printf " Max shared memory: %s\n" "$shm_mem"
       printf " Service name: %s\n" "$service_name"
       printf " Image name: %s\n" "$image_name:$miracl_version"
       printf " Container name: %s\n" "$container_name"
@@ -323,9 +323,9 @@ if [ -x "$(command -v docker)" ]; then
         # Test if docker-compose is installed
         # Should come by default with Docker-Desktop
         if [ -x "$(command -v docker-compose)" ]; then
-          printf "Docker Compose installation found (%s). Run 'docker-compose up -d' to start the ${container_name} container in background.\n" "Run 'docker exec -it ${container_name} bash' to enter the container.\n" "$(docker-compose --version)"
+          printf "Docker Compose installation found (%s). Run 'docker-compose up -d' to start the container in the background and then run 'docker exec -it ${container_name} bash' to enter the container.\n" "$(docker-compose --version)"
         elif dcex=$(docker compose version); then
-          printf "Docker Compose installation found (%s). Run 'docker compose up -d' or use Docker Desktop (if installed) to start the ${container_name} container in background.\n" "Run 'docker exec -it ${container_name} bash' to enter the container.\n" "$dcex"
+          printf "Docker Compose installation found (%s). Run 'docker compose up -d' or use Docker Desktop (if installed) to start the container in the background and then run 'docker exec -it ${container_name} bash' to enter the container.\n" "$dcex"
         else
           printf "Docker Compose installation not found. Please install Docker Compose plugin or standalone version to run the MIRACL container. Instructions on how to install Docker Compose can be found here: https://docs.docker.com/compose/install/\n"
         fi

miracl/flow/ace_gui/README.md

@@ -0,0 +1,11 @@
+# ACE GUI
+
+## Add tab
+
+- Create `miracl_workflow_ace_gui_clusterwise_tab.py`
+	- Import from `QWidget` and `QFormLayout` from `PyQt5.QtWidgets`, `WidgetUtils` from `miracl_workflow_ace_gui_widgets_utils` and `miracl_workflow_ace_parser` from `miracl.flow`
+- Create `ClusterwiseTab` class including the clusterwise layout and the help dictionary created from the parser
+	- Add the clusterwise widgets
+- Import into `miracl_workflow_ace_gui_clusterwise_tab` into `miracl_workflow_ace_gui_tab_manager.py`
+- Create flag associations with `miracl_workflow_ace_gui_flag_creator.py`
+- Add clusterwise flags to `miracl_workflow_ace_gui_controller.py`

miracl/flow/ace_gui/miracl_workflow_ace_gui_clarity_registration_tab.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+CLARITY Registration
+
+Description:
+    View for CLARITY registration. 
+
+Copyright:
+    (c) 2024 AICONs Lab. All rights reserved.
+
+Author:
+    Jonas Osmann
+    j.osmann@mail.utoronto.ca
+
+License:
+    GPL-3.0
+"""
+
+from PyQt5.QtWidgets import (
+    QWidget,
+    QFormLayout,
+)
+
+from PyQt5.QtCore import Qt
+from .miracl_workflow_ace_gui_widget_utils import WidgetUtils as wu
+from miracl.flow import miracl_workflow_ace_parser
+
+
+class ClarityRegistrationTab(QWidget):
+    def __init__(self):
+        super().__init__()
+        clarity_registration_layout = QFormLayout()
+        self.setLayout(clarity_registration_layout)
+        args_parser = miracl_workflow_ace_parser.ACEWorkflowParser()
+        help_dict = wu.extract_help_texts(args_parser)
+
+        self.reg_hemi_input = wu.create_multiple_choice(
+            clarity_registration_layout,
+            "Labels hemisphere",
+            help_dict["rca_hemi"],
+            ["combined", "split"],
+            "combined",
+        )
+
+        (
+            self.reg_allen_lbl_warp_label_input,
+            self.reg_allen_lbl_warp_path_input,
+            self.reg_allen_lbl_warp_button_input,
+        ) = wu.create_path_input_widget(
+            self,
+            clarity_registration_layout,
+            "Allen labels to warp:",
+            help_dict["rca_allen_label"],
+            "Select file",
+            select_files=True,
+            file_filter="All Files (*)",
+        )
+
+        (
+            self.reg_cust_allen_atlas_label_input,
+            self.reg_cust_allen_atlas_path_input,
+            self.reg_cust_allen_atlas_button_input,
+        ) = wu.create_path_input_widget(
+            self,
+            clarity_registration_layout,
+            "Custom Allen atlas:",
+            help_dict["rca_allen_atlas"],
+            "Select file",
+            select_files=True,
+            file_filter="All Files (*)",
+        )
+
+        self.reg_side_input = wu.create_multiple_choice(
+            clarity_registration_layout,
+            "Side:",
+            help_dict["rca_side"],
+            ["right hemisphere", "left hemisphere"],
+            "right hemisphere",
+        )
+
+        self.reg_mosaic_figure_input = wu.create_multiple_choice(
+            clarity_registration_layout,
+            "Create mosaic figure:",
+            help_dict["rca_no_mosaic_fig"],
+            ["yes", "no"],
+            "yes",
+        )
+
+        self.reg_olfactory_bulb_input = wu.create_multiple_choice(
+            clarity_registration_layout,
+            "Olfactory bulb incl.:",
+            help_dict["rca_olfactory_bulb"],
+            ["not included", "included"],
+            "not included",
+        )
+
+        self.reg_util_int_correction_input = wu.create_multiple_choice(
+            clarity_registration_layout,
+            "Utilfn intensity correction:",
+            help_dict["rca_skip_cor"],
+            ["run", "skip"],
+            "run",
+        )
+
+        self.reg_warp_to_allen_input = wu.create_multiple_choice(
+            clarity_registration_layout,
+            "Warp CLARITY to Allen:",
+            help_dict["rca_warp"],
+            ["yes", "no"],
+            "no",
+        )