Adds native spelunker support (#414)

* adds spelunker as main neuroglancer distro

* adds native spelunker support

* run precommit

* bug fixes and readability improvements

* adds inspect for spelunker tasks

* updates email and disables debug

.codespellrc

@@ -1,7 +1,7 @@
 [codespell]
 # Ref: https://github.com/codespell-project/codespell#using-a-config-file
-skip = .git*,*.svg,package-lock.json,*.css,*-min.*,.codespellrc,*.bundle.*,*.map
+skip = .git*,*.svg,package-lock.json,*.css,*-min.*,.codespellrc,*.bundle.*,*.map, *.js
 check-hidden = true
-# ignore-regex = 
+# ignore-regex =
 # some favorite albeit unfortunate variable names
 ignore-words-list = te

.pre-commit-config.yaml

@@ -14,4 +14,3 @@ repos:
     rev: v2.3.0
     hooks:
     - id: codespell
-

neuvue_project/neuvue/settings.py

@@ -9,6 +9,7 @@
 For the full list of settings and their values, see
 https://docs.djangoproject.com/en/3.2/ref/settings/
 """
+
 import os
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
@@ -24,6 +25,8 @@
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
+SITE_ID = 2
+
 ALLOWED_HOSTS = [
     ".neuvue.io",
     ".elasticbeanstalk.com",
@@ -242,4 +245,7 @@
 
     mimetypes.add_type("application/javascript", ".js", True)
 
-STATIC_NG_FILES = os.listdir(os.path.join(BASE_DIR, "workspace", "static", "workspace"))
+STATIC_NG_FILES = os.listdir(
+    os.path.join(BASE_DIR, "workspace", "static", "workspace")
+) + os.listdir(os.path.join(BASE_DIR, "workspace", "static", "spelunker-workspace"))
+# STATIC_NG_FILES = os.listdir(os.path.join(BASE_DIR, "workspace", "static", "workspace-spelunker"))

neuvue_project/neuvue/urls.py

@@ -47,6 +47,11 @@
     path("tasks/", TaskView.as_view(), name="tasks"),
     path("getting-started/", GettingStartedView.as_view(), name="getting-started"),
     path("workspace/<str:namespace>", WorkspaceView.as_view(), name="workspace"),
+    path(
+        "spelunker-workspace/<str:namespace>",
+        WorkspaceView.as_view(),
+        name="spelunker-workspace",
+    ),
     path("admin/", admin.site.urls),
     path("__debug__/", include("debug_toolbar.urls")),
     path("accounts/", include("allauth.urls")),

neuvue_project/templates/about.html

@@ -82,7 +82,7 @@ <h2 class="pt-5 pb-3"> Collaborators </h2>
         <h2 class="pt-5 pb-3"> Contact Us </h2>
 
         <p>
-            We'd love to hear from you! Reach out at <a class="text-secondary-color-activated" href="mailto:info@neuvue.io">info@neuvue.io</a> with questions or if you are interested in collaborating with us.
+            We'd love to hear from you! Reach out at <a class="text-secondary-color-activated" href="mailto:info@neuvue.io">info@bossdb.org</a> with questions or if you are interested in collaborating with us.
         </p>
 
     </div>

neuvue_project/templates/inspect.html

@@ -101,7 +101,11 @@ <h3 class="text-white mb-3"> Inspect Task </h3>
     </div>
   </div>
 </div>
-<script type="text/javascript" src="{% static "workspace/main.bundle.js" %}"></script>
+{% if ng_host == "spelunker" %}
+<script type="module" src="{% static 'spelunker-workspace/562.1d0cecad9cc0725955f0.js' %}"></script><script type="module" src="{% static 'spelunker-workspace/993.ca23372ceee17151541a.js' %}"></script><script type="module" src="{% static 'spelunker-workspace/367.ad6071abcab399305157.js' %}"></script><script type="module" src="{% static 'spelunker-workspace/main.cb26101b553ed6b82856.js' %}"></script><link href="{% static 'spelunker-workspace/main.cb26101b553ed6b82856.css' %}" rel="stylesheet">
+{% else %}
+<script type="text/javascript" src="{% static 'workspace/main.bundle.js' %}"></script>
+{% endif %}
 <script type="text/javascript" src="{% static "js/utils.js" %}"></script>
 {% endif%}
 <style> .overlay-hidden { display:none; } </style>
@@ -111,18 +115,41 @@ <h3 class="text-white mb-3"> Inspect Task </h3>
 <script type="text/javascript">
 
   function getLink() {
-    viewer.postJsonState(true, undefined, true, function() {
-      let url_prefix = "https://neuroglancer.neuvue.io/?json_url="
-      copyToClipboard(url_prefix.concat(viewer.saver.savedUrl));
-    });
+    {% if ng_host == "neuvue" %}
+        viewer.postJsonState(true, undefined, true, function() {
+            let url_prefix = "https://neuroglancer.neuvue.io/?json_url="
+            copyToClipboard(url_prefix.concat(viewer.saver.savedUrl));
+        });
+    {% elif ng_host == "spelunker" %}
+        let url_prefix = "https://spelunker.cave-explorer.org/#!";
+        copyToClipboard(url_prefix.concat(JSON.stringify(viewer.state.toJSON())));
+    {% else %}
+        //pass
+    {% endif %}
+}
+
+  // Function to attempt restoring the viewer state
+  function tryRestoreState() {
+      if (typeof viewer !== 'undefined') {
+      // Viewer is defined, attempt to restore the state
+      try {
+          var received_data = {{ ng_state|safe }};
+          viewer.state.restoreState(received_data);
+          console.log('Viewer state restored successfully.');
+      } catch (error) {
+          console.error('Error restoring viewer state:', error);
+      }
+      } else {
+      // Viewer is not yet defined, retry after a short delay
+      setTimeout(tryRestoreState, 100); // Retry every 100 milliseconds
+      }
   }
 
   // Neuroglancer State Load
   $(document).ready(function() {
     {% if ng_state %}
-    const state = {{ ng_state|safe }};
-      openSideMenu();
-      viewer.state.restoreState(state);
+    openSideMenu();
+    tryRestoreState();
     {% endif %}
   })
 

neuvue_project/templates/tasks.html

@@ -41,7 +41,11 @@
                                     aria-expanded="false">
                                 <span class="sr-only header-box-text">Toggle Dropdown</span>
                               </button>
+                            {% if context.ng_host == "spelunker" %}
+                            <span class="header-button header-outline rightBorderRounded" onclick="document.location='{% url 'spelunker-workspace' namespace=namespace %}'">
+                            {% else %}
                             <span class="header-button header-outline rightBorderRounded" onclick="document.location='{% url 'workspace' namespace=namespace %}'">
+                            {% endif %}
                                 <button class="header-box-text"
                                     onclick="triggerLoadingSpinner('{{namespace}}NgSpinner')">
                                     <div id='{{namespace}}NgSpinner'>
@@ -77,7 +81,11 @@
                             </div>
                         </div>
                         {% else %}
+                        {% if context.ng_host == "spelunker" %}
+                        <div class="header-sizing header-outline rightBorderRounded" onclick="document.location='{% url 'spelunker-workspace' namespace=namespace %}'">
+                        {% else %}
                         <div class="header-sizing header-outline rightBorderRounded" onclick="document.location='{% url 'workspace' namespace=namespace %}'">
+                        {% endif %}
                             <button class="header-box-text"
                                 onclick="triggerLoadingSpinner('{{namespace}}NgSpinner')">
                                 <div id='{{namespace}}NgSpinner'>

neuvue_project/templates/workspace.html

@@ -278,7 +278,13 @@
 <script src="https://cdn.jsdelivr.net/npm/@yaireo/tagify/dist/tagify.polyfills.min.js"></script>
 <script type="text/javascript"  src="{% static 'js/browser-interaction-time.umd.js' %}"></script>
 <script type="text/javascript" src="{% static 'js/utils.js' %}"></script>
+
+{% if ng_host == "spelunker" %}
+<script type="module" src="{% static 'spelunker-workspace/562.1d0cecad9cc0725955f0.js' %}"></script><script type="module" src="{% static 'spelunker-workspace/993.ca23372ceee17151541a.js' %}"></script><script type="module" src="{% static 'spelunker-workspace/367.ad6071abcab399305157.js' %}"></script><script type="module" src="{% static 'spelunker-workspace/main.cb26101b553ed6b82856.js' %}"></script><link href="{% static 'spelunker-workspace/main.cb26101b553ed6b82856.css' %}" rel="stylesheet">
+{% else %}
 <script type="text/javascript" src="{% static 'workspace/main.bundle.js' %}"></script>
+{% endif %}
+
 <script type="text/javascript">
 
     // initialize Tagify on the above input node reference
@@ -299,14 +305,23 @@
     })
 
     function getLink() {
-        viewer.postJsonState(true, undefined, true, function() {
-            let url_prefix = "https://neuroglancer.neuvue.io/?json_url="
-            copyToClipboard(url_prefix.concat(viewer.saver.savedUrl));
-        });
+        {% if ng_host == "neuvue" %}
+            viewer.postJsonState(true, undefined, true, function() {
+                let url_prefix = "https://neuroglancer.neuvue.io/?json_url="
+                copyToClipboard(url_prefix.concat(viewer.saver.savedUrl));
+                triggerToast("Copied link to clipboard");
+            });
+        {% elif ng_host == "spelunker" %}
+            let url_prefix = "https://spelunker.cave-explorer.org/#!";
+            copyToClipboard(url_prefix.concat(JSON.stringify(viewer.state.toJSON())));
+            triggerToast("Copied link to clipboard");
+        {% else %}
+            triggerToast("Copy Link unavailable on embedded Neuroglancer task type");
+        {% endif %}
     }
 
     function saveState() {
-        {% if not ng_url %}
+        {% if ng_host == "neuvue" %}
         viewer.postJsonState(true, undefined, true, function() {
             let state = viewer.saver.savedUrl;
             let post_body = JSON.stringify({'task_id':"{{task_id}}", 'ng_state' : state});
@@ -320,7 +335,8 @@
                 });
         });
         {% else %}
-        triggerToast("Autosave disabled for embedded neuroglancer");
+        triggerToast("State saving disabled for non-native Neuroglancer");
+        removeLoadingSpinner("save_state", "Save State");
         {% endif %}
     }
 
@@ -373,6 +389,16 @@
         }
     }
 
+    function getOperationIdsFromSpelunker(){
+        for (let i = 0; i < viewer.layerManager.managedLayers.length; i++) {
+            const layer = viewer.layerManager.managedLayers[i].layer_;
+            if (layer.type === 'segmentation') {
+                const operationIds = layer.graphConnection.value.operationIds.toString();
+                return operationIds.split(',').map(Number);
+            }
+        }
+    }
+
     function submitForm(value, form='#mainForm') {
         window.removeEventListener('beforeunload', exitAlert);
         let duration = Math.round(browserTimer.getTimeInMilliseconds()/1000);
@@ -383,8 +409,16 @@
         $('<input>').attr('type', 'hidden').attr('name', 'duration').attr('value', duration).appendTo(form);
         $(form).submit();
 
-        {% else %}
+        {% elif ng_host == "spelunker" %}
+        updateTrackedOperations(getOperationIdsFromSpelunker())
+        let state = JSON.stringify(viewer.state.toJSON());
+        $('<input>').attr('type', 'hidden').attr('name', 'ngState').attr('value', state).appendTo(form);
+        $('<input>').attr('type', 'hidden').attr('name', 'button').attr('value', value).appendTo(form);
+        $('<input>').attr('type', 'hidden').attr('name', 'duration').attr('value', duration).appendTo(form);
+
+        $(form).submit();
 
+        {% else %}
         updateTrackedOperations(operation_ids)
         viewer.postJsonState(true, undefined, true, function() {
             let state = viewer.saver.savedUrl;
@@ -453,6 +487,23 @@
         selected_button.classList.toggle("active");
     }
 
+    // Function to attempt restoring the viewer state
+    function tryRestoreState() {
+        if (typeof viewer !== 'undefined') {
+        // Viewer is defined, attempt to restore the state
+        try {
+            var received_data = {{ ng_state|safe }};
+            viewer.state.restoreState(received_data);
+            console.log('Viewer state restored successfully.');
+        } catch (error) {
+            console.error('Error restoring viewer state:', error);
+        }
+        } else {
+        // Viewer is not yet defined, retry after a short delay
+        setTimeout(tryRestoreState, 100); // Retry every 100 milliseconds
+        }
+    }
+
     // Button Submission
     $(document).ready(function() {
 
@@ -468,13 +519,14 @@
 
         // Specific to built-in NG
         {% if not ng_url%}
-        var received_data = {{ ng_state|safe }};
-        viewer.state.restoreState(received_data);
-
-
+        tryRestoreState()
         // Track operation IDs every three seconds
         window.setInterval(function() {
+            {% if ng_host == "spelunker" %}
+            updateTrackedOperations(getOperationIdsFromSpelunker());
+            {% else %}
             updateTrackedOperations(operation_ids);
+            {% endif %}
         }, 3000);
         {% endif %}
 

neuvue_project/workspace/analytics.py

@@ -6,7 +6,7 @@
 
 # import the logging library
 import logging
-
+import warnings
 logging.basicConfig(level=logging.DEBUG)
 # Get an instance of a logger
 logger = logging.getLogger(__name__)
@@ -111,8 +111,10 @@ def create_stats_table(pending_tasks, closed_tasks):
         for namespace, namespace_df in df.groupby("namespace"):
             n_tasks = len(namespace_df)
             m = namespace_df[status].min()
-            average_event_time = (m + (namespace_df[status] - m)).mean().to_pydatetime()
-            changelog.append((average_event_time, n_tasks, status, namespace))
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                average_event_time = (m + (namespace_df[status] - m)).mean().to_pydatetime()
+                changelog.append((average_event_time, n_tasks, status, namespace))
 
     # Sort changelogs by datetime
     daily_changelog_items = sorted(

neuvue_project/workspace/migrations/0022_alter_namespace_ng_host.py

@@ -0,0 +1,30 @@
+# Generated by Django 4.2.11 on 2024-10-21 20:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("workspace", "0021_alter_namespace_ng_host"),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name="namespace",
+            name="ng_host",
+            field=models.CharField(
+                choices=[
+                    ("neuvue", "NeuVue Legacy"),
+                    ("spelunker", "Spelunker Native"),
+                    ("https://neuroglancer.bossdb.io", "neuroglancer.bossdb.io"),
+                    (
+                        "https://spelunker.cave-explorer.org/",
+                        "spelunker.cave-explorer.org",
+                    ),
+                ],
+                default="neuvue",
+                max_length=100,
+            ),
+        ),
+    ]

neuvue_project/workspace/models.py

@@ -39,10 +39,12 @@ class NeuroglancerHost(models.TextChoices):
     spelunker.cave-explorer.org -> embedded neuroglancer developed by AIBS.
     """
 
-    NEUVUE = "neuvue", _("NeuVue Built-in")
+    NEUVUE = "neuvue", _("NeuVue Legacy")
+    SPELUNKER = "spelunker", _("Spelunker Native")
     BOSSDB = "https://neuroglancer.bossdb.io", _("neuroglancer.bossdb.io")
-    # CLIO = "https://clio-ng.janelia.org", _("clio-ng.janelia.org")
-    SPELUNKER = "https://spelunker.cave-explorer.org/", _("spelunker.cave-explorer.org")
+    SPELUNKER_URL = "https://spelunker.cave-explorer.org/", _(
+        "spelunker.cave-explorer.org"
+    )
 
 
 class ForcedChoiceButtonGroup(models.Model):

neuvue_project/workspace/neuroglancer.py

@@ -25,7 +25,13 @@
     ChainedStateBuilder,
 )
 
-from .models import Namespace, NeuroglancerLinkType, PcgChoices, ImageChoices
+from .models import (
+    Namespace,
+    NeuroglancerLinkType,
+    PcgChoices,
+    ImageChoices,
+    NeuroglancerHost,
+)
 
 
 logging.basicConfig(level=logging.DEBUG)
@@ -273,7 +279,10 @@ def construct_proofreading_state(task_df, points, return_as="json"):
 
 
 def construct_url_from_existing(state: str, ng_host: str):
-    return ng_host + "/#!" + state
+    if ng_host in [NeuroglancerHost.SPELUNKER, NeuroglancerHost.SPELUNKER_URL]:
+        return ng_host + "/#!middleauth+" + state
+    else:
+        return ng_host + "/#!" + state
 
 
 @backoff.on_exception(backoff.expo, Exception, max_tries=3)
@@ -285,6 +294,7 @@ def get_from_state_server(url: str):
     Returns:
         (str): JSON String
     """
+    url = url.replace("middleauth+", "")
     headers = {
         "content-type": "application/json",
         "Authorization": f"Bearer {os.environ['CAVECLIENT_TOKEN']}",