diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..4f180117 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,11 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: end-of-file-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.10.0 + hooks: + - id: black + language_version: python3.9 diff --git a/README.md b/README.md index a9853f9a..71abed0d 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Clone this repository recursively so that the neuvue-client submodule will also git clone https://github.com/aplbrain/neuvue-app.git --recursive ``` -Create a python3 virtual environment and install the requirements in neuvue_project/requirements.txt. +Create a python3 virtual environment and install the requirements in neuvue_project/requirements.txt. ```shell python3 -m venv venv @@ -20,6 +20,22 @@ cd neuvue_project pip install -r requirements.txt ``` +## Development Installation + +Install developer python requirements and set up pre-commit enviroment. + +``` +source venv/bin/activate +pip install -r requirements-dev.txt +pre-commit install +``` + +Once changes are staged. Run pre-commit to automatically remove trailing whitespaces, check YAML files, and run black formatting on all python files. + +``` +pre-commit +``` + ## (Optional) Compiling the neuroglancer project A working neuroglancer compilation is included under `neuvue_project/workspace/static/workspace/`. However, if the underlying neuroglancer client needs to change,it must be compiled and linked. @@ -43,7 +59,7 @@ Requirements: [nvm](https://github.com/nvm-sh/nvm) npm i npm link neuroglancer npm run build - ``` + ``` 3. Copy the built files to static ``` cd neuvue_project/workspace/static/ts/wrapper @@ -52,7 +68,7 @@ Requirements: [nvm](https://github.com/nvm-sh/nvm) ## Running a development environment -There is an included `neuvueDB.sqlite3` database file containing the tables needed to run the Django app. By default, the settings are configured for production which uses a cloud-enabled MySQL database server. To enable development mode: +There is an included `neuvueDB.sqlite3` database file containing the tables needed to run the Django app. By default, the settings are configured for production which uses a cloud-enabled MySQL database server. To enable development mode: Run the following convenience script: ``` @@ -63,9 +79,9 @@ Or perform each step individually: 1. Open `neuvue_project/neuvue/settings.py` and set `DEBUG=True` -2. In the same file, modify `NEUVUE_QUEUE_ADDR` variable to the Nuevue-Queue endpoint you would like to use. +2. In the same file, modify `NEUVUE_QUEUE_ADDR` variable to the Nuevue-Queue endpoint you would like to use. -3. Get the recent migrations to the database by running +3. Get the recent migrations to the database by running `python manage.py migrate` @@ -77,11 +93,11 @@ Or perform each step individually: `python manage.py collectstatic --no-input` -6. Run the app with the `runserver` command to start a development instance. Run on the localhost:8000 address and port to allow OAuth client to properly authenticate user. +6. Run the app with the `runserver` command to start a development instance. Run on the localhost:8000 address and port to allow OAuth client to properly authenticate user. `python manage.py runserver localhost:8000` -7. Open your app on http://localhost:8000 +7. Open your app on http://localhost:8000 ## OAuth Set-up @@ -92,7 +108,7 @@ We use `django-allauth` to connect Google OAuth to the Django environment. Users http://localhost:8000/accounts/login/ -Django users, OAuth settings, and site configuration can be modified in the admin console. +Django users, OAuth settings, and site configuration can be modified in the admin console. http://localhost:8000/admin diff --git a/neuvue_project/dashboard/apps.py b/neuvue_project/dashboard/apps.py index 7b1cc053..db15b459 100644 --- a/neuvue_project/dashboard/apps.py +++ b/neuvue_project/dashboard/apps.py @@ -2,5 +2,5 @@ class DashboardConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'dashboard' + default_auto_field = "django.db.models.BigAutoField" + name = "dashboard" diff --git a/neuvue_project/dashboard/templatetags/inclusion_tags.py b/neuvue_project/dashboard/templatetags/inclusion_tags.py index f984d4a5..24496be7 100644 --- a/neuvue_project/dashboard/templatetags/inclusion_tags.py +++ b/neuvue_project/dashboard/templatetags/inclusion_tags.py @@ -2,7 +2,8 @@ register = template.Library() -@register.inclusion_tag('reusable_components/confirm_modal.html') + +@register.inclusion_tag("reusable_components/confirm_modal.html") def confirm_modal(column): return { "column": column.lower(), @@ -12,40 +13,43 @@ def confirm_modal(column): "button_id": column.lower() + "-button", } -@register.inclusion_tag('reusable_components/metrics_card.html') + +@register.inclusion_tag("reusable_components/metrics_card.html") def metrics_card(title, value, color): return { "title": title, "value": value, - "color": color # should be a bootstrap color utility, e.g. primary, secondary + "color": color, # should be a bootstrap color utility, e.g. primary, secondary } -@register.inclusion_tag('modals/flag_modal.html') + +@register.inclusion_tag("modals/flag_modal.html") def flag_modal(task_id): - return { - "task_id": task_id - } + return {"task_id": task_id} + -@register.inclusion_tag('modals/remove_task_modal.html') +@register.inclusion_tag("modals/remove_task_modal.html") def remove_task_modal(): return {} -@register.inclusion_tag('modals/timeout_modal.html') + +@register.inclusion_tag("modals/timeout_modal.html") def timeout_modal(): return {} -@register.inclusion_tag('modals/confirm_selected_segments_modal.html') + +@register.inclusion_tag("modals/confirm_selected_segments_modal.html") def confirm_selected_segments_modal(number_of_selected_segments_expected): return { "number_of_selected_segments_expected": number_of_selected_segments_expected } -@register.inclusion_tag('modals/distribute_task_modal.html') + +@register.inclusion_tag("modals/distribute_task_modal.html") def distribute_task_modal(): return {} -@register.inclusion_tag('reusable_components/instructions.html') + +@register.inclusion_tag("reusable_components/instructions.html") def processed_instructions(instructions): - return { - "instructions": instructions - } \ No newline at end of file + return {"instructions": instructions} diff --git a/neuvue_project/dashboard/views.py b/neuvue_project/dashboard/views.py index c5b9e020..5a1f4c09 100644 --- a/neuvue_project/dashboard/views.py +++ b/neuvue_project/dashboard/views.py @@ -9,105 +9,114 @@ from django.apps import apps from typing import List -from datetime import datetime +from datetime import datetime import plotly.graph_objects as go from neuvue.client import client # import the logging library import logging + logging.basicConfig(level=logging.DEBUG) # Get an instance of a logger logger = logging.getLogger(__name__) # Convenience functions -def _get_users_from_group(group:str): - if group == 'See All Users': # case if all users are queried +def _get_users_from_group(group: str): + if group == "See All Users": # case if all users are queried return [] - else: # case if group is provided - users = Group.objects.get(name=group).user_set.all() + else: # case if group is provided + users = Group.objects.get(name=group).user_set.all() return [x.username for x in users] + def _format_time(x): try: - return x.strftime('%Y-%m-%d %H:%M:%S') + return x.strftime("%Y-%m-%d %H:%M:%S") except: - return 'N/A' + return "N/A" + def _get_status_count(task_df, status): - return task_df['status'].value_counts().get(status, 0) + return task_df["status"].value_counts().get(status, 0) -class DashboardView(View, LoginRequiredMixin): +class DashboardView(View, LoginRequiredMixin): def get(self, request, *args, **kwargs): if not request.user.is_staff: - return redirect(reverse('index')) - - Namespaces = apps.get_model('workspace', 'Namespace') - + return redirect(reverse("index")) + + Namespaces = apps.get_model("workspace", "Namespace") + context = {} - context['all_groups'] = sorted([x.name for x in Group.objects.all()]) - context['all_groups'].append('See All Users') - context['all_namespaces'] = sorted([x.display_name for x in Namespaces.objects.all()]) - context['all_users'] = sorted([x.username for x in User.objects.all()]) + context["all_groups"] = sorted([x.name for x in Group.objects.all()]) + context["all_groups"].append("See All Users") + context["all_namespaces"] = sorted( + [x.display_name for x in Namespaces.objects.all()] + ) + context["all_users"] = sorted([x.username for x in User.objects.all()]) return render(request, "admin_dashboard/dashboard.html", context) def post(self, request, *args, **kwargs): - Namespaces = apps.get_model('workspace', 'Namespace') + Namespaces = apps.get_model("workspace", "Namespace") display_name = request.POST.get("namespace") group = request.POST.get("group") username = request.POST.get("username") if display_name and group: - namespace = Namespaces.objects.get(display_name = display_name).namespace - return redirect(reverse('dashboard', kwargs={"namespace":namespace, "group": group})) + namespace = Namespaces.objects.get(display_name=display_name).namespace + return redirect( + reverse("dashboard", kwargs={"namespace": namespace, "group": group}) + ) elif username: - return redirect(reverse('dashboard', kwargs={"username": username})) + return redirect(reverse("dashboard", kwargs={"username": username})) else: # as long as all html form fields contain required="true" this case should not be reached - return redirect(reverse('dashboard')) + return redirect(reverse("dashboard")) -class DashboardNamespaceView(View, LoginRequiredMixin): +class DashboardNamespaceView(View, LoginRequiredMixin): def get(self, request, group=None, namespace=None, *args, **kwargs): if not request.user.is_staff: - return redirect(reverse('index')) - - Namespaces = apps.get_model('workspace', 'Namespace') - + return redirect(reverse("index")) + + Namespaces = apps.get_model("workspace", "Namespace") + context = {} users = _get_users_from_group(group) table, counts = self._generate_table_and_counts(namespace, users) - - context['group'] = group - context['namespace'] = namespace - context['display_name'] = Namespaces.objects.get(namespace = namespace).display_name - context['table'] = table - context['total_closed'] = counts[0] - context['total_pending'] = counts[1] - context['total_open'] = counts[2] - context['total_errored'] = counts[3] + + context["group"] = group + context["namespace"] = namespace + context["display_name"] = Namespaces.objects.get( + namespace=namespace + ).display_name + context["table"] = table + context["total_closed"] = counts[0] + context["total_pending"] = counts[1] + context["total_open"] = counts[2] + context["total_errored"] = counts[3] n_users_per_group = {} - if group == 'unassigned': - context['user_groups'] = sorted([x.name for x in Group.objects.all()]) - for user_group in context['user_groups']: + if group == "unassigned": + context["user_groups"] = sorted([x.name for x in Group.objects.all()]) + for user_group in context["user_groups"]: n_users_per_group[user_group] = len(_get_users_from_group(user_group)) - context['user_group_counts'] = n_users_per_group + context["user_group_counts"] = n_users_per_group return render(request, "admin_dashboard/dashboard-namespace-view.html", context) - + def post(self, request, *args, **kwargs): unassigned_group = request.POST.get("unassigned-group") - namespace = request.POST.get("namespace") ## GET THIS + namespace = request.POST.get("namespace") ## GET THIS namespace_df = client.get_tasks( sieve={ - 'assignee': unassigned_group, - 'namespace': namespace, + "assignee": unassigned_group, + "namespace": namespace, }, - select=['_id'] + select=["_id"], ) assignee_group = request.POST.get("assignee-group") @@ -116,66 +125,66 @@ def post(self, request, *args, **kwargs): for i in range(len(assignee_group_usernames)): user = assignee_group_usernames[i] - user_tasks = namespace_df.iloc[(i*n_tasks):(n_tasks+(i*n_tasks))] + user_tasks = namespace_df.iloc[(i * n_tasks) : (n_tasks + (i * n_tasks))] for task in user_tasks.index: - client.patch_task(task, assignee=user) - - return redirect(reverse('dashboard', kwargs={"namespace":namespace,"group": 'unassigned'})) + client.patch_task(task, assignee=user) + + return redirect( + reverse("dashboard", kwargs={"namespace": namespace, "group": "unassigned"}) + ) def _generate_table_and_counts(self, namespace: str, users: List): table_rows = [] # Counts tc = tp = to = te = 0 - sieve = {'namespace':namespace} - if users: # case if `group` of users was selected - sieve['assignee'] = users - else: # case if `All Users` was selected - users = list(User.objects.all().values_list('username', flat=True).distinct()) # retrieve all usernames - - task_df = client.get_tasks( - sieve=sieve, - select=['assignee', 'status', 'closed'] - ) - + sieve = {"namespace": namespace} + if users: # case if `group` of users was selected + sieve["assignee"] = users + else: # case if `All Users` was selected + users = list( + User.objects.all().values_list("username", flat=True).distinct() + ) # retrieve all usernames + + task_df = client.get_tasks(sieve=sieve, select=["assignee", "status", "closed"]) + for user in users: - user_df = task_df[task_df['assignee'] == user] - last_closed = _format_time(user_df['closed'].max()) - # Append row info + user_df = task_df[task_df["assignee"] == user] + last_closed = _format_time(user_df["closed"].max()) + # Append row info row = { - 'username': user, - 'pending': _get_status_count(user_df, 'pending'), - 'open': _get_status_count(user_df, 'open'), - 'closed': _get_status_count(user_df, 'closed'), - 'errored': _get_status_count(user_df, 'errored'), - 'last_closed': last_closed + "username": user, + "pending": _get_status_count(user_df, "pending"), + "open": _get_status_count(user_df, "open"), + "closed": _get_status_count(user_df, "closed"), + "errored": _get_status_count(user_df, "errored"), + "last_closed": last_closed, } table_rows.append(row) - tc += int(row['closed']) - tp += int(row['pending']) - to += int(row['open']) - te += int(row['errored']) + tc += int(row["closed"]) + tp += int(row["pending"]) + to += int(row["open"]) + te += int(row["errored"]) return table_rows, (tc, tp, to, te) class DashboardUserView(View, LoginRequiredMixin): - def get(self, request, username=None, filter=None, *args, **kwargs): if not request.user.is_staff: - return redirect(reverse('index')) + return redirect(reverse("index")) context = {} table, counts = self._generate_table_and_counts(username) - - context['username'] = username - context['table'] = table - context['total_closed'] = counts[0] - context['total_pending'] = counts[1] - context['total_open'] = counts[2] - context['total_errored'] = counts[3] - context['filter'] = filter + + context["username"] = username + context["table"] = table + context["total_closed"] = counts[0] + context["total_pending"] = counts[1] + context["total_open"] = counts[2] + context["total_errored"] = counts[3] + context["filter"] = filter return render(request, "admin_dashboard/dashboard-user-view.html", context) @@ -185,23 +194,33 @@ def _generate_table_and_counts(self, user: str): tc = tp = to = te = 0 user_df = client.get_tasks( sieve={ - 'assignee': user, + "assignee": user, }, - select= ['opened', 'closed', 'created', 'duration', 'status', 'namespace', 'priority', 'seg_id', 'tags'] + select=[ + "opened", + "closed", + "created", + "duration", + "status", + "namespace", + "priority", + "seg_id", + "tags", + ], ) - user_df= user_df.sort_values('created', ascending=False) - user_df['task_id'] = user_df.index - user_df['opened'] = user_df['opened'].apply(_format_time) - user_df['closed'] = user_df['closed'].apply(_format_time) - user_df['created'] = user_df['created'].apply(_format_time) - user_df['duration'] = (user_df['duration']/60).round(1) + user_df = user_df.sort_values("created", ascending=False) + user_df["task_id"] = user_df.index + user_df["opened"] = user_df["opened"].apply(_format_time) + user_df["closed"] = user_df["closed"].apply(_format_time) + user_df["created"] = user_df["created"].apply(_format_time) + user_df["duration"] = (user_df["duration"] / 60).round(1) - table_rows = user_df.to_dict('records') - tc = int(_get_status_count(user_df, 'closed')) - tp += int(_get_status_count(user_df, 'pending')) - to += int(_get_status_count(user_df, 'open')) - te += int(_get_status_count(user_df, 'errored')) + table_rows = user_df.to_dict("records") + tc = int(_get_status_count(user_df, "closed")) + tp += int(_get_status_count(user_df, "pending")) + to += int(_get_status_count(user_df, "open")) + te += int(_get_status_count(user_df, "errored")) return table_rows, (tc, tp, to, te) @@ -215,18 +234,18 @@ def post(self, request, *args, **kwargs): selected_tasks = request.POST.getlist("selected_tasks") new_assignee = request.POST.get("assignee-input") new_status = request.POST.get("status-input") - + try: new_priority = int(request.POST.get("priority-input")) except: new_priority = 0 for task in selected_tasks: - if selected_action == 'delete': + if selected_action == "delete": logging.debug(f"Delete task: {task}") client.delete_task(task) elif selected_action == "assignee": - client.patch_task(task,assignee=new_assignee) + client.patch_task(task, assignee=new_assignee) logging.debug(f"Resassigning task {task} to {new_assignee}") elif selected_action == "priority": client.patch_task(task, priority=new_priority) @@ -236,99 +255,123 @@ def post(self, request, *args, **kwargs): logging.debug(f"Updating task {task} to {new_status}") # Redirect to dashboard page from splashpage or modal - return redirect(reverse('dashboard', kwargs={"username": username})) + return redirect(reverse("dashboard", kwargs={"username": username})) -class ReportView(View, LoginRequiredMixin): +class ReportView(View, LoginRequiredMixin): def get(self, request, *args, **kwargs): if not request.user.is_staff: - return redirect(reverse('index')) - - Namespaces = apps.get_model('workspace', 'Namespace') + return redirect(reverse("index")) + + Namespaces = apps.get_model("workspace", "Namespace") context = {} - context['all_groups'] = sorted([x.name for x in Group.objects.all()]) - context['all_namespaces'] = sorted([x.display_name for x in Namespaces.objects.all()]) - + context["all_groups"] = sorted([x.name for x in Group.objects.all()]) + context["all_namespaces"] = sorted( + [x.display_name for x in Namespaces.objects.all()] + ) + try: - context['all_groups'].remove('AuthorizedUsers') + context["all_groups"].remove("AuthorizedUsers") except: - pass + pass return render(request, "report.html", context) def post(self, request, *args, **kwargs): - - Namespaces = apps.get_model('workspace', 'Namespace') - Buttons = apps.get_model('workspace', 'ForcedChoiceButton') - + + Namespaces = apps.get_model("workspace", "Namespace") + Buttons = apps.get_model("workspace", "ForcedChoiceButton") + # Access POST fields - display_name = request.POST.get('namespace') - group = request.POST.get('group') - start_field = request.POST.get('start_field') - start_date = request.POST.get('start_date') - end_field = request.POST.get('end_field') - end_date = request.POST.get('end_date') - + display_name = request.POST.get("namespace") + group = request.POST.get("group") + start_field = request.POST.get("start_field") + start_date = request.POST.get("start_date") + end_field = request.POST.get("end_field") + end_date = request.POST.get("end_date") + # Convert to datetime Objects - start_dt = datetime.strptime(start_date, '%Y-%m-%d') - end_dt = datetime.strptime(end_date, '%Y-%m-%d') + start_dt = datetime.strptime(start_date, "%Y-%m-%d") + end_dt = datetime.strptime(end_date, "%Y-%m-%d") # Retrieve valid tasks - namespace = Namespaces.objects.get(display_name = display_name).namespace + namespace = Namespaces.objects.get(display_name=display_name).namespace users = _get_users_from_group(group) button_sets = set() - for o in Buttons.objects.all(): button_sets.add(str(o.set_name)) + for o in Buttons.objects.all(): + button_sets.add(str(o.set_name)) # add bar chart - decision_namespaces = [x.namespace for x in Namespaces.objects.filter(submission_method__in=list(button_sets)).all()] + decision_namespaces = [ + x.namespace + for x in Namespaces.objects.filter( + submission_method__in=list(button_sets) + ).all() + ] sieve = { - 'assignee': users, - 'namespace': namespace, + "assignee": users, + "namespace": namespace, } if start_field == end_field: - sieve[start_field] = { - '$gt': start_dt, - '$lt': end_dt - } + sieve[start_field] = {"$gt": start_dt, "$lt": end_dt} else: - sieve[start_field] = { - '$gt': start_dt - } - sieve[end_field] = { - '$lt': end_dt - } - + sieve[start_field] = {"$gt": start_dt} + sieve[end_field] = {"$lt": end_dt} + task_df = client.get_tasks( - sieve=sieve, - select=['assignee', 'status', 'duration','metadata','closed','opened'] - ) - + sieve=sieve, + select=["assignee", "status", "duration", "metadata", "closed", "opened"], + ) + if namespace in decision_namespaces: import plotly.express as px from plotly.subplots import make_subplots - task_df['decision'] = task_df['metadata'].apply(lambda x: x.get('decision')) - - - users=task_df['assignee'].unique() - fig_decision = make_subplots(rows=1, cols=2, column_widths=[0.12, 0.85], shared_yaxes=True,horizontal_spacing = 0.02) + + task_df["decision"] = task_df["metadata"].apply(lambda x: x.get("decision")) + + users = task_df["assignee"].unique() + fig_decision = make_subplots( + rows=1, + cols=2, + column_widths=[0.12, 0.85], + shared_yaxes=True, + horizontal_spacing=0.02, + ) color_count = 0 - for decision_type in task_df['decision'].unique(): + for decision_type in task_df["decision"].unique(): if decision_type: - decision_counts = dict(task_df[task_df['decision']==decision_type].value_counts('assignee')) + decision_counts = dict( + task_df[task_df["decision"] == decision_type].value_counts( + "assignee" + ) + ) x = list(decision_counts.keys()) y = list(decision_counts.values()) fig_decision.add_trace( - go.Bar(name=decision_type, x=x, y=y,marker_color=px.colors.qualitative.Plotly[color_count]), - row=1, col=2 + go.Bar( + name=decision_type, + x=x, + y=y, + marker_color=px.colors.qualitative.Plotly[color_count], + ), + row=1, + col=2, ) fig_decision.add_trace( - go.Bar(name=decision_type, x=['total'], y=[sum(y)],marker_color=px.colors.qualitative.Plotly[color_count],showlegend=False), - row=1, col=1 + go.Bar( + name=decision_type, + x=["total"], + y=[sum(y)], + marker_color=px.colors.qualitative.Plotly[color_count], + showlegend=False, + ), + row=1, + col=1, ) - color_count +=1 + color_count += 1 fig_decision.update_layout( title="Decisions for " + namespace + " by " + group, yaxis_title="# of responses", @@ -336,173 +379,230 @@ def post(self, request, *args, **kwargs): ) fig_decision.update_xaxes(title_text="assignees", row=1, col=2) - - columns = ['Username', 'Total Duration (h)', 'Avg Closed Duration (m)' , 'Avg Duration (m)'] - status_states = ['pending','open','closed','errored'] + columns = [ + "Username", + "Total Duration (h)", + "Avg Closed Duration (m)", + "Avg Duration (m)", + ] + status_states = ["pending", "open", "closed", "errored"] columns.extend(status_states) table_rows = [] fig_time = go.Figure() - for assignee, assignee_df in task_df.groupby('assignee'): - total_duration = str(round(assignee_df['duration'].sum()/3600,2)) - avg_closed_duration = str(round(assignee_df[assignee_df['status']=='closed']['duration'].mean()/60,2)) - avg_duration = str(round(assignee_df['duration'].mean()/60,2)) + for assignee, assignee_df in task_df.groupby("assignee"): + total_duration = str(round(assignee_df["duration"].sum() / 3600, 2)) + avg_closed_duration = str( + round( + assignee_df[assignee_df["status"] == "closed"]["duration"].mean() + / 60, + 2, + ) + ) + avg_duration = str(round(assignee_df["duration"].mean() / 60, 2)) user_metrics = [assignee, total_duration, avg_closed_duration, avg_duration] for status in status_states: - number_of_tasks = len(assignee_df[assignee_df['status']==status]) + number_of_tasks = len(assignee_df[assignee_df["status"] == status]) user_metrics.append(number_of_tasks) table_rows.append(user_metrics) - assignee_df['last_interaction'] = assignee_df.apply(lambda x: max(x.opened, x.closed).floor('d'), axis=1) - daily_totals = assignee_df.groupby('last_interaction')['duration'].sum()/3600 + assignee_df["last_interaction"] = assignee_df.apply( + lambda x: max(x.opened, x.closed).floor("d"), axis=1 + ) + daily_totals = ( + assignee_df.groupby("last_interaction")["duration"].sum() / 3600 + ) x = daily_totals.index y = daily_totals.values - fig_time.add_trace(go.Scatter(x=x,y=y,name=assignee)) - fig_time.update_layout(showlegend=True, - title="Platform Time per User Grouped by Date of Last Interaction", - yaxis_title="Duration (h)", - xaxis_title="Date of Last Interaction") - - fields = {"created":"Created By", - "opened": "Opened By", - "closed": "Closed By"} - - context = {"display_name":display_name, - "namespace":namespace, - "group":group, - "start_field":start_field, - "start_date":start_date, - "end_field":end_field, - "end_date":end_date, - "table_columns":columns, - "table_rows":table_rows, - "fields":fields, - "all_groups": [x.name for x in Group.objects.all()], - "all_namespaces": [x.display_name for x in Namespaces.objects.all()], - "fig_time":fig_time.to_html(), - } + fig_time.add_trace(go.Scatter(x=x, y=y, name=assignee)) + fig_time.update_layout( + showlegend=True, + title="Platform Time per User Grouped by Date of Last Interaction", + yaxis_title="Duration (h)", + xaxis_title="Date of Last Interaction", + ) + + fields = {"created": "Created By", "opened": "Opened By", "closed": "Closed By"} + + context = { + "display_name": display_name, + "namespace": namespace, + "group": group, + "start_field": start_field, + "start_date": start_date, + "end_field": end_field, + "end_date": end_date, + "table_columns": columns, + "table_rows": table_rows, + "fields": fields, + "all_groups": [x.name for x in Group.objects.all()], + "all_namespaces": [x.display_name for x in Namespaces.objects.all()], + "fig_time": fig_time.to_html(), + } if namespace in decision_namespaces: context["fig_decision"] = fig_decision.to_html() - return render(request,'report.html',context) + return render(request, "report.html", context) -class UserNamespaceView(View, LoginRequiredMixin): +class UserNamespaceView(View, LoginRequiredMixin): def get(self, request, *args, **kwargs): if not request.user.is_staff: - return redirect(reverse('index')) - - Namespaces = apps.get_model('workspace', 'Namespace') + return redirect(reverse("index")) + + Namespaces = apps.get_model("workspace", "Namespace") context = {} - context['all_users'] = sorted([x.username for x in User.objects.all()]) - context['all_namespaces'] = sorted([x.display_name for x in Namespaces.objects.all()]) - + context["all_users"] = sorted([x.username for x in User.objects.all()]) + context["all_namespaces"] = sorted( + [x.display_name for x in Namespaces.objects.all()] + ) + try: - context['all_groups'].remove('AuthorizedUsers') + context["all_groups"].remove("AuthorizedUsers") except: - pass + pass - fields = {"created":"Created By", - "opened": "Opened By", - "closed": "Closed By"} + fields = {"created": "Created By", "opened": "Opened By", "closed": "Closed By"} - context['fields'] = fields + context["fields"] = fields return render(request, "user-namespace.html", context) def post(self, request, *args, **kwargs): - - Namespaces = apps.get_model('workspace', 'Namespace') - + + Namespaces = apps.get_model("workspace", "Namespace") + # Access POST fields - display_name = request.POST.get('namespace') - username = request.POST.get('user') - start_field = request.POST.get('start_field') - start_date = request.POST.get('start_date') - end_field = request.POST.get('end_field') - end_date = request.POST.get('end_date') - + display_name = request.POST.get("namespace") + username = request.POST.get("user") + start_field = request.POST.get("start_field") + start_date = request.POST.get("start_date") + end_field = request.POST.get("end_field") + end_date = request.POST.get("end_date") + # Convert to datetime Objects - start_dt = datetime.strptime(start_date, '%Y-%m-%d') - end_dt = datetime.strptime(end_date, '%Y-%m-%d') + start_dt = datetime.strptime(start_date, "%Y-%m-%d") + end_dt = datetime.strptime(end_date, "%Y-%m-%d") # Retrieve valid tasks - namespace = Namespaces.objects.get(display_name = display_name).namespace if display_name else '' + namespace = ( + Namespaces.objects.get(display_name=display_name).namespace + if display_name + else "" + ) if display_name: sieve = { - 'namespace': namespace, + "namespace": namespace, } else: sieve = { - 'assignee': username, + "assignee": username, } if start_field == end_field: - sieve[start_field] = { - '$gt': start_dt, - '$lt': end_dt - } + sieve[start_field] = {"$gt": start_dt, "$lt": end_dt} else: - sieve[start_field] = { - '$gt': start_dt - } - sieve[end_field] = { - '$lt': end_dt - } - + sieve[start_field] = {"$gt": start_dt} + sieve[end_field] = {"$lt": end_dt} + task_df = client.get_tasks( - sieve=sieve, - select=['assignee', 'status', 'duration','metadata','closed','opened','namespace'] - ) + sieve=sieve, + select=[ + "assignee", + "status", + "duration", + "metadata", + "closed", + "opened", + "namespace", + ], + ) - if display_name: - columns = ['Username', 'Total Duration (h)', 'Avg Closed Duration (m)' , 'Avg Duration (m)'] - status_states = ['pending','open','closed','errored'] + if display_name: + columns = [ + "Username", + "Total Duration (h)", + "Avg Closed Duration (m)", + "Avg Duration (m)", + ] + status_states = ["pending", "open", "closed", "errored"] columns.extend(status_states) table_rows = [] - for assignee, assignee_df in task_df.groupby('assignee'): - total_duration = str(round(assignee_df['duration'].sum()/3600,2)) - avg_closed_duration = str(round(assignee_df[assignee_df['status']=='closed']['duration'].mean()/60,2)) - avg_duration = str(round(assignee_df['duration'].mean()/60,2)) - user_metrics = [assignee, total_duration, avg_closed_duration, avg_duration] + for assignee, assignee_df in task_df.groupby("assignee"): + total_duration = str(round(assignee_df["duration"].sum() / 3600, 2)) + avg_closed_duration = str( + round( + assignee_df[assignee_df["status"] == "closed"][ + "duration" + ].mean() + / 60, + 2, + ) + ) + avg_duration = str(round(assignee_df["duration"].mean() / 60, 2)) + user_metrics = [ + assignee, + total_duration, + avg_closed_duration, + avg_duration, + ] for status in status_states: - number_of_tasks = len(assignee_df[assignee_df['status']==status]) + number_of_tasks = len(assignee_df[assignee_df["status"] == status]) user_metrics.append(number_of_tasks) table_rows.append(user_metrics) else: - columns = ['Namespace', 'Total Duration (h)', 'Avg Closed Duration (m)' , 'Avg Duration (m)'] - status_states = ['pending','open','closed','errored'] + columns = [ + "Namespace", + "Total Duration (h)", + "Avg Closed Duration (m)", + "Avg Duration (m)", + ] + status_states = ["pending", "open", "closed", "errored"] columns.extend(status_states) table_rows = [] - for namespace, namespace_df in task_df.groupby('namespace'): - total_duration = str(round(namespace_df['duration'].sum()/3600,2)) - avg_closed_duration = str(round(namespace_df[namespace_df['status']=='closed']['duration'].mean()/60,2)) - avg_duration = str(round(namespace_df['duration'].mean()/60,2)) - namespace_metrics = [namespace, total_duration, avg_closed_duration, avg_duration] + for namespace, namespace_df in task_df.groupby("namespace"): + total_duration = str(round(namespace_df["duration"].sum() / 3600, 2)) + avg_closed_duration = str( + round( + namespace_df[namespace_df["status"] == "closed"][ + "duration" + ].mean() + / 60, + 2, + ) + ) + avg_duration = str(round(namespace_df["duration"].mean() / 60, 2)) + namespace_metrics = [ + namespace, + total_duration, + avg_closed_duration, + avg_duration, + ] for status in status_states: - number_of_tasks = len(namespace_df[namespace_df['status']==status]) + number_of_tasks = len( + namespace_df[namespace_df["status"] == status] + ) namespace_metrics.append(number_of_tasks) table_rows.append(namespace_metrics) - fields = {"created":"Created By", - "opened": "Opened By", - "closed": "Closed By"} + fields = {"created": "Created By", "opened": "Opened By", "closed": "Closed By"} filename_field = display_name if display_name else username - context = {"display_name":display_name, - "namespace":namespace, - "start_field":start_field, - "start_date":start_date, - "end_field":end_field, - "end_date":end_date, - 'username':username, - "table_columns":columns, - "table_rows":table_rows, - "fields":fields, - 'all_users': sorted([x.username for x in User.objects.all()]), - "all_namespaces": [x.display_name for x in Namespaces.objects.all()], - 'filename_field': filename_field - } - - return render(request,'user-namespace.html',context) + context = { + "display_name": display_name, + "namespace": namespace, + "start_field": start_field, + "start_date": start_date, + "end_field": end_field, + "end_date": end_date, + "username": username, + "table_columns": columns, + "table_rows": table_rows, + "fields": fields, + "all_users": sorted([x.username for x in User.objects.all()]), + "all_namespaces": [x.display_name for x in Namespaces.objects.all()], + "filename_field": filename_field, + } + + return render(request, "user-namespace.html", context) diff --git a/neuvue_project/manage.py b/neuvue_project/manage.py index 89b6ec03..aae262b2 100755 --- a/neuvue_project/manage.py +++ b/neuvue_project/manage.py @@ -6,7 +6,7 @@ def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'neuvue.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "neuvue.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -18,5 +18,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/neuvue_project/neuvue/__init__.py b/neuvue_project/neuvue/__init__.py index 4c51a321..421945bc 100644 --- a/neuvue_project/neuvue/__init__.py +++ b/neuvue_project/neuvue/__init__.py @@ -1 +1 @@ -from .client import * \ No newline at end of file +from .client import * diff --git a/neuvue_project/neuvue/asgi.py b/neuvue_project/neuvue/asgi.py index cf4060ba..c654032a 100644 --- a/neuvue_project/neuvue/asgi.py +++ b/neuvue_project/neuvue/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'neuvue.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "neuvue.settings") application = get_asgi_application() diff --git a/neuvue_project/neuvue/client.py b/neuvue_project/neuvue/client.py index a5e86a6b..f693b3fe 100644 --- a/neuvue_project/neuvue/client.py +++ b/neuvue_project/neuvue/client.py @@ -1,16 +1,18 @@ from django.conf import settings from neuvueclient import NeuvueQueue -class NeuvueClient(): - # Class variable means that client is accessible without instantiating class at all - client = NeuvueQueue(settings.NEUVUE_QUEUE_ADDR, **settings.NEUVUE_CLIENT_SETTINGS) +class NeuvueClient: + + # Class variable means that client is accessible without instantiating class at all + client = NeuvueQueue(settings.NEUVUE_QUEUE_ADDR, **settings.NEUVUE_CLIENT_SETTINGS) + + # Make sure class is only instantiated once if at all + def __new__(cls): + if not hasattr(cls, "instance"): + cls.instance = super(NeuvueClient, cls).__new__(cls) + return cls.instance - # Make sure class is only instantiated once if at all - def __new__(cls): - if not hasattr(cls, 'instance'): - cls.instance = super(NeuvueClient, cls).__new__(cls) - return cls.instance # Export class variable from file as well to make code pretty -client = NeuvueClient.client \ No newline at end of file +client = NeuvueClient.client diff --git a/neuvue_project/neuvue/settings.py b/neuvue_project/neuvue/settings.py index a3eefc6c..b122e0e8 100644 --- a/neuvue_project/neuvue/settings.py +++ b/neuvue_project/neuvue/settings.py @@ -19,29 +19,34 @@ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ['DJANGO_SECRET_KEY'] +SECRET_KEY = os.environ["DJANGO_SECRET_KEY"] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = False ALLOWED_HOSTS = [ - 'app.neuvue.io', - 'neuvueapp-env.eba-ph8myjrq.us-east-1.elasticbeanstalk.com', - 'localhost', - '127.0.0.1' + "app.neuvue.io", + "neuvueapp-env.eba-ph8myjrq.us-east-1.elasticbeanstalk.com", + "localhost", + "127.0.0.1", ] -INTERNAL_IPS = [ - 'localhost', - '127.0.0.1' -] +INTERNAL_IPS = ["localhost", "127.0.0.1"] if DEBUG is False: - # Fix Health Check issues + # Fix Health Check issues import requests + try: - token = requests.put("http://169.254.169.254/latest/api/token", headers={"X-aws-ec2-metadata-token-ttl-seconds":"21600"}).text - internal_ip = requests.get('http://169.254.169.254/latest/meta-data/local-ipv4', timeout=3, headers={"X-aws-ec2-metadata-token": token}).text + token = requests.put( + "http://169.254.169.254/latest/api/token", + headers={"X-aws-ec2-metadata-token-ttl-seconds": "21600"}, + ).text + internal_ip = requests.get( + "http://169.254.169.254/latest/meta-data/local-ipv4", + timeout=3, + headers={"X-aws-ec2-metadata-token": token}, + ).text except requests.exceptions.ConnectionError: pass else: @@ -51,80 +56,80 @@ # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.sites', - 'allauth', - 'allauth.account', - 'allauth.socialaccount', - 'allauth.socialaccount.providers.google', - 'workspace', - 'dashboard', - 'webpack_loader', - 'preferences', - 'debug_toolbar', - 'colorfield' + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.sites", + "allauth", + "allauth.account", + "allauth.socialaccount", + "allauth.socialaccount.providers.google", + "workspace", + "dashboard", + "webpack_loader", + "preferences", + "debug_toolbar", + "colorfield", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'debug_toolbar.middleware.DebugToolbarMiddleware' + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "debug_toolbar.middleware.DebugToolbarMiddleware", ] -ROOT_URLCONF = 'neuvue.urls' +ROOT_URLCONF = "neuvue.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [os.path.join(BASE_DIR, "templates")], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates")], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -TEMPLATE_DIR = os.path.join(BASE_DIR, "templates") +TEMPLATE_DIR = os.path.join(BASE_DIR, "templates") -WSGI_APPLICATION = 'neuvue.wsgi.application' +WSGI_APPLICATION = "neuvue.wsgi.application" # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases -if 'RDS_HOSTNAME' in os.environ and DEBUG is False: +if "RDS_HOSTNAME" in os.environ and DEBUG is False: DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'NAME': os.environ['RDS_DB_NAME'], - 'USER': os.environ['RDS_USERNAME'], - 'PASSWORD': os.environ['RDS_PASSWORD'], - 'HOST': os.environ['RDS_HOSTNAME'], - 'PORT': os.environ['RDS_PORT'], + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": os.environ["RDS_DB_NAME"], + "USER": os.environ["RDS_USERNAME"], + "PASSWORD": os.environ["RDS_PASSWORD"], + "HOST": os.environ["RDS_HOSTNAME"], + "PORT": os.environ["RDS_PORT"], } } else: DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'neuvueDB.sqlite3', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "neuvueDB.sqlite3", } } @@ -133,16 +138,16 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -150,9 +155,9 @@ # Internationalization # https://docs.djangoproject.com/en/3.2/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'EST' +TIME_ZONE = "EST" USE_I18N = True @@ -164,13 +169,11 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" -STATIC_ROOT = os.path.join(BASE_DIR, 'static') +STATIC_ROOT = os.path.join(BASE_DIR, "static") -STATICFILES_DIRS = [ - os.path.join(BASE_DIR, 'workspace/static') -] +STATICFILES_DIRS = [os.path.join(BASE_DIR, "workspace/static")] DATA_UPLOAD_MAX_MEMORY_SIZE = 10485760 @@ -178,39 +181,39 @@ # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" AUTHENTICATION_BACKENDS = [ - 'django.contrib.auth.backends.ModelBackend', - 'allauth.account.auth_backends.AuthenticationBackend' + "django.contrib.auth.backends.ModelBackend", + "allauth.account.auth_backends.AuthenticationBackend", ] SOCIALACCOUNT_PROVIDERS = { - 'google': { - 'SCOPE': [ - 'profile', - 'email', + "google": { + "SCOPE": [ + "profile", + "email", ], - 'AUTH_PARAMS': { - 'access_type': 'online', - } + "AUTH_PARAMS": { + "access_type": "online", + }, } } -LOGIN_URL = 'index' -LOGIN_REDIRECT_URL = '/tasks' -LOGOUT_REDIRECT_URL = '/' +LOGIN_URL = "index" +LOGIN_REDIRECT_URL = "/tasks" +LOGOUT_REDIRECT_URL = "/" # Neuvue Specific Settings NEUVUE_QUEUE_ADDR = "https://queue.neuvue.io/" -SANDBOX_ID = '6269888a101fc4da81fdd410' +SANDBOX_ID = "6269888a101fc4da81fdd410" NEUVUE_CLIENT_SETTINGS = { # "local" : True } # Annotation Tables -NEURON_TABLE = 'nucleus_neuron_svm' -CELL_CLASS_TABLE = 'allen_soma_coarse_cell_class_model_v2' +NEURON_TABLE = "nucleus_neuron_svm" +CELL_CLASS_TABLE = "allen_soma_coarse_cell_class_model_v2" DAYS_UNTIL_EXPIRED = 3 CACHED_TABLES_PATH = os.path.join(STATIC_ROOT, "tables") @@ -223,16 +226,14 @@ JSON_STATE_SERVER = "https://global.daf-apis.com/nglstate/post" DATASET_VIEWER_OPTIONS = { "https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em": { - "contrast": { - "black": 0.35, - "white": 0.7 - } + "contrast": {"black": 0.35, "white": 0.7} } } VOXEL_RESOLUTION = (4, 4, 40) if DEBUG: import mimetypes + mimetypes.add_type("application/javascript", ".js", True) -STATIC_NG_FILES = os.listdir(os.path.join(BASE_DIR, 'workspace', 'static', 'workspace')) +STATIC_NG_FILES = os.listdir(os.path.join(BASE_DIR, "workspace", "static", "workspace")) diff --git a/neuvue_project/neuvue/urls.py b/neuvue_project/neuvue/urls.py index e97b76ec..de107ea2 100644 --- a/neuvue_project/neuvue/urls.py +++ b/neuvue_project/neuvue/urls.py @@ -30,48 +30,61 @@ NucleiView, GettingStartedView, SaveStateView, - SaveOperationsView - ) + SaveOperationsView, +) from preferences.views import PreferencesView from dashboard.views import ( - DashboardView, - DashboardNamespaceView, - DashboardUserView, - ReportView, - UserNamespaceView + DashboardView, + DashboardNamespaceView, + DashboardUserView, + ReportView, + UserNamespaceView, ) urlpatterns = [ - path('', IndexView.as_view(), name="index"), - path('preferences/', PreferencesView.as_view(), name="preferences"), - path('tasks/', TaskView.as_view(), name="tasks"), - path('getting-started/', GettingStartedView.as_view(), name="getting-started"), - path('workspace/', WorkspaceView.as_view(), name="workspace"), - path('admin/', admin.site.urls), - path('__debug__/', include('debug_toolbar.urls')), - path('accounts/', include('allauth.urls')), - path('logout/', LogoutView.as_view(), name="logout"), - path('dashboard/', DashboardView.as_view(), name="dashboard"), - path('dashboard/namespace//group/', DashboardNamespaceView.as_view(), name="dashboard"), - path('dashboard/username/', DashboardUserView.as_view(), name="dashboard"), - path('dashboard/username//', DashboardUserView.as_view(), name="dashboard"), - path('auth_redirect.html',AuthView.as_view(),name='auth_redirect'), - path('token/', TokenView.as_view(), name='token'), - path('inspect/', InspectTaskView.as_view(), name="inspect"), - path('inspect/', InspectTaskView.as_view(), name="inspect"), - path('lineage/', LineageView.as_view(), name="lineage"), - path('lineage/', LineageView.as_view(), name="lineage"), - path('synapse/', SynapseView.as_view(), name="synapse"), - path('synapse/', SynapseView.as_view(), name="synapse"), - path('synapse/////', SynapseView.as_view(), name="synapse"), - path('nuclei/', NucleiView.as_view(), name="nuclei"), - path('nuclei/', NucleiView.as_view(), name="nuclei"), - path('report/', ReportView.as_view(), name="report"), - path('userNamespace/', UserNamespaceView.as_view(), name="user-namespace"), - path('save_state', SaveStateView.as_view(), name="save-state"), - path('save_operations', SaveOperationsView.as_view(), name="save-operations"), - path('about', AboutView.as_view(), name="about"), + path("", IndexView.as_view(), name="index"), + path("preferences/", PreferencesView.as_view(), name="preferences"), + path("tasks/", TaskView.as_view(), name="tasks"), + path("getting-started/", GettingStartedView.as_view(), name="getting-started"), + path("workspace/", WorkspaceView.as_view(), name="workspace"), + path("admin/", admin.site.urls), + path("__debug__/", include("debug_toolbar.urls")), + path("accounts/", include("allauth.urls")), + path("logout/", LogoutView.as_view(), name="logout"), + path("dashboard/", DashboardView.as_view(), name="dashboard"), + path( + "dashboard/namespace//group/", + DashboardNamespaceView.as_view(), + name="dashboard", + ), + path( + "dashboard/username/", + DashboardUserView.as_view(), + name="dashboard", + ), + path( + "dashboard/username//", + DashboardUserView.as_view(), + name="dashboard", + ), + path("auth_redirect.html", AuthView.as_view(), name="auth_redirect"), + path("token/", TokenView.as_view(), name="token"), + path("inspect/", InspectTaskView.as_view(), name="inspect"), + path("inspect/", InspectTaskView.as_view(), name="inspect"), + path("lineage/", LineageView.as_view(), name="lineage"), + path("lineage/", LineageView.as_view(), name="lineage"), + path("synapse/", SynapseView.as_view(), name="synapse"), + path("synapse/", SynapseView.as_view(), name="synapse"), + path( + "synapse/////", + SynapseView.as_view(), + name="synapse", + ), + path("nuclei/", NucleiView.as_view(), name="nuclei"), + path("nuclei/", NucleiView.as_view(), name="nuclei"), + path("report/", ReportView.as_view(), name="report"), + path("userNamespace/", UserNamespaceView.as_view(), name="user-namespace"), + path("save_state", SaveStateView.as_view(), name="save-state"), + path("save_operations", SaveOperationsView.as_view(), name="save-operations"), + path("about", AboutView.as_view(), name="about"), ] - - - diff --git a/neuvue_project/neuvue/webpack.py b/neuvue_project/neuvue/webpack.py index 05e7789b..53142502 100644 --- a/neuvue_project/neuvue/webpack.py +++ b/neuvue_project/neuvue/webpack.py @@ -1,16 +1,23 @@ from webpack_loader.loader import WebpackLoader import json + class MultipleWebpackLoader(WebpackLoader): def load_assets(self): - complete_stats = {'status':None,'assets':{}, 'chunks':{}} + complete_stats = {"status": None, "assets": {}, "chunks": {}} - for fn in self.config['STATS_FILES']: + for fn in self.config["STATS_FILES"]: with open(fn) as fp: stats = json.load(fp) - if stats['status'] != "done": + if stats["status"] != "done": raise ValueError(f"Failed to load webpack status: {fn}") - complete_stats['assets'] = {**complete_stats['assets'], **stats['assets']} - complete_stats['chunks'] = {**complete_stats['chunks'], **stats['chunks']} - complete_stats['status'] = 'done' - return complete_stats \ No newline at end of file + complete_stats["assets"] = { + **complete_stats["assets"], + **stats["assets"], + } + complete_stats["chunks"] = { + **complete_stats["chunks"], + **stats["chunks"], + } + complete_stats["status"] = "done" + return complete_stats diff --git a/neuvue_project/neuvue/wsgi.py b/neuvue_project/neuvue/wsgi.py index 9b8388b7..47699d0b 100644 --- a/neuvue_project/neuvue/wsgi.py +++ b/neuvue_project/neuvue/wsgi.py @@ -15,6 +15,6 @@ if BASE_DIR not in sys.path: sys.path.insert(0, BASE_DIR) -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'neuvue.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "neuvue.settings") application = get_wsgi_application() diff --git a/neuvue_project/preferences/apps.py b/neuvue_project/preferences/apps.py index 170dea8b..5bee935e 100644 --- a/neuvue_project/preferences/apps.py +++ b/neuvue_project/preferences/apps.py @@ -2,5 +2,5 @@ class PreferencesConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'preferences' + default_auto_field = "django.db.models.BigAutoField" + name = "preferences" diff --git a/neuvue_project/preferences/migrations/0001_initial.py b/neuvue_project/preferences/migrations/0001_initial.py index 3fba3b82..9d140df0 100644 --- a/neuvue_project/preferences/migrations/0001_initial.py +++ b/neuvue_project/preferences/migrations/0001_initial.py @@ -7,19 +7,26 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Config', + name="Config", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('user', models.CharField(default=None, max_length=50)), - ('enabled', models.BooleanField(default=False)), - ('alpha_selected', models.CharField(default='0.85', max_length=10)), - ('alpha_3d', models.CharField(default='0.5', max_length=10)), - ('layout', models.CharField(default='xy-3d', max_length=10)), + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("user", models.CharField(default=None, max_length=50)), + ("enabled", models.BooleanField(default=False)), + ("alpha_selected", models.CharField(default="0.85", max_length=10)), + ("alpha_3d", models.CharField(default="0.5", max_length=10)), + ("layout", models.CharField(default="xy-3d", max_length=10)), ], ), ] diff --git a/neuvue_project/preferences/migrations/0002_auto_20220222_1313.py b/neuvue_project/preferences/migrations/0002_auto_20220222_1313.py index e3fe3093..16180168 100644 --- a/neuvue_project/preferences/migrations/0002_auto_20220222_1313.py +++ b/neuvue_project/preferences/migrations/0002_auto_20220222_1313.py @@ -6,23 +6,23 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0001_initial'), + ("preferences", "0001_initial"), ] operations = [ migrations.AddField( - model_name='config', - name='chunk_requests', - field=models.CharField(default='32', max_length=10), + model_name="config", + name="chunk_requests", + field=models.CharField(default="32", max_length=10), ), migrations.AddField( - model_name='config', - name='gpu_limit', - field=models.CharField(default='1.0', max_length=10), + model_name="config", + name="gpu_limit", + field=models.CharField(default="1.0", max_length=10), ), migrations.AddField( - model_name='config', - name='sys_limit', - field=models.CharField(default='2.0', max_length=10), + model_name="config", + name="sys_limit", + field=models.CharField(default="2.0", max_length=10), ), ] diff --git a/neuvue_project/preferences/migrations/0003_config_annotation_color.py b/neuvue_project/preferences/migrations/0003_config_annotation_color.py index e4c7dabc..f21da3a1 100644 --- a/neuvue_project/preferences/migrations/0003_config_annotation_color.py +++ b/neuvue_project/preferences/migrations/0003_config_annotation_color.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0002_auto_20220222_1313'), + ("preferences", "0002_auto_20220222_1313"), ] operations = [ migrations.AddField( - model_name='config', - name='annotation_color', - field=models.CharField(default='#ffff00', max_length=10), + model_name="config", + name="annotation_color", + field=models.CharField(default="#ffff00", max_length=10), ), ] diff --git a/neuvue_project/preferences/migrations/0004_auto_20220314_1514.py b/neuvue_project/preferences/migrations/0004_auto_20220314_1514.py index 28619ee5..e4852678 100644 --- a/neuvue_project/preferences/migrations/0004_auto_20220314_1514.py +++ b/neuvue_project/preferences/migrations/0004_auto_20220314_1514.py @@ -6,43 +6,43 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0003_config_annotation_color'), + ("preferences", "0003_config_annotation_color"), ] operations = [ migrations.AddField( - model_name='config', - name='alpha_3d_switch', + model_name="config", + name="alpha_3d_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='alpha_selected_switch', + model_name="config", + name="alpha_selected_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='annotation_color_switch', + model_name="config", + name="annotation_color_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='chunk_requests_switch', + model_name="config", + name="chunk_requests_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='gpu_limit_switch', + model_name="config", + name="gpu_limit_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='layout_switch', + model_name="config", + name="layout_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='sys_limit_switch', + model_name="config", + name="sys_limit_switch", field=models.BooleanField(default=False), ), ] diff --git a/neuvue_project/preferences/migrations/0005_auto_20220420_1448.py b/neuvue_project/preferences/migrations/0005_auto_20220420_1448.py index b9f355cb..6e90e259 100644 --- a/neuvue_project/preferences/migrations/0005_auto_20220420_1448.py +++ b/neuvue_project/preferences/migrations/0005_auto_20220420_1448.py @@ -6,28 +6,28 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0004_auto_20220314_1514'), + ("preferences", "0004_auto_20220314_1514"), ] operations = [ migrations.AddField( - model_name='config', - name='show_slices', + model_name="config", + name="show_slices", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='show_slices_switch', + model_name="config", + name="show_slices_switch", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='zoom_level', - field=models.CharField(default='20', max_length=10), + model_name="config", + name="zoom_level", + field=models.CharField(default="20", max_length=10), ), migrations.AddField( - model_name='config', - name='zoom_level_switch', + model_name="config", + name="zoom_level_switch", field=models.BooleanField(default=False), ), ] diff --git a/neuvue_project/preferences/migrations/0006_auto_20220427_2149.py b/neuvue_project/preferences/migrations/0006_auto_20220427_2149.py index 26fb7f0d..e2e6f546 100644 --- a/neuvue_project/preferences/migrations/0006_auto_20220427_2149.py +++ b/neuvue_project/preferences/migrations/0006_auto_20220427_2149.py @@ -6,22 +6,24 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0005_auto_20220420_1448'), + ("preferences", "0005_auto_20220420_1448"), ] operations = [ migrations.RenameField( - model_name='config', - old_name='annotation_color_switch', - new_name='annotation_color_palette_switch', + model_name="config", + old_name="annotation_color_switch", + new_name="annotation_color_palette_switch", ), migrations.RemoveField( - model_name='config', - name='annotation_color', + model_name="config", + name="annotation_color", ), migrations.AddField( - model_name='config', - name='annotation_color_palette', - field=models.CharField(blank=True, default='palette1', max_length=10, null=True), + model_name="config", + name="annotation_color_palette", + field=models.CharField( + blank=True, default="palette1", max_length=10, null=True + ), ), ] diff --git a/neuvue_project/preferences/migrations/0007_auto_20220603_1425.py b/neuvue_project/preferences/migrations/0007_auto_20220603_1425.py index 4a9228ab..29a786f4 100644 --- a/neuvue_project/preferences/migrations/0007_auto_20220603_1425.py +++ b/neuvue_project/preferences/migrations/0007_auto_20220603_1425.py @@ -6,18 +6,20 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0006_auto_20220427_2149'), + ("preferences", "0006_auto_20220427_2149"), ] operations = [ migrations.AddField( - model_name='config', - name='mesh_color_palette', - field=models.CharField(blank=True, default='palette1', max_length=10, null=True), + model_name="config", + name="mesh_color_palette", + field=models.CharField( + blank=True, default="palette1", max_length=10, null=True + ), ), migrations.AddField( - model_name='config', - name='mesh_color_palette_switch', + model_name="config", + name="mesh_color_palette_switch", field=models.BooleanField(default=False), ), ] diff --git a/neuvue_project/preferences/migrations/0008_auto_20220617_1341.py b/neuvue_project/preferences/migrations/0008_auto_20220617_1341.py index ed50f682..1e767d6a 100644 --- a/neuvue_project/preferences/migrations/0008_auto_20220617_1341.py +++ b/neuvue_project/preferences/migrations/0008_auto_20220617_1341.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('preferences', '0007_auto_20220603_1425'), + ("preferences", "0007_auto_20220603_1425"), ] operations = [ migrations.AddField( - model_name='config', - name='enable_sound', + model_name="config", + name="enable_sound", field=models.BooleanField(default=False), ), migrations.AddField( - model_name='config', - name='enable_sound_switch', + model_name="config", + name="enable_sound_switch", field=models.BooleanField(default=False), ), ] diff --git a/neuvue_project/preferences/models.py b/neuvue_project/preferences/models.py index 7de9d49b..c5f9c664 100644 --- a/neuvue_project/preferences/models.py +++ b/neuvue_project/preferences/models.py @@ -1,32 +1,38 @@ from django.db import models from django.utils.translation import gettext_lazy as _ + # Create your models here. + class Config(models.Model): - user = models.CharField(max_length=50, default = None) + user = models.CharField(max_length=50, default=None) enabled = models.BooleanField(default=False) - + # Annotation Color Palette - annotation_color_palette = models.CharField(max_length=10, default='palette1', null=True, blank=True) + annotation_color_palette = models.CharField( + max_length=10, default="palette1", null=True, blank=True + ) annotation_color_palette_switch = models.BooleanField(default=False) # Mesh Color Palette - mesh_color_palette = models.CharField(max_length=10, default='palette1', null=True, blank=True) + mesh_color_palette = models.CharField( + max_length=10, default="palette1", null=True, blank=True + ) mesh_color_palette_switch = models.BooleanField(default=False) # 2D Show Slices show_slices = models.BooleanField(default=False) show_slices_switch = models.BooleanField(default=False) - # Zoom Level + # Zoom Level zoom_level = models.CharField(max_length=10, default="20") zoom_level_switch = models.BooleanField(default=False) - # 2D Segmentation Opacity + # 2D Segmentation Opacity alpha_selected = models.CharField(max_length=10, default="0.85") alpha_selected_switch = models.BooleanField(default=False) - # 3D Segmentation Opacity + # 3D Segmentation Opacity alpha_3d = models.CharField(max_length=10, default="0.5") alpha_3d_switch = models.BooleanField(default=False) diff --git a/neuvue_project/preferences/views.py b/neuvue_project/preferences/views.py index 14c0109e..ced7ee51 100644 --- a/neuvue_project/preferences/views.py +++ b/neuvue_project/preferences/views.py @@ -7,6 +7,7 @@ # import the logging library import logging + logging.basicConfig(level=logging.DEBUG) # Get an instance of a logger logger = logging.getLogger(__name__) @@ -20,78 +21,86 @@ def get(self, request, *args, **kwargs): else: logging.debug(f"Getting Config for {request.user}.") - config = Config.objects.filter(user=str(request.user)).order_by('-id')[0] # latest - + config = Config.objects.filter(user=str(request.user)).order_by("-id")[ + 0 + ] # latest + context = { - 'enabled': config.enabled, - 'annotationColorPalette': config.annotation_color_palette, - 'annotationColorPaletteSwitch': config.annotation_color_palette_switch, - 'meshColorPalette': config.mesh_color_palette, - 'meshColorPaletteSwitch': config.mesh_color_palette_switch, - 'showSlices':config.show_slices, - 'showSlicesSwitch':config.show_slices_switch, - 'alphaSelected': config.alpha_selected, - 'alphaSelectedSwitch': config.alpha_selected_switch, - 'alpha3D': config.alpha_3d, - 'alpha3DSwitch': config.alpha_3d_switch, - 'gpuLimit': config.gpu_limit, - 'gpuLimitSwitch': config.gpu_limit_switch, - 'sysLimit': config.sys_limit, - 'sysLimitSwitch': config.sys_limit_switch, - 'chunkReq': config.chunk_requests, - 'chunkReqSwitch': config.chunk_requests_switch, - 'layout': config.layout, - 'layoutSwitch': config.layout_switch, - 'zoomLevel': config.zoom_level, - 'zoomLevelSwitch': config.zoom_level_switch, - 'enableSound': config.enable_sound, - 'enableSoundSwitch': config.enable_sound_switch + "enabled": config.enabled, + "annotationColorPalette": config.annotation_color_palette, + "annotationColorPaletteSwitch": config.annotation_color_palette_switch, + "meshColorPalette": config.mesh_color_palette, + "meshColorPaletteSwitch": config.mesh_color_palette_switch, + "showSlices": config.show_slices, + "showSlicesSwitch": config.show_slices_switch, + "alphaSelected": config.alpha_selected, + "alphaSelectedSwitch": config.alpha_selected_switch, + "alpha3D": config.alpha_3d, + "alpha3DSwitch": config.alpha_3d_switch, + "gpuLimit": config.gpu_limit, + "gpuLimitSwitch": config.gpu_limit_switch, + "sysLimit": config.sys_limit, + "sysLimitSwitch": config.sys_limit_switch, + "chunkReq": config.chunk_requests, + "chunkReqSwitch": config.chunk_requests_switch, + "layout": config.layout, + "layoutSwitch": config.layout_switch, + "zoomLevel": config.zoom_level, + "zoomLevelSwitch": config.zoom_level_switch, + "enableSound": config.enable_sound, + "enableSoundSwitch": config.enable_sound_switch, } return render(request, "preferences.html", context) def post(self, request, *args, **kwargs): logging.debug(f"Update Config for {request.user}.") - config = Config.objects.filter(user=str(request.user)).order_by('-id')[0] - if request.POST.get('reset') == 'true' : + config = Config.objects.filter(user=str(request.user)).order_by("-id")[0] + if request.POST.get("reset") == "true": for field in Config._meta.get_fields(): - if field.name != 'user': + if field.name != "user": setattr(config, field.name, field.get_default()) config.save() else: - config.enabled = request.POST.get('enabled') == 'true' + config.enabled = request.POST.get("enabled") == "true" - config.annotation_color_palette = request.POST.get('annotationColorPalette') - config.annotation_color_palette_switch = request.POST.get('annotationColorPaletteSwitch') == 'true' + config.annotation_color_palette = request.POST.get("annotationColorPalette") + config.annotation_color_palette_switch = ( + request.POST.get("annotationColorPaletteSwitch") == "true" + ) - config.mesh_color_palette_switch = request.POST.get('meshColorPaletteSwitch') == 'true' - config.mesh_color_palette = request.POST.get('meshColorPalette') + config.mesh_color_palette_switch = ( + request.POST.get("meshColorPaletteSwitch") == "true" + ) + config.mesh_color_palette = request.POST.get("meshColorPalette") - config.show_slices = request.POST.get('showSlices') == 'true' - config.show_slices_switch = request.POST.get('showSlicesSwitch') == 'true' + config.show_slices = request.POST.get("showSlices") == "true" + config.show_slices_switch = request.POST.get("showSlicesSwitch") == "true" - config.zoom_level = request.POST.get('zoomLevel') - config.zoom_level_switch = request.POST.get('zoomLevelSwitch') == 'true' + config.zoom_level = request.POST.get("zoomLevel") + config.zoom_level_switch = request.POST.get("zoomLevelSwitch") == "true" - config.alpha_selected = request.POST.get('alphaSelected') - config.alpha_selected_switch = request.POST.get('alphaSelectedSwitch') == 'true' + config.alpha_selected = request.POST.get("alphaSelected") + config.alpha_selected_switch = ( + request.POST.get("alphaSelectedSwitch") == "true" + ) - config.alpha_3d = request.POST.get('alpha3D') - config.alpha_3d_switch = request.POST.get('alpha3DSwitch') == 'true' + config.alpha_3d = request.POST.get("alpha3D") + config.alpha_3d_switch = request.POST.get("alpha3DSwitch") == "true" - config.gpu_limit = request.POST.get('gpuLimit') - config.gpu_limit_switch = request.POST.get('gpuLimitSwitch') == 'true' + config.gpu_limit = request.POST.get("gpuLimit") + config.gpu_limit_switch = request.POST.get("gpuLimitSwitch") == "true" - config.sys_limit = request.POST.get('sysLimit') - config.sys_limit_switch = request.POST.get('sysLimitSwitch') == 'true' + config.sys_limit = request.POST.get("sysLimit") + config.sys_limit_switch = request.POST.get("sysLimitSwitch") == "true" - config.chunk_requests = request.POST.get('chunkReq') - config.chunk_requests_switch = request.POST.get('chunkReqSwitch') == 'true' + config.chunk_requests = request.POST.get("chunkReq") + config.chunk_requests_switch = request.POST.get("chunkReqSwitch") == "true" - config.layout = request.POST.get('layout') - config.layout_switch = request.POST.get('layoutSwitch') == 'true' + config.layout = request.POST.get("layout") + config.layout_switch = request.POST.get("layoutSwitch") == "true" - config.enable_sound = request.POST.get('enableSound') == 'true' - config.enable_sound_switch = request.POST.get('enableSoundSwitch') == 'true' + config.enable_sound = request.POST.get("enableSound") == "true" + config.enable_sound_switch = request.POST.get("enableSoundSwitch") == "true" config.save() - return redirect(reverse('preferences')) \ No newline at end of file + return redirect(reverse("preferences")) diff --git a/neuvue_project/run-dev-server.py b/neuvue_project/run-dev-server.py index 52a2e015..b951b1e4 100755 --- a/neuvue_project/run-dev-server.py +++ b/neuvue_project/run-dev-server.py @@ -3,18 +3,18 @@ import subprocess # Turn debug mode on in settings.py -with open('neuvue/settings.py', 'r') as f: +with open("neuvue/settings.py", "r") as f: settings = f.read() settings = settings.replace("DEBUG = False", "DEBUG = True") -with open('neuvue/settings.py', 'w') as f: +with open("neuvue/settings.py", "w") as f: f.write(settings) # Get recent migrations to database -subprocess.run(['python3', 'manage.py', 'migrate']) +subprocess.run(["python3", "manage.py", "migrate"]) # Collect static files -subprocess.run(['python3', 'manage.py', 'collectstatic', '--no-input']) +subprocess.run(["python3", "manage.py", "collectstatic", "--no-input"]) # Run Dev server on localhost -subprocess.run(['python3', 'manage.py', 'runserver', 'localhost:8000']) +subprocess.run(["python3", "manage.py", "runserver", "localhost:8000"]) diff --git a/neuvue_project/workspace/admin.py b/neuvue_project/workspace/admin.py index 2ac7e15d..1cfe29df 100644 --- a/neuvue_project/workspace/admin.py +++ b/neuvue_project/workspace/admin.py @@ -8,6 +8,7 @@ admin.site.unregister(User) admin.site.unregister(Group) + class ButtonsInline(admin.TabularInline): model = ForcedChoiceButton verbose_name = "forced choice button" @@ -15,44 +16,93 @@ class ButtonsInline(admin.TabularInline): extra = 1 max_num = 8 + @admin.register(ForcedChoiceButtonGroup) class ForcedChoiceAdmin(admin.ModelAdmin): - inlines = (ButtonsInline, ) + inlines = (ButtonsInline,) + class NamespaceAdmin(admin.ModelAdmin): - list_display = ('namespace', 'namespace_enabled') + list_display = ("namespace", "namespace_enabled") fieldsets = [ - ('Namespace Information', {'fields': ['namespace_enabled', 'namespace', 'display_name', 'ng_link_type', 'submission_method', 'pcg_source', - 'img_source', 'track_operation_ids', 'refresh_selected_root_ids', 'number_of_tasks_users_can_self_assign', 'max_number_of_pending_tasks_per_user', 'track_selected_segments']}), - ('Novice Fields', {'fields': ['novice_pull_from', 'novice_push_to']}), - ('Intermediate Fields', {'fields': ['intermediate_pull_from', 'intermediate_push_to']}), - ('Expert Fields', {'fields': ['expert_pull_from', 'expert_push_to']}), + ( + "Namespace Information", + { + "fields": [ + "namespace_enabled", + "namespace", + "display_name", + "ng_link_type", + "submission_method", + "pcg_source", + "img_source", + "track_operation_ids", + "refresh_selected_root_ids", + "number_of_tasks_users_can_self_assign", + "max_number_of_pending_tasks_per_user", + "track_selected_segments", + ] + }, + ), + ("Novice Fields", {"fields": ["novice_pull_from", "novice_push_to"]}), + ( + "Intermediate Fields", + {"fields": ["intermediate_pull_from", "intermediate_push_to"]}, + ), + ("Expert Fields", {"fields": ["expert_pull_from", "expert_push_to"]}), ] + class UserProfileInline(admin.StackedInline): model = UserProfile - filter_horizontal = ('intermediate_namespaces', 'expert_namespaces') + filter_horizontal = ("intermediate_namespaces", "expert_namespaces") + class CustomUserAdmin(UserAdmin): - #filter_horizontal = ('user_permissions', 'groups', 'ope') + # filter_horizontal = ('user_permissions', 'groups', 'ope') save_on_top = True - fieldsets = [('User Information', {'fields': ['username', 'password']}), - ('Personal Info', {'fields': ['first_name', 'last_name', 'email']}), - ('Important Dates', {'fields': ['last_login', 'date_joined']}), - ('Permissions', {'fields': ['is_active', 'is_staff', 'is_superuser','groups', 'user_permissions']}), + fieldsets = [ + ("User Information", {"fields": ["username", "password"]}), + ("Personal Info", {"fields": ["first_name", "last_name", "email"]}), + ("Important Dates", {"fields": ["last_login", "date_joined"]}), + ( + "Permissions", + { + "fields": [ + "is_active", + "is_staff", + "is_superuser", + "groups", + "user_permissions", + ] + }, + ), ] inlines = [UserProfileInline] + NOVICE = "novice" INTERMEDIATE = "intermediate" EXPERT = "expert" + class SetExpertiseLevelForNamespaceForm(admin.helpers.ActionForm): - namespace = forms.ModelChoiceField(Namespace.objects, label=False, empty_label="Namespace", widget=forms.Select(attrs={"class": "mb-1"}), required=False) + namespace = forms.ModelChoiceField( + Namespace.objects, + label=False, + empty_label="Namespace", + widget=forms.Select(attrs={"class": "mb-1"}), + required=False, + ) + class CustomGroupAdmin(GroupAdmin): action_form = SetExpertiseLevelForNamespaceForm - actions = ['set_expertise_level_for_namespace_novice', 'set_expertise_level_for_namespace_intermediate', 'set_expertise_level_for_namespace_expert'] + actions = [ + "set_expertise_level_for_namespace_novice", + "set_expertise_level_for_namespace_intermediate", + "set_expertise_level_for_namespace_expert", + ] def set_expertise_level_for_namespace_novice(self, request, queryset): self.set_expertise_level_for_namespace(NOVICE, request, queryset) @@ -62,12 +112,12 @@ def set_expertise_level_for_namespace_intermediate(self, request, queryset): def set_expertise_level_for_namespace_expert(self, request, queryset): self.set_expertise_level_for_namespace(EXPERT, request, queryset) - + def set_expertise_level_for_namespace(self, level, request, queryset): namespace = "" try: - namespace = Namespace.objects.get(namespace = request.POST['namespace']) + namespace = Namespace.objects.get(namespace=request.POST["namespace"]) except: messages.error(request, "Please select a namespace.") return @@ -75,7 +125,7 @@ def set_expertise_level_for_namespace(self, level, request, queryset): for group in queryset: users = Group.objects.get(name=group).user_set.all() for user in users: - userProfile, _ = UserProfile.objects.get_or_create(user = user) + userProfile, _ = UserProfile.objects.get_or_create(user=user) if level == NOVICE: userProfile.intermediate_namespaces.remove(namespace) userProfile.expert_namespaces.remove(namespace) @@ -85,13 +135,25 @@ def set_expertise_level_for_namespace(self, level, request, queryset): if level == EXPERT: userProfile.intermediate_namespaces.remove(namespace) userProfile.expert_namespaces.add(namespace) - messages.success(request, "Designated all members of {} as {} for {}".format(", ".join([q.name for q in queryset]), level, namespace)) + messages.success( + request, + "Designated all members of {} as {} for {}".format( + ", ".join([q.name for q in queryset]), level, namespace + ), + ) + + set_expertise_level_for_namespace_novice.short_description = ( + "Assign all members of group as novice for namespace" + ) + set_expertise_level_for_namespace_intermediate.short_description = ( + "Assign all members of group as intermediate for namespace" + ) + set_expertise_level_for_namespace_expert.short_description = ( + "Assign all members of group as expert for namespace" + ) - set_expertise_level_for_namespace_novice.short_description = 'Assign all members of group as novice for namespace' - set_expertise_level_for_namespace_intermediate.short_description = 'Assign all members of group as intermediate for namespace' - set_expertise_level_for_namespace_expert.short_description = 'Assign all members of group as expert for namespace' # Register your models here. admin.site.register(Namespace, NamespaceAdmin) admin.site.register(User, CustomUserAdmin) -admin.site.register(Group, CustomGroupAdmin) \ No newline at end of file +admin.site.register(Group, CustomGroupAdmin) diff --git a/neuvue_project/workspace/analytics.py b/neuvue_project/workspace/analytics.py index 5403f539..f34efa96 100644 --- a/neuvue_project/workspace/analytics.py +++ b/neuvue_project/workspace/analytics.py @@ -6,10 +6,12 @@ # import the logging library import logging + logging.basicConfig(level=logging.DEBUG) # Get an instance of a logger logger = logging.getLogger(__name__) + def is_lastweek(timestamp): """Returns if the datetime provided is in the last week or not. @@ -19,33 +21,36 @@ def is_lastweek(timestamp): Returns: bool: True if in last week """ - eastern = timezone('US/Eastern') - - dt = timestamp.to_pydatetime(warn=False) # do not warn if nanoseconds are nonzero + eastern = timezone("US/Eastern") + + dt = timestamp.to_pydatetime(warn=False) # do not warn if nanoseconds are nonzero now = eastern.localize(datetime.now()) weekago = now - timedelta(days=7) - + return weekago <= dt <= now - + + def get_sum_time(table): - seconds = np.array([x['duration'] for x in table]).sum() - hours = round(seconds/(60*60), 1) + seconds = np.array([x["duration"] for x in table]).sum() + hours = round(seconds / (60 * 60), 1) return hours - + + def get_rate(table): - durations = np.array([x['duration'] for x in table]) + durations = np.array([x["duration"] for x in table]) if len(durations) > 0: - mean = durations.mean() - minutes = round(mean/(60)) + mean = durations.mean() + minutes = round(mean / (60)) return minutes else: return 0 + def user_stats(table): # look at weekly stats - weekly_table = [x for x in table if is_lastweek(x['closed'])] + weekly_table = [x for x in table if is_lastweek(x["closed"])] try: stats = { "total_tasks": len(table), @@ -53,52 +58,77 @@ def user_stats(table): "total_time": get_sum_time(table), "weekly_time": get_sum_time(weekly_table), "total_rate": get_rate(table), - "weekly_rate": get_rate(weekly_table) + "weekly_rate": get_rate(weekly_table), } except Exception as e: logging.error(f"Error computing analytics: {e}") stats = { - "total_tasks": 'n/a', - "weekly_tasks": 'n/a', - "total_time": 'n/a', - "weekly_time": 'n/a', - "total_rate": 'n/a', - "weekly_rate": 'n/a' + "total_tasks": "n/a", + "weekly_tasks": "n/a", + "total_time": "n/a", + "weekly_time": "n/a", + "total_rate": "n/a", + "weekly_rate": "n/a", } - + return stats + def create_stats_table(pending_tasks, closed_tasks): - all_user_tasks = pd.concat([pending_tasks,closed_tasks]) - all_user_tasks = all_user_tasks[~all_user_tasks.index.duplicated(keep='first')].reset_index(drop=True) + all_user_tasks = pd.concat([pending_tasks, closed_tasks]) + all_user_tasks = all_user_tasks[ + ~all_user_tasks.index.duplicated(keep="first") + ].reset_index(drop=True) twentyFour_hrs_ago = datetime.now() - timedelta(days=1) daily_changelog_items = [] full_changelog_items = [] frames = [ - (daily_changelog_items, 'closed', all_user_tasks[all_user_tasks.closed >= twentyFour_hrs_ago]), - (daily_changelog_items, 'created', all_user_tasks[all_user_tasks.created >= twentyFour_hrs_ago]), - (full_changelog_items, 'closed', all_user_tasks[all_user_tasks.closed < twentyFour_hrs_ago]), - (full_changelog_items, 'created', all_user_tasks[all_user_tasks.created < twentyFour_hrs_ago]) + ( + daily_changelog_items, + "closed", + all_user_tasks[all_user_tasks.closed >= twentyFour_hrs_ago], + ), + ( + daily_changelog_items, + "created", + all_user_tasks[all_user_tasks.created >= twentyFour_hrs_ago], + ), + ( + full_changelog_items, + "closed", + all_user_tasks[all_user_tasks.closed < twentyFour_hrs_ago], + ), + ( + full_changelog_items, + "created", + all_user_tasks[all_user_tasks.created < twentyFour_hrs_ago], + ), ] for changelog, status, df in frames: - for namespace, namespace_df in df.groupby('namespace'): + for namespace, namespace_df in df.groupby("namespace"): n_tasks = len(namespace_df) m = namespace_df[status].min() - average_event_time = (m + (namespace_df[status]-m)).mean().to_pydatetime() + average_event_time = (m + (namespace_df[status] - m)).mean().to_pydatetime() changelog.append((average_event_time, n_tasks, status, namespace)) - + # Sort changelogs by datetime - daily_changelog_items = sorted(daily_changelog_items, key=lambda x: x[0], reverse=True) - full_changelog_items = sorted(full_changelog_items, key=lambda x: x[0], reverse=True) - + daily_changelog_items = sorted( + daily_changelog_items, key=lambda x: x[0], reverse=True + ) + full_changelog_items = sorted( + full_changelog_items, key=lambda x: x[0], reverse=True + ) + def generate_changelog_text(changelog_items): changelog_strings = [ - f"{x[0].strftime('%m/%d/%Y, %H:%M:%S')}: {x[1]} tasks {x[2]} from {x[3]}
" + f"{x[0].strftime('%m/%d/%Y, %H:%M:%S')}: {x[1]} tasks {x[2]} from {x[3]}
" for x in changelog_items - ] - return '
' + "\n".join(changelog_strings) + '
' + ] + return "
" + "\n".join(changelog_strings) + "
" - return generate_changelog_text(daily_changelog_items), generate_changelog_text(full_changelog_items) \ No newline at end of file + return generate_changelog_text(daily_changelog_items), generate_changelog_text( + full_changelog_items + ) diff --git a/neuvue_project/workspace/apps.py b/neuvue_project/workspace/apps.py index 98012f2e..4fda5f61 100644 --- a/neuvue_project/workspace/apps.py +++ b/neuvue_project/workspace/apps.py @@ -2,10 +2,11 @@ class WorkspaceConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'workspace' + default_auto_field = "django.db.models.BigAutoField" + name = "workspace" -#test + +# test class TestConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'tests' + default_auto_field = "django.db.models.BigAutoField" + name = "tests" diff --git a/neuvue_project/workspace/migrations/0001_initial.py b/neuvue_project/workspace/migrations/0001_initial.py index b6a771ae..0d4bb726 100644 --- a/neuvue_project/workspace/migrations/0001_initial.py +++ b/neuvue_project/workspace/migrations/0001_initial.py @@ -7,17 +7,40 @@ class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='Namespace', + name="Namespace", fields=[ - ('namespace', models.CharField(max_length=50, primary_key=True, serialize=False)), - ('display_name', models.CharField(max_length=100)), - ('ng_link_type', models.CharField(choices=[('path', 'Path Layer'), ('point', 'Annotation Layer'), ('pregen', 'Pregenerated')], default='point', max_length=50)), - ('submission_method', models.CharField(choices=[('submit', 'Submit Button'), ('forced_choice', 'Yes/No/Maybe Button')], default='submit', max_length=50)), + ( + "namespace", + models.CharField(max_length=50, primary_key=True, serialize=False), + ), + ("display_name", models.CharField(max_length=100)), + ( + "ng_link_type", + models.CharField( + choices=[ + ("path", "Path Layer"), + ("point", "Annotation Layer"), + ("pregen", "Pregenerated"), + ], + default="point", + max_length=50, + ), + ), + ( + "submission_method", + models.CharField( + choices=[ + ("submit", "Submit Button"), + ("forced_choice", "Yes/No/Maybe Button"), + ], + default="submit", + max_length=50, + ), + ), ], ), ] diff --git a/neuvue_project/workspace/migrations/0002_auto_20211206_0227.py b/neuvue_project/workspace/migrations/0002_auto_20211206_0227.py index 7be0a488..96d707b1 100644 --- a/neuvue_project/workspace/migrations/0002_auto_20211206_0227.py +++ b/neuvue_project/workspace/migrations/0002_auto_20211206_0227.py @@ -6,18 +6,44 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0001_initial'), + ("workspace", "0001_initial"), ] operations = [ migrations.AddField( - model_name='namespace', - name='img_source', - field=models.CharField(choices=[('https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em', 'Minnie65'), ('gs://microns_public_datasets/pinky100_v0/son_of_alignment_v15_rechunked', 'Pinky')], default='https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em', max_length=300), + model_name="namespace", + name="img_source", + field=models.CharField( + choices=[ + ( + "https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em", + "Minnie65", + ), + ( + "gs://microns_public_datasets/pinky100_v0/son_of_alignment_v15_rechunked", + "Pinky", + ), + ], + default="https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em", + max_length=300, + ), ), migrations.AddField( - model_name='namespace', - name='pcg_source', - field=models.CharField(choices=[('https://minnie.microns-daf.com/segmentation/table/minnie3_v1', 'Minnie65'), ('https://minnie.microns-daf.com/segmentation/table/pinky_nf_v2', 'Pinky')], default='https://minnie.microns-daf.com/segmentation/table/minnie3_v1', max_length=300), + model_name="namespace", + name="pcg_source", + field=models.CharField( + choices=[ + ( + "https://minnie.microns-daf.com/segmentation/table/minnie3_v1", + "Minnie65", + ), + ( + "https://minnie.microns-daf.com/segmentation/table/pinky_nf_v2", + "Pinky", + ), + ], + default="https://minnie.microns-daf.com/segmentation/table/minnie3_v1", + max_length=300, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0003_auto_20220125_1238.py b/neuvue_project/workspace/migrations/0003_auto_20220125_1238.py index 84168d5e..6f7844e1 100644 --- a/neuvue_project/workspace/migrations/0003_auto_20220125_1238.py +++ b/neuvue_project/workspace/migrations/0003_auto_20220125_1238.py @@ -6,18 +6,39 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0002_auto_20211206_0227'), + ("workspace", "0002_auto_20211206_0227"), ] operations = [ migrations.AlterField( - model_name='namespace', - name='ng_link_type', - field=models.CharField(choices=[('path', 'Path Layer'), ('point', 'Annotation Layer'), ('pregen', 'Pregenerated')], default='pregen', max_length=50), + model_name="namespace", + name="ng_link_type", + field=models.CharField( + choices=[ + ("path", "Path Layer"), + ("point", "Annotation Layer"), + ("pregen", "Pregenerated"), + ], + default="pregen", + max_length=50, + ), ), migrations.AlterField( - model_name='namespace', - name='pcg_source', - field=models.CharField(choices=[('https://minnie.microns-daf.com/segmentation/table/minnie3_v1', 'Minnie65'), ('https://minnie.microns-daf.com/segmentation/table/pinky_v2_microns_sandbox', 'Pinky')], default='https://minnie.microns-daf.com/segmentation/table/minnie3_v1', max_length=300), + model_name="namespace", + name="pcg_source", + field=models.CharField( + choices=[ + ( + "https://minnie.microns-daf.com/segmentation/table/minnie3_v1", + "Minnie65", + ), + ( + "https://minnie.microns-daf.com/segmentation/table/pinky_v2_microns_sandbox", + "Pinky", + ), + ], + default="https://minnie.microns-daf.com/segmentation/table/minnie3_v1", + max_length=300, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0004_alter_namespace_submission_method.py b/neuvue_project/workspace/migrations/0004_alter_namespace_submission_method.py index 7e11959c..2d7c00ed 100644 --- a/neuvue_project/workspace/migrations/0004_alter_namespace_submission_method.py +++ b/neuvue_project/workspace/migrations/0004_alter_namespace_submission_method.py @@ -6,13 +6,21 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0003_auto_20220125_1238'), + ("workspace", "0003_auto_20220125_1238"), ] operations = [ migrations.AlterField( - model_name='namespace', - name='submission_method', - field=models.CharField(choices=[('submit', 'Submit Button'), ('forced_choice', 'Yes/No/Maybe Button'), ('decide_and_submit', 'Decide and Submit Button')], default='submit', max_length=50), + model_name="namespace", + name="submission_method", + field=models.CharField( + choices=[ + ("submit", "Submit Button"), + ("forced_choice", "Yes/No/Maybe Button"), + ("decide_and_submit", "Decide and Submit Button"), + ], + default="submit", + max_length=50, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0005_namespace_track_operation_ids.py b/neuvue_project/workspace/migrations/0005_namespace_track_operation_ids.py index 455cb595..7e8f32d1 100644 --- a/neuvue_project/workspace/migrations/0005_namespace_track_operation_ids.py +++ b/neuvue_project/workspace/migrations/0005_namespace_track_operation_ids.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0004_alter_namespace_submission_method'), + ("workspace", "0004_alter_namespace_submission_method"), ] operations = [ migrations.AddField( - model_name='namespace', - name='track_operation_ids', + model_name="namespace", + name="track_operation_ids", field=models.BooleanField(default=True), ), ] diff --git a/neuvue_project/workspace/migrations/0006_namespace_namespace_enabled.py b/neuvue_project/workspace/migrations/0006_namespace_namespace_enabled.py index 64462048..7a44eab8 100644 --- a/neuvue_project/workspace/migrations/0006_namespace_namespace_enabled.py +++ b/neuvue_project/workspace/migrations/0006_namespace_namespace_enabled.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0005_namespace_track_operation_ids'), + ("workspace", "0005_namespace_track_operation_ids"), ] operations = [ migrations.AddField( - model_name='namespace', - name='namespace_enabled', + model_name="namespace", + name="namespace_enabled", field=models.BooleanField(default=True), ), ] diff --git a/neuvue_project/workspace/migrations/0007_namespace_refresh_selected_root_ids.py b/neuvue_project/workspace/migrations/0007_namespace_refresh_selected_root_ids.py index 0808f50d..84dc2e1f 100644 --- a/neuvue_project/workspace/migrations/0007_namespace_refresh_selected_root_ids.py +++ b/neuvue_project/workspace/migrations/0007_namespace_refresh_selected_root_ids.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0006_namespace_namespace_enabled'), + ("workspace", "0006_namespace_namespace_enabled"), ] operations = [ migrations.AddField( - model_name='namespace', - name='refresh_selected_root_ids', + model_name="namespace", + name="refresh_selected_root_ids", field=models.BooleanField(default=False), ), ] diff --git a/neuvue_project/workspace/migrations/0008_auto_20220314_2318.py b/neuvue_project/workspace/migrations/0008_auto_20220314_2318.py index 5003b6d5..e1427df5 100644 --- a/neuvue_project/workspace/migrations/0008_auto_20220314_2318.py +++ b/neuvue_project/workspace/migrations/0008_auto_20220314_2318.py @@ -6,18 +6,18 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0007_namespace_refresh_selected_root_ids'), + ("workspace", "0007_namespace_refresh_selected_root_ids"), ] operations = [ migrations.AddField( - model_name='namespace', - name='max_number_of_pending_tasks_per_user', + model_name="namespace", + name="max_number_of_pending_tasks_per_user", field=models.IntegerField(default=200), ), migrations.AddField( - model_name='namespace', - name='number_of_tasks_users_can_self_assign', + model_name="namespace", + name="number_of_tasks_users_can_self_assign", field=models.IntegerField(default=10), ), ] diff --git a/neuvue_project/workspace/migrations/0009_auto_20220322_1616.py b/neuvue_project/workspace/migrations/0009_auto_20220322_1616.py index fcaf2f29..e132c78d 100644 --- a/neuvue_project/workspace/migrations/0009_auto_20220322_1616.py +++ b/neuvue_project/workspace/migrations/0009_auto_20220322_1616.py @@ -9,47 +9,129 @@ class Migration(migrations.Migration): dependencies = [ migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('workspace', '0008_auto_20220314_2318'), + ("workspace", "0008_auto_20220314_2318"), ] operations = [ migrations.AddField( - model_name='namespace', - name='expert_pull_from', - field=models.CharField(choices=[('Reassign Tasks Not Allowed', 'Null'), ('unassigned_novice', 'Novice'), ('unassigned_intermediate', 'Intermediate'), ('unassigned_expert', 'Expert')], default='Reassign Tasks Not Allowed', max_length=50), + model_name="namespace", + name="expert_pull_from", + field=models.CharField( + choices=[ + ("Reassign Tasks Not Allowed", "Null"), + ("unassigned_novice", "Novice"), + ("unassigned_intermediate", "Intermediate"), + ("unassigned_expert", "Expert"), + ], + default="Reassign Tasks Not Allowed", + max_length=50, + ), ), migrations.AddField( - model_name='namespace', - name='expert_push_to', - field=models.CharField(choices=[('Queue Tasks Not Allowed', 'Null'), ('unassigned_novice', 'Novice'), ('unassigned_intermediate', 'Intermediate'), ('unassigned_expert', 'Expert')], default='Queue Tasks Not Allowed', max_length=50), + model_name="namespace", + name="expert_push_to", + field=models.CharField( + choices=[ + ("Queue Tasks Not Allowed", "Null"), + ("unassigned_novice", "Novice"), + ("unassigned_intermediate", "Intermediate"), + ("unassigned_expert", "Expert"), + ], + default="Queue Tasks Not Allowed", + max_length=50, + ), ), migrations.AddField( - model_name='namespace', - name='intermediate_pull_from', - field=models.CharField(choices=[('Reassign Tasks Not Allowed', 'Null'), ('unassigned_novice', 'Novice'), ('unassigned_intermediate', 'Intermediate'), ('unassigned_expert', 'Expert')], default='Reassign Tasks Not Allowed', max_length=50), + model_name="namespace", + name="intermediate_pull_from", + field=models.CharField( + choices=[ + ("Reassign Tasks Not Allowed", "Null"), + ("unassigned_novice", "Novice"), + ("unassigned_intermediate", "Intermediate"), + ("unassigned_expert", "Expert"), + ], + default="Reassign Tasks Not Allowed", + max_length=50, + ), ), migrations.AddField( - model_name='namespace', - name='intermediate_push_to', - field=models.CharField(choices=[('Queue Tasks Not Allowed', 'Null'), ('unassigned_novice', 'Novice'), ('unassigned_intermediate', 'Intermediate'), ('unassigned_expert', 'Expert')], default='Queue Tasks Not Allowed', max_length=50), + model_name="namespace", + name="intermediate_push_to", + field=models.CharField( + choices=[ + ("Queue Tasks Not Allowed", "Null"), + ("unassigned_novice", "Novice"), + ("unassigned_intermediate", "Intermediate"), + ("unassigned_expert", "Expert"), + ], + default="Queue Tasks Not Allowed", + max_length=50, + ), ), migrations.AddField( - model_name='namespace', - name='novice_pull_from', - field=models.CharField(choices=[('Reassign Tasks Not Allowed', 'Null'), ('unassigned_novice', 'Novice'), ('unassigned_intermediate', 'Intermediate'), ('unassigned_expert', 'Expert')], default='Reassign Tasks Not Allowed', max_length=50), + model_name="namespace", + name="novice_pull_from", + field=models.CharField( + choices=[ + ("Reassign Tasks Not Allowed", "Null"), + ("unassigned_novice", "Novice"), + ("unassigned_intermediate", "Intermediate"), + ("unassigned_expert", "Expert"), + ], + default="Reassign Tasks Not Allowed", + max_length=50, + ), ), migrations.AddField( - model_name='namespace', - name='novice_push_to', - field=models.CharField(choices=[('Queue Tasks Not Allowed', 'Null'), ('unassigned_novice', 'Novice'), ('unassigned_intermediate', 'Intermediate'), ('unassigned_expert', 'Expert')], default='Queue Tasks Not Allowed', max_length=50), + model_name="namespace", + name="novice_push_to", + field=models.CharField( + choices=[ + ("Queue Tasks Not Allowed", "Null"), + ("unassigned_novice", "Novice"), + ("unassigned_intermediate", "Intermediate"), + ("unassigned_expert", "Expert"), + ], + default="Queue Tasks Not Allowed", + max_length=50, + ), ), migrations.CreateModel( - name='UserProfile', + name="UserProfile", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('expert_namespaces', models.ManyToManyField(blank=True, related_name='expert_namespaces', to='workspace.Namespace')), - ('intermediate_namespaces', models.ManyToManyField(blank=True, related_name='intermediate_namespaces', to='workspace.Namespace')), - ('user', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "expert_namespaces", + models.ManyToManyField( + blank=True, + related_name="expert_namespaces", + to="workspace.Namespace", + ), + ), + ( + "intermediate_namespaces", + models.ManyToManyField( + blank=True, + related_name="intermediate_namespaces", + to="workspace.Namespace", + ), + ), + ( + "user", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], ), ] diff --git a/neuvue_project/workspace/migrations/0010_alter_namespace_submission_method.py b/neuvue_project/workspace/migrations/0010_alter_namespace_submission_method.py index 7016f6a3..eb03764e 100644 --- a/neuvue_project/workspace/migrations/0010_alter_namespace_submission_method.py +++ b/neuvue_project/workspace/migrations/0010_alter_namespace_submission_method.py @@ -6,13 +6,22 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0009_auto_20220322_1616'), + ("workspace", "0009_auto_20220322_1616"), ] operations = [ migrations.AlterField( - model_name='namespace', - name='submission_method', - field=models.CharField(choices=[('submit', 'Submit Button'), ('forced_choice', 'Forced Choice'), ('yes_no_maybe', 'Y/N/M'), ('decide_and_submit', 'Decide and Submit Button')], default='submit', max_length=50), + model_name="namespace", + name="submission_method", + field=models.CharField( + choices=[ + ("submit", "Submit Button"), + ("forced_choice", "Forced Choice"), + ("yes_no_maybe", "Y/N/M"), + ("decide_and_submit", "Decide and Submit Button"), + ], + default="submit", + max_length=50, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0011_alter_namespace_submission_method.py b/neuvue_project/workspace/migrations/0011_alter_namespace_submission_method.py index 37035849..574be6b1 100644 --- a/neuvue_project/workspace/migrations/0011_alter_namespace_submission_method.py +++ b/neuvue_project/workspace/migrations/0011_alter_namespace_submission_method.py @@ -6,13 +6,22 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0010_alter_namespace_submission_method'), + ("workspace", "0010_alter_namespace_submission_method"), ] operations = [ migrations.AlterField( - model_name='namespace', - name='submission_method', - field=models.CharField(choices=[('submit', 'Submit Button'), ('forced_choice', 'Forced Choice'), ('yes_no', 'Y/N'), ('decide_and_submit', 'Decide and Submit Button')], default='submit', max_length=50), + model_name="namespace", + name="submission_method", + field=models.CharField( + choices=[ + ("submit", "Submit Button"), + ("forced_choice", "Forced Choice"), + ("yes_no", "Y/N"), + ("decide_and_submit", "Decide and Submit Button"), + ], + default="submit", + max_length=50, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0012_alter_namespace_submission_method.py b/neuvue_project/workspace/migrations/0012_alter_namespace_submission_method.py index 1f3bacce..b26756d5 100644 --- a/neuvue_project/workspace/migrations/0012_alter_namespace_submission_method.py +++ b/neuvue_project/workspace/migrations/0012_alter_namespace_submission_method.py @@ -6,13 +6,23 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0011_alter_namespace_submission_method'), + ("workspace", "0011_alter_namespace_submission_method"), ] operations = [ migrations.AlterField( - model_name='namespace', - name='submission_method', - field=models.CharField(choices=[('submit', 'Submit Button'), ('forced_choice', 'Forced Choice'), ('yes_no', 'Y/N'), ('decide_and_submit', 'Decide and Submit Button'), ('extension_choice', 'Extension Choice')], default='submit', max_length=50), + model_name="namespace", + name="submission_method", + field=models.CharField( + choices=[ + ("submit", "Submit Button"), + ("forced_choice", "Forced Choice"), + ("yes_no", "Y/N"), + ("decide_and_submit", "Decide and Submit Button"), + ("extension_choice", "Extension Choice"), + ], + default="submit", + max_length=50, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0013_auto_20220611_1200.py b/neuvue_project/workspace/migrations/0013_auto_20220611_1200.py index 8317d8f6..a62f1cbc 100644 --- a/neuvue_project/workspace/migrations/0013_auto_20220611_1200.py +++ b/neuvue_project/workspace/migrations/0013_auto_20220611_1200.py @@ -9,36 +9,93 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0012_alter_namespace_submission_method'), + ("workspace", "0012_alter_namespace_submission_method"), ] operations = [ migrations.CreateModel( - name='ForcedChoiceButtonGroup', + name="ForcedChoiceButtonGroup", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('group_name', models.CharField(help_text='(snake case)', max_length=100, unique=True)), - ('submit_task_button', models.BooleanField(default=True)), + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "group_name", + models.CharField( + help_text="(snake case)", max_length=100, unique=True + ), + ), + ("submit_task_button", models.BooleanField(default=True)), ], options={ - 'verbose_name': 'Forced Choice Button Group', - 'verbose_name_plural': 'Forced Choice Button Groups', + "verbose_name": "Forced Choice Button Group", + "verbose_name_plural": "Forced Choice Button Groups", }, ), migrations.CreateModel( - name='ForcedChoiceButton', + name="ForcedChoiceButton", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('display_name', models.CharField(max_length=100)), - ('submission_value', models.CharField(max_length=100, validators=[workspace.validators.validate_submission_value])), - ('button_color', colorfield.fields.ColorField(default='#FFFFFFFF', image_field=None, max_length=18, samples=None)), - ('button_color_active', colorfield.fields.ColorField(default='#FFFFFFFF', image_field=None, max_length=18, samples=None)), - ('set_name', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='workspace.forcedchoicebuttongroup', to_field='group_name')), + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("display_name", models.CharField(max_length=100)), + ( + "submission_value", + models.CharField( + max_length=100, + validators=[workspace.validators.validate_submission_value], + ), + ), + ( + "button_color", + colorfield.fields.ColorField( + default="#FFFFFFFF", + image_field=None, + max_length=18, + samples=None, + ), + ), + ( + "button_color_active", + colorfield.fields.ColorField( + default="#FFFFFFFF", + image_field=None, + max_length=18, + samples=None, + ), + ), + ( + "set_name", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="workspace.forcedchoicebuttongroup", + to_field="group_name", + ), + ), ], ), migrations.AlterField( - model_name='namespace', - name='submission_method', - field=models.ForeignKey(blank=True, db_column='submission_method', null=True, on_delete=django.db.models.deletion.PROTECT, to='workspace.forcedchoicebuttongroup', to_field='group_name'), + model_name="namespace", + name="submission_method", + field=models.ForeignKey( + blank=True, + db_column="submission_method", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="workspace.forcedchoicebuttongroup", + to_field="group_name", + ), ), ] diff --git a/neuvue_project/workspace/migrations/0014_forcedchoicebutton_hotkey.py b/neuvue_project/workspace/migrations/0014_forcedchoicebutton_hotkey.py index 3c0cc431..b14ccf28 100644 --- a/neuvue_project/workspace/migrations/0014_forcedchoicebutton_hotkey.py +++ b/neuvue_project/workspace/migrations/0014_forcedchoicebutton_hotkey.py @@ -6,13 +6,30 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0013_auto_20220611_1200'), + ("workspace", "0013_auto_20220611_1200"), ] operations = [ migrations.AddField( - model_name='forcedchoicebutton', - name='hotkey', - field=models.CharField(blank=True, choices=[('c', 'C'), ('d', 'D'), ('j', 'J'), ('m', 'M'), ('q', 'Q'), ('r', 'R'), ('t', 'T'), ('v', 'V'), ('w', 'W'), ('y', 'Y'), ('z', 'Z')], max_length=300, null=True), + model_name="forcedchoicebutton", + name="hotkey", + field=models.CharField( + blank=True, + choices=[ + ("c", "C"), + ("d", "D"), + ("j", "J"), + ("m", "M"), + ("q", "Q"), + ("r", "R"), + ("t", "T"), + ("v", "V"), + ("w", "W"), + ("y", "Y"), + ("z", "Z"), + ], + max_length=300, + null=True, + ), ), ] diff --git a/neuvue_project/workspace/migrations/0015_namespace_track_selected_segments.py b/neuvue_project/workspace/migrations/0015_namespace_track_selected_segments.py index 0d7c2221..e7c283ea 100644 --- a/neuvue_project/workspace/migrations/0015_namespace_track_selected_segments.py +++ b/neuvue_project/workspace/migrations/0015_namespace_track_selected_segments.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0014_forcedchoicebutton_hotkey'), + ("workspace", "0014_forcedchoicebutton_hotkey"), ] operations = [ migrations.AddField( - model_name='namespace', - name='track_selected_segments', + model_name="namespace", + name="track_selected_segments", field=models.BooleanField(default=False), ), ] diff --git a/neuvue_project/workspace/migrations/0016_forcedchoicebuttongroup_number_of_selected_segments_expected.py b/neuvue_project/workspace/migrations/0016_forcedchoicebuttongroup_number_of_selected_segments_expected.py index 42c28466..8e3988c6 100644 --- a/neuvue_project/workspace/migrations/0016_forcedchoicebuttongroup_number_of_selected_segments_expected.py +++ b/neuvue_project/workspace/migrations/0016_forcedchoicebuttongroup_number_of_selected_segments_expected.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('workspace', '0015_namespace_track_selected_segments'), + ("workspace", "0015_namespace_track_selected_segments"), ] operations = [ migrations.AddField( - model_name='forcedchoicebuttongroup', - name='number_of_selected_segments_expected', + model_name="forcedchoicebuttongroup", + name="number_of_selected_segments_expected", field=models.IntegerField(blank=True, null=True), ), ] diff --git a/neuvue_project/workspace/models.py b/neuvue_project/workspace/models.py index 781cd812..a59a4c8a 100644 --- a/neuvue_project/workspace/models.py +++ b/neuvue_project/workspace/models.py @@ -4,88 +4,132 @@ from django.contrib.auth.models import User from django.contrib import admin from .validators import validate_submission_value + # Create your models here. + class NeuroglancerLinkType(models.TextChoices): """Enum for neuroglancer link types. Currently supported: - path -> expects a list of coordinates (metadata) and two soma points (points). + path -> expects a list of coordinates (metadata) and two soma points (points). Draws a path between all coordinates. - - point -> expects a list of coordinates (metadata), description (metadata), and - group (metadata). Needs atleast one seed point (points). Places dot points for + + point -> expects a list of coordinates (metadata), description (metadata), and + group (metadata). Needs atleast one seed point (points). Places dot points for all coordinates listed. pregenerated -> neuroglancer state already added to task. Useful for external outputs like CV or automated proofreading. """ - PATH = 'path', _('Path Layer') - POINT = 'point', _('Annotation Layer') - PREGENERATED = 'pregen', _('Pregenerated') + + PATH = "path", _("Path Layer") + POINT = "point", _("Annotation Layer") + PREGENERATED = "pregen", _("Pregenerated") + class ForcedChoiceButtonGroup(models.Model): group_name = models.CharField(max_length=100, unique=True, help_text="(snake case)") submit_task_button = models.BooleanField(default=True) number_of_selected_segments_expected = models.IntegerField(null=True, blank=True) + def __str__(self): return self.group_name + class Meta: verbose_name = "Forced Choice Button Group" verbose_name_plural = "Forced Choice Button Groups" + class HotkeyChoices(models.TextChoices): - C = 'c' - D = 'd' - J = 'j' - M = 'm' - Q = 'q' - R = 'r' - T = 't' - V = 'v' - W = 'w' - Y = 'y' - Z = 'z' + C = "c" + D = "d" + J = "j" + M = "m" + Q = "q" + R = "r" + T = "t" + V = "v" + W = "w" + Y = "y" + Z = "z" class ForcedChoiceButton(models.Model): - set_name = models.ForeignKey(ForcedChoiceButtonGroup, to_field='group_name', on_delete=models.CASCADE) + set_name = models.ForeignKey( + ForcedChoiceButtonGroup, to_field="group_name", on_delete=models.CASCADE + ) display_name = models.CharField(max_length=100) - submission_value = models.CharField(max_length=100, validators=[validate_submission_value]) - button_color = ColorField(format='hexa') - button_color_active = ColorField(format='hexa') - hotkey = models.CharField(max_length=300, choices=HotkeyChoices.choices, blank=True, null=True) + submission_value = models.CharField( + max_length=100, validators=[validate_submission_value] + ) + button_color = ColorField(format="hexa") + button_color_active = ColorField(format="hexa") + hotkey = models.CharField( + max_length=300, choices=HotkeyChoices.choices, blank=True, null=True + ) + def __str__(self): return str(self.set_name) + class PcgChoices(models.TextChoices): - MINNIE = 'https://minnie.microns-daf.com/segmentation/table/minnie3_v1', _('Minnie65') - PINKY = 'https://minnie.microns-daf.com/segmentation/table/pinky_v2_microns_sandbox', _('Pinky') + MINNIE = "https://minnie.microns-daf.com/segmentation/table/minnie3_v1", _( + "Minnie65" + ) + PINKY = ( + "https://minnie.microns-daf.com/segmentation/table/pinky_v2_microns_sandbox", + _("Pinky"), + ) + class ImageChoices(models.TextChoices): - MINNIE = 'https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em', _('Minnie65') - PINKY = 'gs://microns_public_datasets/pinky100_v0/son_of_alignment_v15_rechunked', _('Pinky') + MINNIE = ( + "https://bossdb-open-data.s3.amazonaws.com/iarpa_microns/minnie/minnie65/em", + _("Minnie65"), + ) + PINKY = ( + "gs://microns_public_datasets/pinky100_v0/son_of_alignment_v15_rechunked", + _("Pinky"), + ) class PushToChoices(models.TextChoices): - NULL = 'Queue Tasks Not Allowed' - NOVICE = 'unassigned_novice' - INTERMEDIATE = 'unassigned_intermediate' - EXPERT = 'unassigned_expert' + NULL = "Queue Tasks Not Allowed" + NOVICE = "unassigned_novice" + INTERMEDIATE = "unassigned_intermediate" + EXPERT = "unassigned_expert" + class PullFromChoices(models.TextChoices): - NULL = 'Reassign Tasks Not Allowed' - NOVICE = 'unassigned_novice' - INTERMEDIATE = 'unassigned_intermediate' - EXPERT = 'unassigned_expert' + NULL = "Reassign Tasks Not Allowed" + NOVICE = "unassigned_novice" + INTERMEDIATE = "unassigned_intermediate" + EXPERT = "unassigned_expert" + class Namespace(models.Model): namespace_enabled = models.BooleanField(default=True) namespace = models.CharField(max_length=50, primary_key=True) display_name = models.CharField(max_length=100) - ng_link_type = models.CharField(max_length=50, choices = NeuroglancerLinkType.choices, default= NeuroglancerLinkType.PREGENERATED) - submission_method = models.ForeignKey(ForcedChoiceButtonGroup, on_delete=models.PROTECT, blank=True, null=True, to_field="group_name", db_column="submission_method") - pcg_source = models.CharField(max_length=300, choices=PcgChoices.choices, default=PcgChoices.MINNIE) - img_source = models.CharField(max_length=300, choices=ImageChoices.choices, default=ImageChoices.MINNIE) + ng_link_type = models.CharField( + max_length=50, + choices=NeuroglancerLinkType.choices, + default=NeuroglancerLinkType.PREGENERATED, + ) + submission_method = models.ForeignKey( + ForcedChoiceButtonGroup, + on_delete=models.PROTECT, + blank=True, + null=True, + to_field="group_name", + db_column="submission_method", + ) + pcg_source = models.CharField( + max_length=300, choices=PcgChoices.choices, default=PcgChoices.MINNIE + ) + img_source = models.CharField( + max_length=300, choices=ImageChoices.choices, default=ImageChoices.MINNIE + ) track_operation_ids = models.BooleanField(default=True) refresh_selected_root_ids = models.BooleanField(default=False) number_of_tasks_users_can_self_assign = models.IntegerField(default=10) @@ -93,25 +137,39 @@ class Namespace(models.Model): track_selected_segments = models.BooleanField(default=False) """Pull From Push To Novice""" - novice_pull_from = models.CharField(max_length=50, choices=PullFromChoices.choices, default=PullFromChoices.NULL) - novice_push_to = models.CharField(max_length=50, choices=PushToChoices.choices, default=PushToChoices.NULL) + novice_pull_from = models.CharField( + max_length=50, choices=PullFromChoices.choices, default=PullFromChoices.NULL + ) + novice_push_to = models.CharField( + max_length=50, choices=PushToChoices.choices, default=PushToChoices.NULL + ) """Pull From Push To Intermediate""" - intermediate_pull_from = models.CharField(max_length=50, choices=PullFromChoices.choices, default=PullFromChoices.NULL) - intermediate_push_to = models.CharField(max_length=50, choices=PushToChoices.choices, default=PushToChoices.NULL) + intermediate_pull_from = models.CharField( + max_length=50, choices=PullFromChoices.choices, default=PullFromChoices.NULL + ) + intermediate_push_to = models.CharField( + max_length=50, choices=PushToChoices.choices, default=PushToChoices.NULL + ) """Pull From Push To Expert""" - expert_pull_from = models.CharField(max_length=50, choices=PullFromChoices.choices, default=PullFromChoices.NULL) - expert_push_to = models.CharField(max_length=50, choices=PushToChoices.choices, default=PushToChoices.NULL) - + expert_pull_from = models.CharField( + max_length=50, choices=PullFromChoices.choices, default=PullFromChoices.NULL + ) + expert_push_to = models.CharField( + max_length=50, choices=PushToChoices.choices, default=PushToChoices.NULL + ) + def __str__(self): """String for representing the MyModelName object (in Admin site etc.).""" return self.namespace + class UserProfile(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE) - intermediate_namespaces = models.ManyToManyField(Namespace, related_name='intermediate_namespaces', blank=True) - expert_namespaces = models.ManyToManyField(Namespace, related_name='expert_namespaces', blank=True) - - - + intermediate_namespaces = models.ManyToManyField( + Namespace, related_name="intermediate_namespaces", blank=True + ) + expert_namespaces = models.ManyToManyField( + Namespace, related_name="expert_namespaces", blank=True + ) diff --git a/neuvue_project/workspace/neuroglancer.py b/neuvue_project/workspace/neuroglancer.py index ae7c5ce3..758a0ea7 100644 --- a/neuvue_project/workspace/neuroglancer.py +++ b/neuvue_project/workspace/neuroglancer.py @@ -16,14 +16,14 @@ import numpy as np from caveclient import CAVEclient from nglui.statebuilder import ( - ImageLayerConfig, - SegmentationLayerConfig, - AnnotationLayerConfig, + ImageLayerConfig, + SegmentationLayerConfig, + AnnotationLayerConfig, LineMapper, PointMapper, StateBuilder, - ChainedStateBuilder - ) + ChainedStateBuilder, +) from .models import Namespace, NeuroglancerLinkType, PcgChoices, ImageChoices @@ -31,33 +31,34 @@ logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -Config = apps.get_model('preferences', 'Config') +Config = apps.get_model("preferences", "Config") def get_df_from_static(cave_client, table_name): - @backoff.on_exception(backoff.expo, Exception, max_tries=3) def query_new_table(table_name): - logging.info(f'Downloading new table for {table_name}.') + logging.info(f"Downloading new table for {table_name}.") df = cave_client.materialize.query_table(table_name) - fn = str(round(time.time()))+ '_' + table_name+'.pkl' + fn = str(round(time.time())) + "_" + table_name + ".pkl" df.to_pickle(os.path.join(settings.CACHED_TABLES_PATH, fn)) return df - + try: if not os.path.exists(settings.CACHED_TABLES_PATH): os.makedirs(settings.CACHED_TABLES_PATH) cached_tables = glob.glob( - os.path.join(settings.CACHED_TABLES_PATH, '*_'+table_name+'.pkl') + os.path.join(settings.CACHED_TABLES_PATH, "*_" + table_name + ".pkl") ) # filter to table of interest if len(cached_tables): file_path = cached_tables[0] file_name = os.path.split(file_path)[1] - file_date = int(file_name.split('_')[0]) - if (datetime.fromtimestamp(file_date) - datetime.fromtimestamp(time.time())) < timedelta(days=settings.DAYS_UNTIL_EXPIRED): - logging.info(f'Using cached table for {table_name}.') + file_date = int(file_name.split("_")[0]) + if ( + datetime.fromtimestamp(file_date) - datetime.fromtimestamp(time.time()) + ) < timedelta(days=settings.DAYS_UNTIL_EXPIRED): + logging.info(f"Using cached table for {table_name}.") df = pd.read_pickle(file_path) else: os.remove(file_path) @@ -65,12 +66,13 @@ def query_new_table(table_name): else: df = query_new_table(table_name) return df - except Exception: - logger.error('Resource table cannot be queried.') - raise Exception(f'Table {table_name} unavailable.') + except Exception: + logger.error("Resource table cannot be queried.") + raise Exception(f"Table {table_name} unavailable.") + def create_base_state(seg_ids, coordinate, namespace=None): - """Generates a base state containing imagery and segmentation layers. + """Generates a base state containing imagery and segmentation layers. Args: seg_ids (list): seg_ids to select in the view @@ -79,51 +81,49 @@ def create_base_state(seg_ids, coordinate, namespace=None): Returns: StateBuilder: Base State """ - + # Create ImageLayerConfig if namespace: - img_source = "precomputed://" + Namespace.objects.get(namespace = namespace).img_source + img_source = ( + "precomputed://" + Namespace.objects.get(namespace=namespace).img_source + ) else: img_source = "precomputed://" + ImageChoices.MINNIE - try: - black = settings.DATASET_VIEWER_OPTIONS[img_source]['contrast']["black"] - white = settings.DATASET_VIEWER_OPTIONS[img_source]['contrast']["white"] + try: + black = settings.DATASET_VIEWER_OPTIONS[img_source]["contrast"]["black"] + white = settings.DATASET_VIEWER_OPTIONS[img_source]["contrast"]["white"] except KeyError: black = 0 white = 1 - + img_layer = ImageLayerConfig( - name='em', - source=img_source, - contrast_controls=True, - black=black, - white=white - ) - + name="em", source=img_source, contrast_controls=True, black=black, white=white + ) + # Create SegmentationLayerConfig if namespace: - seg_source = "graphene://" + Namespace.objects.get(namespace = namespace).pcg_source + seg_source = ( + "graphene://" + Namespace.objects.get(namespace=namespace).pcg_source + ) else: seg_source = "graphene://" + PcgChoices.MINNIE - - segmentation_view_options = { - 'alpha_selected': 0.6, - 'alpha_3d': 0.3 - } + + segmentation_view_options = {"alpha_selected": 0.6, "alpha_3d": 0.3} seg_layer = SegmentationLayerConfig( - name='seg', - source=seg_source, + name="seg", + source=seg_source, fixed_ids=seg_ids, - view_kws=segmentation_view_options - ) + view_kws=segmentation_view_options, + ) - view_options = {'position': coordinate, 'zoom_image': 20} + view_options = {"position": coordinate, "zoom_image": 20} return StateBuilder(layers=[img_layer, seg_layer], view_kws=view_options) + def generate_path_df(points): - """Generates the point A to point B dataframe for all points. Points are + """Generates the point A to point B dataframe for all points. Points are assumed to be in sequential order and all part of the same group. Args: @@ -134,7 +134,7 @@ def generate_path_df(points): """ point_column_a = points[:-1].tolist() point_column_b = points[1:].tolist() - + group = np.ones(len(point_column_a)).tolist() return pd.DataFrame( { @@ -144,8 +144,9 @@ def generate_path_df(points): } ) + def generate_point_df(points, description=None, group=None): - """Generates the point A dataframe for all points. Points are + """Generates the point A dataframe for all points. Points are assumed to be all part of the same group. Args: @@ -170,7 +171,7 @@ def generate_point_df(points, description=None, group=None): { "point_column_a": point_column_a, "group": group, - "description": description + "description": description, } ) else: @@ -183,33 +184,39 @@ def create_path_state(): Returns: StateBuilder: Annotation State """ - path = AnnotationLayerConfig("selected_paths", active=False, - mapping_rules=LineMapper("point_column_a", "point_column_b", group_column="group"), + path = AnnotationLayerConfig( + "selected_paths", + active=False, + mapping_rules=LineMapper( + "point_column_a", "point_column_b", group_column="group" + ), ) anno = AnnotationLayerConfig("merge_point") return StateBuilder(layers=[path, anno], resolution=settings.VOXEL_RESOLUTION) -def create_point_state(name='annotations', group=None, description=None, color=None): +def create_point_state(name="annotations", group=None, description=None, color=None): """Create the annotation state for points. Dont use linemapper, just creates a neuroglancer link that is just Points nglui statebuilder Returns: StateBuilder: Annotation State """ - anno = AnnotationLayerConfig(name, + anno = AnnotationLayerConfig( + name, mapping_rules=PointMapper( - "point_column_a", - group_column=group, + "point_column_a", + group_column=group, description_column=description, - set_position=False), - color=color + set_position=False, + ), + color=color, ) return StateBuilder(layers=[anno], resolution=settings.VOXEL_RESOLUTION) -def construct_proofreading_state(task_df, points, return_as='json'): +def construct_proofreading_state(task_df, points, return_as="json"): """Generates a Neuroglancer URL with the path/annotation information preloaded. Args: @@ -219,115 +226,122 @@ def construct_proofreading_state(task_df, points, return_as='json'): Returns: string: Neuroglancer URL """ - # TODO: Automatically iterate through Namespaces and map them to the - # appropriate Neuroglancer functions. - seg_ids = [task_df['seg_id']] - base_state = create_base_state(seg_ids, points[0], task_df['namespace']) + # TODO: Automatically iterate through Namespaces and map them to the + # appropriate Neuroglancer functions. + seg_ids = [task_df["seg_id"]] + base_state = create_base_state(seg_ids, points[0], task_df["namespace"]) # Get any annotation coordinates. Append original points. - coordinates = task_df['metadata'].get('coordinates', []) + coordinates = task_df["metadata"].get("coordinates", []) - # Create a list of dataframes used for state creation. Since first state is - # the base layer, the first element is None. + # Create a list of dataframes used for state creation. Since first state is + # the base layer, the first element is None. data_list = [None] - ng_type = Namespace.objects.get(namespace = task_df['namespace']).ng_link_type + ng_type = Namespace.objects.get(namespace=task_df["namespace"]).ng_link_type if ng_type == NeuroglancerLinkType.PATH: if points: # Append start and end soma coordinates - coordinates.insert(0 ,points[0]) + coordinates.insert(0, points[0]) coordinates.append(points[-1]) coordinates = np.array(coordinates) - - data_list.append( generate_path_df(coordinates)) + + data_list.append(generate_path_df(coordinates)) path_state = create_path_state() chained_state = ChainedStateBuilder([base_state, path_state]) - - elif ng_type == NeuroglancerLinkType.POINT: + + elif ng_type == NeuroglancerLinkType.POINT: # Get grouping and annotation descriptions, if they exist coordinates = np.array(coordinates) - group = task_df['metadata'].get('group') - description = task_df['metadata'].get('description') - data_list.append( generate_point_df(coordinates, description=description, group=group)) + group = task_df["metadata"].get("group") + description = task_df["metadata"].get("description") + data_list.append( + generate_point_df(coordinates, description=description, group=group) + ) point_state = create_point_state(bool(description)) chained_state = ChainedStateBuilder([base_state, point_state]) - elif ng_type == NeuroglancerLinkType.PREGENERATED and task_df.get('ng_state'): - if return_as == 'json': - return task_df['ng_state'] - elif return_as == 'url': - return construct_url_from_existing(json.dumps(task_df['ng_state'])) - + elif ng_type == NeuroglancerLinkType.PREGENERATED and task_df.get("ng_state"): + if return_as == "json": + return task_df["ng_state"] + elif return_as == "url": + return construct_url_from_existing(json.dumps(task_df["ng_state"])) + return chained_state.render_state( - data_list, return_as=return_as, url_prefix=settings.NG_CLIENT - ) - + data_list, return_as=return_as, url_prefix=settings.NG_CLIENT + ) + + def construct_url_from_existing(state: str): - return settings.NG_CLIENT + '/#!' + state + return settings.NG_CLIENT + "/#!" + state + @backoff.on_exception(backoff.expo, Exception, max_tries=3) -def get_from_state_server(url: str): +def get_from_state_server(url: str): """Gets JSON state string from state server Args: url (str): json state server link Returns: - (str): JSON String + (str): JSON String """ headers = { - 'content-type': 'application/json', - 'Authorization': f"Bearer {os.environ['CAVECLIENT_TOKEN']}" + "content-type": "application/json", + "Authorization": f"Bearer {os.environ['CAVECLIENT_TOKEN']}", } resp = requests.get(url, headers=headers) if resp.status_code != 200: raise Exception("GET Unsuccessful") - + # TODO: Make sure its JSON String return resp.text + @backoff.on_exception(backoff.expo, Exception, max_tries=3) -def post_to_state_server(state: str): +def post_to_state_server(state: str): """Posts JSON string to state server Args: state (str): NG State string - + Returns: str: url string """ # Get the authorization token from caveclient headers = { - 'content-type': 'application/json', - 'Authorization': f"Bearer {os.environ['CAVECLIENT_TOKEN']}" + "content-type": "application/json", + "Authorization": f"Bearer {os.environ['CAVECLIENT_TOKEN']}", } - # Post! + # Post! resp = requests.post(settings.JSON_STATE_SERVER, data=state, headers=headers) if resp.status_code != 200: raise Exception("POST Unsuccessful") - + # Response will contain the URL for the state you just posted return str(resp.json()) + def get_from_json(raw_state: str): """Get Neuroglancer state from JSON string #TODO: Apply config settings here eventually. - + Args: raw state (str): neuroglancer string Returns: str: validated neuroglancer state """ state_obj = json.loads(raw_state) - if state_obj.get('value'): - return json.dumps(state_obj['value']) + if state_obj.get("value"): + return json.dumps(state_obj["value"]) else: return raw_state + @backoff.on_exception(backoff.expo, Exception, max_tries=3) -def _get_lineage_graph(root_id:str, cave_client): +def _get_lineage_graph(root_id: str, cave_client): """Get a lineage graph with exponential backoff. Args: @@ -342,11 +356,16 @@ def _get_lineage_graph(root_id:str, cave_client): """ try: - return cave_client.chunkedgraph.get_lineage_graph([root_id], timestamp_past=datetime(year=2021, month=11, day=1), as_nx_graph=True) + return cave_client.chunkedgraph.get_lineage_graph( + [root_id], + timestamp_past=datetime(year=2021, month=11, day=1), + as_nx_graph=True, + ) except Exception as e: logging.error(e) raise e + @backoff.on_exception(backoff.expo, Exception, max_tries=3) def _get_soma_center(root_ids: List, cave_client): """Get the first soma center of a list of root IDs with exponential backoff. @@ -361,44 +380,65 @@ def _get_soma_center(root_ids: List, cave_client): array: array for the position of the soma """ try: - soma_df = cave_client.materialize.query_table(settings.NEURON_TABLE, filter_in_dict={ - 'pt_root_id': root_ids[:3] - }) + soma_df = cave_client.materialize.query_table( + settings.NEURON_TABLE, filter_in_dict={"pt_root_id": root_ids[:3]} + ) if not len(soma_df): - soma_df = cave_client.materialize.query_table(settings.NEURON_TABLE, filter_in_dict={ - 'pt_root_id': root_ids - }) + soma_df = cave_client.materialize.query_table( + settings.NEURON_TABLE, filter_in_dict={"pt_root_id": root_ids} + ) if len(soma_df) > 3: soma_df = soma_df.head(3) - pt_position = soma_df.iloc[0]['pt_position'] - present_root_ids = list(soma_df['pt_root_id']) + pt_position = soma_df.iloc[0]["pt_position"] + present_root_ids = list(soma_df["pt_root_id"]) except IndexError as e: logging.error(e) - raise Exception('Unable to find Soma Center') + raise Exception("Unable to find Soma Center") except Exception as e: logging.error(e) raise Exception(e) return pt_position, present_root_ids + def _get_nx_graph_image(nx_graph): def networkx_to_graphViz(nx_graph): import graphviz import networkx as nx - gv_graph = graphviz.Digraph('lineage',format = 'svg',graph_attr={'size':'6,{}','ratio':"compress",'ranksep':'0.5'},node_attr={'fontsize':'18','fontname':"Arial"}) # 'size':'6,3', - timestamps = nx.get_node_attributes(nx_graph, "timestamp") # dictionary of all timestamps in the graph, key: node - operation_ids = nx.get_node_attributes(nx_graph, "operation_id") # dictionary of all operation ids in the graph, key: node + + gv_graph = graphviz.Digraph( + "lineage", + format="svg", + graph_attr={"size": "6,{}", "ratio": "compress", "ranksep": "0.5"}, + node_attr={"fontsize": "18", "fontname": "Arial"}, + ) # 'size':'6,3', + timestamps = nx.get_node_attributes( + nx_graph, "timestamp" + ) # dictionary of all timestamps in the graph, key: node + operation_ids = nx.get_node_attributes( + nx_graph, "operation_id" + ) # dictionary of all operation ids in the graph, key: node for node in nx_graph.nodes(): label_str = str(node) - label_str += '\n' + datetime.fromtimestamp(timestamps.get(node)).strftime('%Y-%m-%d') if timestamps.get(node) else '' # add timestamp if it exists - label_str += '\n id: ' + str(operation_ids.get(node)) if operation_ids.get(node) else '' # add operation id if it exists - gv_graph.node(str(node),label=label_str) + label_str += ( + "\n" + datetime.fromtimestamp(timestamps.get(node)).strftime("%Y-%m-%d") + if timestamps.get(node) + else "" + ) # add timestamp if it exists + label_str += ( + "\n id: " + str(operation_ids.get(node)) + if operation_ids.get(node) + else "" + ) # add operation id if it exists + gv_graph.node(str(node), label=label_str) for edge0, edge1 in nx_graph.edges(): gv_graph.edge(str(edge0), str(edge1)) - gv_graph = gv_graph.unflatten(stagger=10) - return gv_graph.pipe(encoding='utf-8') + gv_graph = gv_graph.unflatten(stagger=10) + return gv_graph.pipe(encoding="utf-8") + return networkx_to_graphViz(nx_graph) -def construct_lineage_state_and_graph(root_id:str): + +def construct_lineage_state_and_graph(root_id: str): """Construct state for the lineage viewer. Args: @@ -408,7 +448,9 @@ def construct_lineage_state_and_graph(root_id:str): string: json-formatted state """ root_id = root_id.strip() - cave_client = CAVEclient('minnie65_phase3_v1', auth_token=os.environ['CAVECLIENT_TOKEN']) + cave_client = CAVEclient( + "minnie65_phase3_v1", auth_token=os.environ["CAVECLIENT_TOKEN"] + ) # Lineage graph gives you the nodes and edges of a root IDs history lineage_graph = _get_lineage_graph(root_id, cave_client) @@ -419,34 +461,37 @@ def construct_lineage_state_and_graph(root_id:str): # Ensure original root ID is in list of shown IDs root_ids.add(root_id) root_ids = list(root_ids) - + position, root_ids_with_center = _get_soma_center(root_ids, cave_client) base_state = create_base_state(root_ids_with_center, position) # For the rest of the IDs, we can add them to the seg layer as unselected. - base_state_dict = base_state.render_state(return_as='dict') - base_state_dict['layout'] = '3d' + base_state_dict = base_state.render_state(return_as="dict") + base_state_dict["layout"] = "3d" base_state_dict["selectedLayer"] = {"layer": "seg", "visible": True} - for layer in base_state_dict['layers']: - if layer['name'] == 'seg': - selected_segments = layer['segments'] - layer['hiddenSegments'] = [root_id for root_id in root_ids if root_id not in selected_segments] + for layer in base_state_dict["layers"]: + if layer["name"] == "seg": + selected_segments = layer["segments"] + layer["hiddenSegments"] = [ + root_id for root_id in root_ids if root_id not in selected_segments + ] return json.dumps(base_state_dict), graph_image -def apply_state_config(state:str, username:str): + +def apply_state_config(state: str, username: str): cdict = json.loads(state) - cdict['jsonStateServer'] = settings.JSON_STATE_SERVER - #make ng state preferences changes, json string to dict + cdict["jsonStateServer"] = settings.JSON_STATE_SERVER + # make ng state preferences changes, json string to dict try: - config = Config.objects.filter(user=username).order_by('-id')[0] + config = Config.objects.filter(user=username).order_by("-id")[0] except Exception as e: - logging.error(e) + logging.error(e) return json.dumps(cdict) if not config.enabled: return json.dumps(cdict) - + annotation_color_palette = config.annotation_color_palette mesh_color_palette = config.mesh_color_palette alpha_selected = config.alpha_selected @@ -463,66 +508,203 @@ def apply_state_config(state:str, username:str): if config.layout_switch: cdict["layout"] = str(layout) if config.gpu_limit_switch: - cdict["gpuMemoryLimit"] = int(float(gpu_limit) * 1E9) + cdict["gpuMemoryLimit"] = int(float(gpu_limit) * 1e9) if config.sys_limit_switch: - cdict["systemMemoryLimit"] = int(float(sys_limit) * 1E9) + cdict["systemMemoryLimit"] = int(float(sys_limit) * 1e9) if config.chunk_requests_switch: cdict["concurrentDownloads"] = int(chunk_requests) if config.zoom_level_switch: cdict["navigation"]["zoomFactor"] = int(zoom_level) if config.enable_sound_switch: - cdict['enableSound'] == enable_sound - + cdict["enableSound"] == enable_sound + # create color palette dictionary - annotation_color_palette_dict = {'palette1' : ['#FFADAD', '#FFD6A5', '#FDFFB6', '#CAFFBF', '#9BF6FF', '#A0C4FF', '#BDB2FF', '#FFC6FF', '#FFFFFC'], - 'palette2' : ['#F72585', '#B5179E', '#7209B7', '#560BAD', '#480CA8', '#3A0CA3', '#3F37C9', '#4361EE', '#4895EF', '#4CC9F0'], - 'palette3' : ['#7400B8', '#6930C3', '#5E60CE', '#5390D9', '#4EA8DE', '#48BFE3', '#56CFE1', '#64DFDF', '#72EFDD', '#80FFDB'], - 'palette4' : ['#F94144', '#F3722C', '#F8961E', '#F9844A', '#F9C74F', '#90BE6D', '#43AA8B', '#4D908E', '#577590', '#277DA1'], - 'palette5' : ['#005F73', '#0A9396', '#94D2BD', '#E9D8A6', '#EE9B00', '#CA6702', '#BB3E03', '#AE2012', '#9B2226'], - 'palette6' : ['#011A51', '#1957DB', '#487BEA', '#7EA3F1', '#C8D7F9', '#B83700', '#F06C00', '#FAB129', '#FBC55F', '#FDE9C3']} + annotation_color_palette_dict = { + "palette1": [ + "#FFADAD", + "#FFD6A5", + "#FDFFB6", + "#CAFFBF", + "#9BF6FF", + "#A0C4FF", + "#BDB2FF", + "#FFC6FF", + "#FFFFFC", + ], + "palette2": [ + "#F72585", + "#B5179E", + "#7209B7", + "#560BAD", + "#480CA8", + "#3A0CA3", + "#3F37C9", + "#4361EE", + "#4895EF", + "#4CC9F0", + ], + "palette3": [ + "#7400B8", + "#6930C3", + "#5E60CE", + "#5390D9", + "#4EA8DE", + "#48BFE3", + "#56CFE1", + "#64DFDF", + "#72EFDD", + "#80FFDB", + ], + "palette4": [ + "#F94144", + "#F3722C", + "#F8961E", + "#F9844A", + "#F9C74F", + "#90BE6D", + "#43AA8B", + "#4D908E", + "#577590", + "#277DA1", + ], + "palette5": [ + "#005F73", + "#0A9396", + "#94D2BD", + "#E9D8A6", + "#EE9B00", + "#CA6702", + "#BB3E03", + "#AE2012", + "#9B2226", + ], + "palette6": [ + "#011A51", + "#1957DB", + "#487BEA", + "#7EA3F1", + "#C8D7F9", + "#B83700", + "#F06C00", + "#FAB129", + "#FBC55F", + "#FDE9C3", + ], + } # create mesh color palette dictionary - mesh_color_palette_dict = {'palette1' : ['#FFADAD', '#FFD6A5', '#FDFFB6', '#CAFFBF', '#9BF6FF', '#A0C4FF', '#BDB2FF', '#FFC6FF', '#FFFFFC'], - 'palette2' : ['#F72585', '#B5179E', '#7209B7', '#560BAD', '#480CA8', '#3A0CA3', '#3F37C9', '#4361EE', '#4895EF', '#4CC9F0'], - 'palette3' : ['#7400B8', '#6930C3', '#5E60CE', '#5390D9', '#4EA8DE', '#48BFE3', '#56CFE1', '#64DFDF', '#72EFDD', '#80FFDB'], - 'palette4' : ['#F94144', '#F3722C', '#F8961E', '#F9844A', '#F9C74F', '#90BE6D', '#43AA8B', '#4D908E', '#577590', '#277DA1'], - 'palette5' : ['#005F73', '#0A9396', '#94D2BD', '#E9D8A6', '#EE9B00', '#CA6702', '#BB3E03', '#AE2012', '#9B2226'], - 'palette6' : ['#011A51', '#1957DB', '#487BEA', '#7EA3F1', '#C8D7F9', '#B83700', '#F06C00', '#FAB129', '#FBC55F', '#FDE9C3']} - - + mesh_color_palette_dict = { + "palette1": [ + "#FFADAD", + "#FFD6A5", + "#FDFFB6", + "#CAFFBF", + "#9BF6FF", + "#A0C4FF", + "#BDB2FF", + "#FFC6FF", + "#FFFFFC", + ], + "palette2": [ + "#F72585", + "#B5179E", + "#7209B7", + "#560BAD", + "#480CA8", + "#3A0CA3", + "#3F37C9", + "#4361EE", + "#4895EF", + "#4CC9F0", + ], + "palette3": [ + "#7400B8", + "#6930C3", + "#5E60CE", + "#5390D9", + "#4EA8DE", + "#48BFE3", + "#56CFE1", + "#64DFDF", + "#72EFDD", + "#80FFDB", + ], + "palette4": [ + "#F94144", + "#F3722C", + "#F8961E", + "#F9844A", + "#F9C74F", + "#90BE6D", + "#43AA8B", + "#4D908E", + "#577590", + "#277DA1", + ], + "palette5": [ + "#005F73", + "#0A9396", + "#94D2BD", + "#E9D8A6", + "#EE9B00", + "#CA6702", + "#BB3E03", + "#AE2012", + "#9B2226", + ], + "palette6": [ + "#011A51", + "#1957DB", + "#487BEA", + "#7EA3F1", + "#C8D7F9", + "#B83700", + "#F06C00", + "#FAB129", + "#FBC55F", + "#FDE9C3", + ], + } # generate random color def getRandomHexColor(): - random_number = random.randint(0,16777215) + random_number = random.randint(0, 16777215) hex_number = str(hex(random_number)) - hex_color = '#'+ hex_number[2:].upper() + hex_color = "#" + hex_number[2:].upper() return hex_color - + annotation_layer_count = 0 - for layer in cdict['layers']: + for layer in cdict["layers"]: # handle alpha - if 'segmentation' in layer.get('type', '') and config.alpha_selected_switch: - layer['selectedAlpha'] = float(alpha_selected) - - if 'segmentation' in layer.get('type', '') and config.alpha_3d_switch: - layer['objectAlpha'] = float(alpha_3d) - + if "segmentation" in layer.get("type", "") and config.alpha_selected_switch: + layer["selectedAlpha"] = float(alpha_selected) + + if "segmentation" in layer.get("type", "") and config.alpha_3d_switch: + layer["objectAlpha"] = float(alpha_3d) + # handle annotation layer colors - if layer.get('type', '') == 'annotation' and config.annotation_color_palette_switch: - annotation_color_palette_list = annotation_color_palette_dict[annotation_color_palette] - - # set annotation color + if ( + layer.get("type", "") == "annotation" + and config.annotation_color_palette_switch + ): + annotation_color_palette_list = annotation_color_palette_dict[ + annotation_color_palette + ] + + # set annotation color # get a random color, if the layer is within the number of available color palette colors, grab next color paltte color annotation_color = getRandomHexColor() # set this to random color if annotation_layer_count < len(annotation_color_palette_list): - annotation_color = annotation_color_palette_list[annotation_layer_count%len(annotation_color_palette_list)] - layer['annotationColor'] = str(annotation_color) + annotation_color = annotation_color_palette_list[ + annotation_layer_count % len(annotation_color_palette_list) + ] + layer["annotationColor"] = str(annotation_color) annotation_layer_count += 1 - + # handle mesh layer colors - if 'segmentation' in layer.get('type', '') and config.mesh_color_palette_switch: + if "segmentation" in layer.get("type", "") and config.mesh_color_palette_switch: mesh_color_palette_list = mesh_color_palette_dict[mesh_color_palette] - segments = layer.get('segments', []) + segments = layer.get("segments", []) segmentColors = {} # populate segment colors dictionary @@ -531,17 +713,18 @@ def getRandomHexColor(): for segment in segments: mesh_color = getRandomHexColor() if mesh_layer_count < len(mesh_color_palette_list): - mesh_color = mesh_color_palette_list[mesh_layer_count%len(mesh_color_palette_list)] + mesh_color = mesh_color_palette_list[ + mesh_layer_count % len(mesh_color_palette_list) + ] segmentColors[segment] = mesh_color mesh_layer_count += 1 - - # add segment colors dictionary to layer state - layer['segmentColors'] = segmentColors - + # add segment colors dictionary to layer state + layer["segmentColors"] = segmentColors return json.dumps(cdict) + def construct_synapse_state(root_ids: List, flags: dict = None): """Construct state for the synapse viewer. @@ -557,22 +740,30 @@ def construct_synapse_state(root_ids: List, flags: dict = None): string: json-formatted state dict: synapse stats """ - cave_client = CAVEclient('minnie65_phase3_v1', auth_token=os.environ['CAVECLIENT_TOKEN']) + cave_client = CAVEclient( + "minnie65_phase3_v1", auth_token=os.environ["CAVECLIENT_TOKEN"] + ) int_root_ids = [int(x) for x in root_ids] # Error checking - if flags['pre_synapses'] != 'True' and flags['post_synapses'] != 'True': - raise Exception("You must pick at least one of the following: Pre-Synapses, Post Synapses") + if flags["pre_synapses"] != "True" and flags["post_synapses"] != "True": + raise Exception( + "You must pick at least one of the following: Pre-Synapses, Post Synapses" + ) # Pre-synapses - if flags['pre_synapses'] == 'True': - if flags['timestamp'] != 'None': + if flags["pre_synapses"] == "True": + if flags["timestamp"] != "None": try: pre_synapses = cave_client.materialize.query_table( "synapses_pni_2", filter_in_dict={"pre_pt_root_id": int_root_ids}, - select_columns=['ctr_pt_position', 'pre_pt_root_id', 'post_pt_root_id'], - timestamp=datetime.strptime(flags['timestamp'], '%Y-%m-%d') + select_columns=[ + "ctr_pt_position", + "pre_pt_root_id", + "post_pt_root_id", + ], + timestamp=datetime.strptime(flags["timestamp"], "%Y-%m-%d"), ) except Exception as index: raise Exception(f"Root ID {index} not found for this timestamp") @@ -580,25 +771,31 @@ def construct_synapse_state(root_ids: List, flags: dict = None): pre_synapses = cave_client.materialize.query_table( "synapses_pni_2", filter_in_dict={"pre_pt_root_id": int_root_ids}, - select_columns=['ctr_pt_position', 'pre_pt_root_id', 'post_pt_root_id'], + select_columns=["ctr_pt_position", "pre_pt_root_id", "post_pt_root_id"], ) - pre_synapses['ctr_pt_position'] = pre_synapses['ctr_pt_position'].apply(lambda x: x.tolist()) - pre_synapses['pre_pt_root_id'] = pre_synapses['pre_pt_root_id'].astype(str) - pre_synapses['post_pt_root_id'] = pre_synapses['post_pt_root_id'].astype(str) + pre_synapses["ctr_pt_position"] = pre_synapses["ctr_pt_position"].apply( + lambda x: x.tolist() + ) + pre_synapses["pre_pt_root_id"] = pre_synapses["pre_pt_root_id"].astype(str) + pre_synapses["post_pt_root_id"] = pre_synapses["post_pt_root_id"].astype(str) - if len(pre_synapses['ctr_pt_position']) == 0: - raise Exception('No pre-synapses found for root ids.') - position = np.random.choice(pre_synapses['ctr_pt_position'].to_numpy()) + if len(pre_synapses["ctr_pt_position"]) == 0: + raise Exception("No pre-synapses found for root ids.") + position = np.random.choice(pre_synapses["ctr_pt_position"].to_numpy()) # Post-synapses - if flags['post_synapses'] == 'True': - if flags['timestamp'] != 'None': + if flags["post_synapses"] == "True": + if flags["timestamp"] != "None": try: post_synapses = cave_client.materialize.query_table( "synapses_pni_2", filter_in_dict={"post_pt_root_id": int_root_ids}, - select_columns=['ctr_pt_position', 'post_pt_root_id', 'pre_pt_root_id'], - timestamp=datetime.strptime(flags['timestamp'], '%Y-%m-%d') + select_columns=[ + "ctr_pt_position", + "post_pt_root_id", + "pre_pt_root_id", + ], + timestamp=datetime.strptime(flags["timestamp"], "%Y-%m-%d"), ) except Exception as index: raise Exception(f"Root ID {index} not found for this timestamp") @@ -606,15 +803,17 @@ def construct_synapse_state(root_ids: List, flags: dict = None): post_synapses = cave_client.materialize.query_table( "synapses_pni_2", filter_in_dict={"post_pt_root_id": int_root_ids}, - select_columns=['ctr_pt_position', 'post_pt_root_id', 'pre_pt_root_id'], + select_columns=["ctr_pt_position", "post_pt_root_id", "pre_pt_root_id"], ) - post_synapses['ctr_pt_position'] = post_synapses['ctr_pt_position'].apply(lambda x: x.tolist()) - post_synapses['post_pt_root_id'] = post_synapses['post_pt_root_id'].astype(str) - post_synapses['pre_pt_root_id'] = post_synapses['pre_pt_root_id'].astype(str) + post_synapses["ctr_pt_position"] = post_synapses["ctr_pt_position"].apply( + lambda x: x.tolist() + ) + post_synapses["post_pt_root_id"] = post_synapses["post_pt_root_id"].astype(str) + post_synapses["pre_pt_root_id"] = post_synapses["pre_pt_root_id"].astype(str) - if len(post_synapses['ctr_pt_position']) == 0: - raise Exception('No post-synapses found for root ids.') - position = np.random.choice(post_synapses['ctr_pt_position'].to_numpy()) + if len(post_synapses["ctr_pt_position"]) == 0: + raise Exception("No post-synapses found for root ids.") + position = np.random.choice(post_synapses["ctr_pt_position"].to_numpy()) data_list = [None] base_state = create_base_state(root_ids, position) @@ -623,40 +822,58 @@ def construct_synapse_state(root_ids: List, flags: dict = None): r = lambda: random.randint(0, 255) states = [base_state] for root_id in root_ids: - if flags['pre_synapses'] == 'True': - pre_points = pre_synapses[pre_synapses["pre_pt_root_id"] == root_id]['ctr_pt_position'].to_numpy() + if flags["pre_synapses"] == "True": + pre_points = pre_synapses[pre_synapses["pre_pt_root_id"] == root_id][ + "ctr_pt_position" + ].to_numpy() data_list.append(generate_point_df(pre_points)) states.append( - create_point_state(name=f'pre_synapses_{root_id}', color='#{:02x}{:02x}{:02x}'.format(r(), r(), r()))) - if flags['post_synapses'] == 'True': - post_points = post_synapses[post_synapses["post_pt_root_id"] == root_id]['ctr_pt_position'].to_numpy() + create_point_state( + name=f"pre_synapses_{root_id}", + color="#{:02x}{:02x}{:02x}".format(r(), r(), r()), + ) + ) + if flags["post_synapses"] == "True": + post_points = post_synapses[post_synapses["post_pt_root_id"] == root_id][ + "ctr_pt_position" + ].to_numpy() data_list.append(generate_point_df(post_points)) states.append( - create_point_state(name=f'post_synapses_{root_id}', color='#{:02x}{:02x}{:02x}'.format(r(), r(), r()))) + create_point_state( + name=f"post_synapses_{root_id}", + color="#{:02x}{:02x}{:02x}".format(r(), r(), r()), + ) + ) chained_state = ChainedStateBuilder(states) - state_dict = chained_state.render_state(return_as='dict', data_list=data_list) - state_dict['layout'] = '3d' + state_dict = chained_state.render_state(return_as="dict", data_list=data_list) + state_dict["layout"] = "3d" state_dict["selectedLayer"] = {"layer": "seg", "visible": True} - state_dict['jsonStateServer'] = settings.JSON_STATE_SERVER + state_dict["jsonStateServer"] = settings.JSON_STATE_SERVER # Metrics for each root_id synapse_stats = {} - if flags['pre_synapses'] == flags['post_synapses'] == 'True': + if flags["pre_synapses"] == flags["post_synapses"] == "True": for root_id in root_ids: # Pre-Synaptic Metrics - pre_synapses_slice = pre_synapses[pre_synapses['pre_pt_root_id'] == root_id] + pre_synapses_slice = pre_synapses[pre_synapses["pre_pt_root_id"] == root_id] num_pre_synapses = len(pre_synapses_slice) - num_pre_targets = len(np.unique(pre_synapses_slice['post_pt_root_id'])) - pre_synapses_to_targets = round(num_pre_synapses/num_pre_targets, ndigits=3) + num_pre_targets = len(np.unique(pre_synapses_slice["post_pt_root_id"])) + pre_synapses_to_targets = round( + num_pre_synapses / num_pre_targets, ndigits=3 + ) # Post-Synaptic Metrics - post_synapses_slice = post_synapses[post_synapses['post_pt_root_id'] == root_id] + post_synapses_slice = post_synapses[ + post_synapses["post_pt_root_id"] == root_id + ] num_post_synapses = len(post_synapses_slice) - num_post_targets = len(np.unique(post_synapses['pre_pt_root_id'])) - post_synapses_to_targets = round(num_post_synapses/num_post_targets, ndigits=3) + num_post_targets = len(np.unique(post_synapses["pre_pt_root_id"])) + post_synapses_to_targets = round( + num_post_synapses / num_post_targets, ndigits=3 + ) # Output to dict synapse_stats[root_id] = { @@ -667,49 +884,58 @@ def construct_synapse_state(root_ids: List, flags: dict = None): "post_synapses": True, "num_post_synapses": num_post_synapses, "num_post_targets": num_post_targets, - "post_synapses_to_targets": post_synapses_to_targets + "post_synapses_to_targets": post_synapses_to_targets, } # Only Pre-Synaptic Metrics - elif flags['pre_synapses'] == 'True': + elif flags["pre_synapses"] == "True": for root_id in root_ids: - pre_synapses_slice = pre_synapses[pre_synapses['pre_pt_root_id'] == root_id] + pre_synapses_slice = pre_synapses[pre_synapses["pre_pt_root_id"] == root_id] num_pre_synapses = len(pre_synapses_slice) - num_pre_targets = len(np.unique(pre_synapses_slice['post_pt_root_id'])) - pre_synapses_to_targets = round(num_pre_synapses/num_pre_targets, ndigits=3) + num_pre_targets = len(np.unique(pre_synapses_slice["post_pt_root_id"])) + pre_synapses_to_targets = round( + num_pre_synapses / num_pre_targets, ndigits=3 + ) # Output to dict synapse_stats[root_id] = { "pre_synapses": True, "num_pre_synapses": num_pre_synapses, "num_pre_targets": num_pre_targets, - "pre_synapses_to_targets": pre_synapses_to_targets + "pre_synapses_to_targets": pre_synapses_to_targets, } # Only Post-Synaptic Metrics else: for root_id in root_ids: - post_synapses_slice = post_synapses[post_synapses['post_pt_root_id'] == root_id] + post_synapses_slice = post_synapses[ + post_synapses["post_pt_root_id"] == root_id + ] num_post_synapses = len(post_synapses_slice) - num_post_targets = len(np.unique(post_synapses['pre_pt_root_id'])) - post_synapses_to_targets = round(num_post_synapses/num_post_targets, ndigits=3) + num_post_targets = len(np.unique(post_synapses["pre_pt_root_id"])) + post_synapses_to_targets = round( + num_post_synapses / num_post_targets, ndigits=3 + ) # Output to dict synapse_stats[root_id] = { "post_synapses": True, "num_post_synapses": num_post_synapses, "num_post_targets": num_post_targets, - "post_synapses_to_targets": post_synapses_to_targets + "post_synapses_to_targets": post_synapses_to_targets, } # Append clefts layers to state - if flags['cleft_layer'] == 'True': - state_dict['layers'].append({ - "type": "segmentation", - "source": "precomputed://s3://bossdb-open-data/iarpa_microns/minnie/minnie65/clefts-sharded", - "tab": "source", - "name": "clefts-sharded", - "visible": False - }) + if flags["cleft_layer"] == "True": + state_dict["layers"].append( + { + "type": "segmentation", + "source": "precomputed://s3://bossdb-open-data/iarpa_microns/minnie/minnie65/clefts-sharded", + "tab": "source", + "name": "clefts-sharded", + "visible": False, + } + ) return json.dumps(state_dict), synapse_stats + def construct_nuclei_state(given_ids: List): """Construct state for the synapse viewer. @@ -721,19 +947,26 @@ def construct_nuclei_state(given_ids: List): dict: synapse stats """ given_ids = [int(x) for x in given_ids] - cave_client = CAVEclient('minnie65_phase3_v1', auth_token=os.environ['CAVECLIENT_TOKEN']) - - + cave_client = CAVEclient( + "minnie65_phase3_v1", auth_token=os.environ["CAVECLIENT_TOKEN"] + ) + soma_df = get_df_from_static(cave_client, settings.NEURON_TABLE) - soma_df = soma_df[(soma_df.id.isin(given_ids))|(soma_df.pt_root_id.isin(given_ids))] - + soma_df = soma_df[ + (soma_df.id.isin(given_ids)) | (soma_df.pt_root_id.isin(given_ids)) + ] + # identify inputs that were not found in the table and format to display to user - ids_not_found = list(set(given_ids) - set().union(soma_df.id,soma_df.pt_root_id)) - formatted_not_found_ids = ', '.join([str(id) for id in ids_not_found]) if len(ids_not_found) else '' + ids_not_found = list(set(given_ids) - set().union(soma_df.id, soma_df.pt_root_id)) + formatted_not_found_ids = ( + ", ".join([str(id) for id in ids_not_found]) if len(ids_not_found) else "" + ) - root_ids = soma_df['pt_root_id'].values - nuclei_points = np.array(soma_df['pt_position'].values) - position = nuclei_points[0] if len(nuclei_points) else [] # check what happens when bad values are returned -- add an error case + root_ids = soma_df["pt_root_id"].values + nuclei_points = np.array(soma_df["pt_position"].values) + position = ( + nuclei_points[0] if len(nuclei_points) else [] + ) # check what happens when bad values are returned -- add an error case if not len(root_ids): raise Exception("ID is outdated or does not exist.") @@ -749,59 +982,78 @@ def generate_cell_type_table(soma_df): """Generates Cell type table returns filtered list of valid ids (i.e. listed in NUCLEUS_NEURON_SVM table) and their cell types if available, else NaN """ + def get_cell_type(nuclei_id, cell_class_df): filtered_row = cell_class_df[cell_class_df.id == nuclei_id] cell_type = filtered_row.cell_type.values[0] if len(filtered_row) else "NaN" return cell_type cell_class_info_df = get_df_from_static(cave_client, settings.CELL_CLASS_TABLE) - cell_class_info_df = cell_class_info_df[cell_class_info_df.id.isin(soma_df.id.values)] + cell_class_info_df = cell_class_info_df[ + cell_class_info_df.id.isin(soma_df.id.values) + ] + updated_soma_df = pd.merge(soma_df, cell_class_info_df, on="id", how="outer") + updated_soma_df.cell_type_y = updated_soma_df.cell_type_y.fillna("unknown") - updated_soma_df = pd.merge(soma_df, cell_class_info_df, on='id', how='outer') - updated_soma_df.cell_type_y = updated_soma_df.cell_type_y.fillna('unknown') - - type_table = 'Nuclei IDSeg IDType' - for nucleus_id, seg_id in zip(updated_soma_df.id.values, updated_soma_df.pt_root_id_x.values): + type_table = "Nuclei IDSeg IDType" + for nucleus_id, seg_id in zip( + updated_soma_df.id.values, updated_soma_df.pt_root_id_x.values + ): cell_type = get_cell_type(nucleus_id, cell_class_info_df) - type_table += ''+str(nucleus_id)+''+str(seg_id)+''+cell_type+'' - type_table += '' + type_table += ( + "" + + str(nucleus_id) + + "" + + str(seg_id) + + "" + + cell_type + + "" + ) + type_table += "" return type_table, updated_soma_df - cell_type_table, soma_df = generate_cell_type_table(soma_df) - for cell_type, type_df in soma_df.groupby('cell_type_y'): - data_list.append(generate_point_df(np.array(type_df['pt_position_x'].values))) + for cell_type, type_df in soma_df.groupby("cell_type_y"): + data_list.append(generate_point_df(np.array(type_df["pt_position_x"].values))) states.append( - create_point_state(name=f'{cell_type}_nuclei_points', color='#{:02x}{:02x}{:02x}'.format(r(), r(), r()))) + create_point_state( + name=f"{cell_type}_nuclei_points", + color="#{:02x}{:02x}{:02x}".format(r(), r(), r()), + ) + ) chained_state = ChainedStateBuilder(states) - state_dict = chained_state.render_state(return_as='dict', data_list=data_list) - state_dict['layout'] = '3d' + state_dict = chained_state.render_state(return_as="dict", data_list=data_list) + state_dict["layout"] = "3d" state_dict["selectedLayer"] = {"layer": "seg", "visible": True} - state_dict['jsonStateServer'] = settings.JSON_STATE_SERVER + state_dict["jsonStateServer"] = settings.JSON_STATE_SERVER return json.dumps(state_dict), cell_type_table, formatted_not_found_ids -def refresh_ids(ng_state:str, namespace:str): +def refresh_ids(ng_state: str, namespace: str): namespace = Namespace.objects.get(namespace=namespace) if not namespace.refresh_selected_root_ids: return ng_state - + if namespace.pcg_source == PcgChoices.PINKY: return ng_state else: - cave_client = CAVEclient('minnie65_phase3_v1', auth_token=os.environ['CAVECLIENT_TOKEN']) - + cave_client = CAVEclient( + "minnie65_phase3_v1", auth_token=os.environ["CAVECLIENT_TOKEN"] + ) + state = json.loads(ng_state) - for layer in state['layers']: - if layer['type'] == "segmentation_with_graph" and len(layer.get("segments", [])): + for layer in state["layers"]: + if layer["type"] == "segmentation_with_graph" and len( + layer.get("segments", []) + ): latest_ids = set() - for root_id in layer['segments']: + for root_id in layer["segments"]: try: roots = cave_client.chunkedgraph.get_latest_roots(root_id).tolist() roots = list(map(str, roots)) @@ -810,5 +1062,5 @@ def refresh_ids(ng_state:str, namespace:str): logging.error(f"CaveClient Exception: {e}") return ng_state - layer['segments'] = list(latest_ids) + layer["segments"] = list(latest_ids) return json.dumps(state) diff --git a/neuvue_project/workspace/templatetags/in_group.py b/neuvue_project/workspace/templatetags/in_group.py index abd39d05..2f0e0167 100644 --- a/neuvue_project/workspace/templatetags/in_group.py +++ b/neuvue_project/workspace/templatetags/in_group.py @@ -3,11 +3,12 @@ register = template.Library() -@register.filter(name='in_group') + +@register.filter(name="in_group") def in_group(user, group_name): try: Group.objects.get(name=group_name) except Group.DoesNotExist: return False - - return user.groups.filter(name=group_name).exists() \ No newline at end of file + + return user.groups.filter(name=group_name).exists() diff --git a/neuvue_project/workspace/utils.py b/neuvue_project/workspace/utils.py index 87a0ddaa..e6202007 100644 --- a/neuvue_project/workspace/utils.py +++ b/neuvue_project/workspace/utils.py @@ -3,6 +3,7 @@ from pytz import timezone import pytz + def is_url(value): validate = URLValidator() try: @@ -11,6 +12,7 @@ def is_url(value): except: return False + def is_json(value): try: json.loads(value) @@ -18,12 +20,15 @@ def is_json(value): except: return False + def is_authorized(user): - return user.is_authenticated and user.groups.filter(name='AuthorizedUsers').exists() + return user.is_authenticated and user.groups.filter(name="AuthorizedUsers").exists() + def is_member(user, group): return user.groups.filter(name=group).exists() + def utc_to_eastern(time_value): """Converts a pandas datetime object to a US/Easten datetime. @@ -35,9 +40,11 @@ def utc_to_eastern(time_value): """ try: utc = pytz.UTC - eastern = timezone('US/Eastern') - date_time = time_value.to_pydatetime(warn=False) # do not warn if nanoseconds are nonzero + eastern = timezone("US/Eastern") + date_time = time_value.to_pydatetime( + warn=False + ) # do not warn if nanoseconds are nonzero date_time = utc.localize(time_value) return date_time.astimezone(eastern) except: - return time_value \ No newline at end of file + return time_value diff --git a/neuvue_project/workspace/validators.py b/neuvue_project/workspace/validators.py index ec05551d..4a196b58 100644 --- a/neuvue_project/workspace/validators.py +++ b/neuvue_project/workspace/validators.py @@ -1,9 +1,10 @@ from django.core.exceptions import ValidationError from django.utils.translation import gettext_lazy as _ + def validate_submission_value(value): - if value in ['start', 'stop', 'skip', 'flag', 'remove', 'submit']: + if value in ["start", "stop", "skip", "flag", "remove", "submit"]: raise ValidationError( - _(f'{value} is not an valid submission value'), - params={'value': value}, - ) \ No newline at end of file + _(f"{value} is not an valid submission value"), + params={"value": value}, + ) diff --git a/neuvue_project/workspace/views/save.py b/neuvue_project/workspace/views/save.py index 1025372a..d9741d66 100644 --- a/neuvue_project/workspace/views/save.py +++ b/neuvue_project/workspace/views/save.py @@ -19,7 +19,6 @@ class SaveStateView(View): - def post(self, request, *args, **kwargs): data = str(request.body.decode("utf-8")) data = json.loads(data) @@ -49,7 +48,6 @@ def post(self, request, *args, **kwargs): class SaveOperationsView(View): - def post(self, request, *args, **kwargs): data = str(request.body.decode("utf-8")) diff --git a/neuvue_project/workspace/views/task.py b/neuvue_project/workspace/views/task.py index 3ece5436..8bbf7736 100644 --- a/neuvue_project/workspace/views/task.py +++ b/neuvue_project/workspace/views/task.py @@ -22,7 +22,6 @@ class TaskView(LoginRequiredMixin, View): - def get(self, request, *args, **kwargs): context = {} self_assign_group = "Can self assign tasks" diff --git a/neuvue_project/workspace/views/tools.py b/neuvue_project/workspace/views/tools.py index a42eb3e9..1aac229d 100644 --- a/neuvue_project/workspace/views/tools.py +++ b/neuvue_project/workspace/views/tools.py @@ -24,7 +24,6 @@ class InspectTaskView(View): - def get(self, request, task_id=None, *args, **kwargs): if task_id in settings.STATIC_NG_FILES: return redirect( @@ -207,35 +206,41 @@ def post(self, request, *args, **kwargs): class NucleiView(View): def get(self, request, given_ids=None, *args, **kwargs): if not is_authorized(request.user): - logging.warning(f'Unauthorized requests from {request.user}.') - return redirect(reverse('index')) + logging.warning(f"Unauthorized requests from {request.user}.") + return redirect(reverse("index")) if given_ids in settings.STATIC_NG_FILES: - return redirect(f'/static/workspace/{given_ids}', content_type='application/javascript') + return redirect( + f"/static/workspace/{given_ids}", content_type="application/javascript" + ) - context = { - "given_ids": None, - "error": None - } + context = {"given_ids": None, "error": None} if given_ids is None: return render(request, "nuclei.html", context) - given_ids = [x.strip() for x in given_ids.split(',')] + given_ids = [x.strip() for x in given_ids.split(",")] try: - context['given_ids'] = given_ids - context['ng_state'], context['cell_types'], context['ids_not_found'] = construct_nuclei_state(given_ids=given_ids) - except Exception as e: - context['error'] = e + context["given_ids"] = given_ids + ( + context["ng_state"], + context["cell_types"], + context["ids_not_found"], + ) = construct_nuclei_state(given_ids=given_ids) + except Exception as e: + context["error"] = e return render(request, "nuclei.html", context) - def post(self, request, *args, **kwargs): given_ids = request.POST.get("given_ids") - return redirect(reverse('nuclei', kwargs={ - "given_ids": given_ids, - })) - + return redirect( + reverse( + "nuclei", + kwargs={ + "given_ids": given_ids, + }, + ) + ) diff --git a/neuvue_project/workspace/views/workspace.py b/neuvue_project/workspace/views/workspace.py index 4754c444..7a214740 100644 --- a/neuvue_project/workspace/views/workspace.py +++ b/neuvue_project/workspace/views/workspace.py @@ -28,7 +28,6 @@ class WorkspaceView(LoginRequiredMixin, View): - def get(self, request, namespace=None, **kwargs): # TODO: # This redirects NG static files. Currently, NG redirects directly to root in their js @@ -157,9 +156,7 @@ def get(self, request, namespace=None, **kwargs): else: # Manually get the points for now, populate in client later. - points = [ - client.get_point(x)["coordinate"] for x in task_df["points"] - ] + points = [client.get_point(x)["coordinate"] for x in task_df["points"]] context["ng_state"] = construct_proofreading_state( task_df, points, return_as="json" ) diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 00000000..9f520906 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +pre-commit +black \ No newline at end of file