feat(webinterface): wait for single file analysis

also adds a button to the 'analysis is outdated' message to update it

src/intercom/back_end_binding.py

@@ -2,6 +2,7 @@
 
 import difflib
 import logging
+from multiprocessing import Manager
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -38,6 +39,7 @@ def __init__(
         self.compare_service = compare_service
         self.unpacking_service = unpacking_service
         self.unpacking_locks = unpacking_locks
+        self.manager = Manager()
         self.listeners = [
             InterComBackEndAnalysisTask(self.unpacking_service.add_task),
             InterComBackEndReAnalyzeTask(self.unpacking_service.add_task),
@@ -51,7 +53,10 @@ def __init__(
                 unpacking_locks=self.unpacking_locks,
                 db_interface=DbInterfaceCommon(),
             ),
-            InterComBackEndSingleFileTask(self.analysis_service.update_analysis_of_single_object),
+            InterComBackEndSingleFileTask(
+                self.analysis_service.update_analysis_of_single_object,
+                manager=self.manager,
+            ),
             InterComBackEndPeekBinaryTask(),
             InterComBackEndLogsTask(),
         ]
@@ -65,6 +70,7 @@ def start(self):
     def shutdown(self):
         for listener in self.listeners:
             listener.shutdown()
+        self.manager.shutdown()
         stop_processes(
             [listener.process for listener in self.listeners if listener],
             config.backend.intercom_poll_delay + 1,
@@ -101,8 +107,31 @@ class InterComBackEndUpdateTask(InterComBackEndReAnalyzeTask):
     CONNECTION_TYPE = 'update_task'
 
 
-class InterComBackEndSingleFileTask(InterComBackEndReAnalyzeTask):
+class InterComBackEndSingleFileTask(InterComListenerAndResponder):
     CONNECTION_TYPE = 'single_file_task'
+    OUTGOING_CONNECTION_TYPE = 'single_file_task_resp'
+
+    def __init__(self, *args, manager: Manager):
+        super().__init__(*args)
+        self.manager = manager
+        self.events = self.manager.dict()
+
+    def pre_process(self, task: Firmware, task_id):
+        analysis_finished_event = self.manager.Event()
+        self.events[task.uid] = analysis_finished_event
+        task.callback = analysis_finished_event.set
+        return super().pre_process(task, task_id)
+
+    def get_response(self, task: Firmware):
+        try:
+            event = self.events.pop(task.uid)
+            event.wait(timeout=60)
+            if not event.is_set():
+                raise TimeoutError
+            return True
+        except (TimeoutError, KeyError):
+            logging.exception('Single file update failed.')
+            return False
 
 
 class InterComBackEndCompareTask(InterComListener):

src/intercom/common_redis_binding.py

@@ -4,6 +4,7 @@
 import os
 import pickle
 from multiprocessing import Process, Value
+from threading import Thread
 from time import sleep, time
 from typing import TYPE_CHECKING, Any, Callable
 
@@ -87,10 +88,15 @@ class InterComListenerAndResponder(InterComListener):
 
     def pre_process(self, task, task_id):
         logging.debug(f'request received: {self.CONNECTION_TYPE} -> {task_id}')
+        # fetch the response in a different thread so that the listener is not blocked while waiting for the result
+        tread = Thread(target=self._get_response_asynchronously, args=(task, task_id))
+        tread.start()
+        return task
+
+    def _get_response_asynchronously(self, task, task_id):
         response = self.get_response(task)
         self.redis.set(task_id, response)
-        logging.debug(f'response send: {self.OUTGOING_CONNECTION_TYPE} -> {task_id}')
-        return task
+        logging.debug(f'response sent: {self.OUTGOING_CONNECTION_TYPE} -> {task_id}')
 
     def get_response(self, task):
         """

src/intercom/front_end_binding.py

@@ -27,7 +27,7 @@ def add_re_analyze_task(self, fw, unpack=True):
             self._add_to_redis_queue('update_task', fw, fw.uid)
 
     def add_single_file_task(self, fw):
-        self._add_to_redis_queue('single_file_task', fw, fw.uid)
+        return self._request_response_listener(fw, 'single_file_task', 'single_file_task_resp')
 
     def add_compare_task(self, compare_id, force=False):
         self._add_to_redis_queue('compare_task', (compare_id, force), compare_id)
@@ -77,7 +77,7 @@ def _response_listener(self, response_connection, request_id, timeout=None):
             timeout = time() + int(config.frontend.communication_timeout)
         while timeout > time():
             output_data = self.redis.get(request_id)
-            if output_data:
+            if output_data is not None:
                 logging.debug(f'Response received: {response_connection} -> {request_id}')
                 break
             logging.debug(f'No response yet: {response_connection} -> {request_id}')

src/objects/file.py

@@ -96,6 +96,9 @@ def __init__(
         #: for debugging purposes and as placeholder in UI.
         self.analysis_exception = None
 
+        #: Optional callback method called after the analysis finished in the analysis scheduler
+        self.callback = None
+
         if binary is not None:
             self.set_binary(binary)
         else:

src/scheduler/analysis/scheduler.py

@@ -549,9 +549,18 @@ def _check_further_process_or_complete(self, fw_object):
         if not fw_object.scheduled_analysis:
             logging.info(f'Analysis Completed: {fw_object.uid}')
             self.status.remove_object(fw_object)
+            self._do_callback(fw_object)
         else:
             self.process_queue.put(fw_object)
 
+    @staticmethod
+    def _do_callback(fw_object: FileObject):
+        if fw_object.callback is not None:
+            try:
+                fw_object.callback()
+            except Exception as error:
+                logging.exception(f'Callback failed for {fw_object.uid}: {error}')
+
     # ---- miscellaneous functions ----
 
     def get_combined_analysis_workload(self):

src/test/integration/intercom/test_task_communication.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from concurrent.futures import ThreadPoolExecutor
+from multiprocessing import Manager
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 from time import sleep
@@ -67,7 +68,7 @@ def test_analysis_task(self, intercom_frontend):
         assert Path(task.file_path).exists(), 'file does not exist'
 
     def test_single_file_task(self, intercom_frontend):
-        task_listener = InterComBackEndSingleFileTask()
+        task_listener = InterComBackEndSingleFileTask(manager=Manager())
         test_fw = create_test_firmware()
         test_fw.file_path = None
         test_fw.scheduled_analysis = ['binwalk']

src/test/unit/web_interface/test_app_show_analysis.py

@@ -80,9 +80,11 @@ def test_app_single_file_analysis(self, test_client, intercom_task_list):
             f'/analysis/{TEST_FW.uid}',
             content_type='multipart/form-data',
             data={'analysis_systems': ['plugin_a', 'plugin_b']},
+            follow_redirects=True,
         )
 
-        assert post_new.status_code == HTTPStatus.FOUND
+        assert post_new.status_code == HTTPStatus.OK
+        assert 'success' in post_new.json
         assert intercom_task_list
         assert intercom_task_list[0].scheduled_analysis == ['plugin_a', 'plugin_b']
 

src/web_interface/components/analysis_routes.py

@@ -7,7 +7,7 @@
 from typing import TYPE_CHECKING
 
 from common_helper_files import get_binary_from_file
-from flask import flash, redirect, render_template, render_template_string, request, url_for
+from flask import flash, render_template, render_template_string, request
 from flask_login.utils import current_user
 
 import config
@@ -105,14 +105,16 @@ def _get_correct_template(self, selected_analysis: str | None, fw_object: Firmwa
     @AppRoute('/analysis/<uid>/ro/<root_uid>', POST)
     @AppRoute('/analysis/<uid>/<selected_analysis>', POST)
     @AppRoute('/analysis/<uid>/<selected_analysis>/ro/<root_uid>', POST)
-    def start_single_file_analysis(self, uid, selected_analysis=None, root_uid=None):
+    @AppRoute('/analysis/single-update/<uid>/<plugin>', POST)
+    def start_single_file_analysis(self, uid, selected_analysis=None, root_uid=None, plugin=None):  # noqa: ARG002
         file_object = self.db.frontend.get_object(uid)
-        file_object.scheduled_analysis = request.form.getlist('analysis_systems')
+        if plugin:
+            file_object.scheduled_analysis = [plugin]
+        else:
+            file_object.scheduled_analysis = request.form.getlist('analysis_systems')
         file_object.force_update = request.form.get('force_update') == 'true'
-        self.intercom.add_single_file_task(file_object)
-        return redirect(
-            url_for(self.show_analysis.__name__, uid=uid, root_uid=root_uid, selected_analysis=selected_analysis)
-        )
+        success = self.intercom.add_single_file_task(file_object)
+        return {'success': success}
 
     @staticmethod
     def _get_used_and_unused_plugins(processed_analysis: dict, all_plugins: list) -> dict:

src/web_interface/static/js/show_analysis_single_update.js

@@ -0,0 +1,48 @@
+// single file analysis (update)
+function waitForAnalysis(url, element, formData=null) {
+    // wait until the analysis is finished and then (re)load the page to show it
+    element.innerHTML = `<i class="fas fa-spinner fa-fw margin-right-md fa-spin"></i> analysis in progress ...`;
+    const message = 'Timeout when waiting for analysis. Please manually refresh the page.';
+    fetch(url, {
+        method: 'POST',
+        body: formData,
+        signal: AbortSignal.timeout(60000)
+    })
+    .then(response => response.json())
+    .then(data => {
+        if (data.success) {
+            console.log('Analysis successful');
+            localStorage.setItem('analysisUpdated', `${selected_analysis}-${uid}`);
+            if (formData != null) {
+                const checkedOptions = [];
+                formData.forEach((value, key) => {
+                    if (key === 'analysis_systems') {
+                        checkedOptions.push(value);
+                    }
+                });
+                if (checkedOptions.length > 0 && checkedOptions.indexOf(selected_analysis) === -1) {
+                    let url = window.location.href;
+                    const someSelectedPlugin = checkedOptions[0];
+                    localStorage.setItem('analysisUpdated', `${someSelectedPlugin}-${uid}`);
+                    if (selected_analysis === 'None') {
+                        // no plugin is currently selected
+                        url = `/analysis/${uid}/${someSelectedPlugin}`;
+                    } else {
+                        // another plugin (that was not updated) is selected → replace it in the URL
+                        url = url.replace(selected_analysis, someSelectedPlugin);
+                    }
+                    window.location.href = url;
+                    return;
+                }
+            }
+            window.location.reload();
+        } else {
+            console.log('Analysis failed');
+            element.innerHTML = message;
+        }
+    })
+    .catch(error => {
+        console.error('Error during single file analysis:', error);
+        element.innerHTML = message;
+    });
+}

src/web_interface/templates/analysis_plugins/general_information.html

@@ -2,7 +2,7 @@
 
 {% block analysis_result %}
 
-    <table class="table table-bordered" style="table-layout: fixed">
+    <table id="show-analysis-table" class="table table-bordered"  style="table-layout: fixed">
         <colgroup>
              <col style="width: 12.5%">
              <col style="width: 87.5%">
@@ -72,12 +72,21 @@
             {% else %}
                 <tr>
                     <td class="table-warning">Analysis outdated</td>
-                    <td>
+                    <td id="analysis-outdated-td">
                         The backend plugin version ({{ version_backend | string }}) is incompatible with
                         the version ({{ version_database | string }}) of the analysis result.
 
-                        Please update the analysis.
+                        <button class="btn btn-primary btn-sm" onclick="startSingleAnalysis()">
+                            Update Analysis
+                        </button>
                     </td>
+                    <script>
+                        function startSingleAnalysis() {
+                            const url = `/analysis/single-update/${uid}/${selected_analysis}`;
+                            let element = document.getElementById("analysis-outdated-td");
+                            waitForAnalysis(url, element);
+                        }
+                    </script>
                 </tr>
             {% endif %}
         </tbody>

src/web_interface/templates/show_analysis.html

@@ -37,6 +37,25 @@
         const selected_analysis = "{{ selected_analysis }}";
     </script>
     <script src="{{ url_for('static', filename='js/file_tree.js') }}"></script>
+    <script type="text/javascript" src="{{ url_for('static', filename='js/show_analysis_single_update.js') }}"></script>
+    <script>
+        window.addEventListener('load', function() {
+            // add highlighting animation if analysis was updated
+            if (localStorage.getItem('analysisUpdated') === `${selected_analysis}-${uid}`) {
+                document.getElementById('show-analysis-table').classList.add('updated-analysis');
+                localStorage.setItem('analysisUpdated', 'false');
+            }
+        });
+    </script>
+    <style>
+        @keyframes updated-analysis {
+            from { box-shadow: 0 0 10px 5px rgba(0, 105, 217, 0.5); }
+            to { box-shadow: none; }
+        }
+        .updated-analysis {
+            animation: updated-analysis 2s ease-in-out;
+        }
+    </style>
 {% endblock %}
 
 
@@ -267,8 +286,8 @@ <h5 class="modal-title">Add analysis to file</h5>
                         <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">&times;</span></button>
                     </div>
 
-                    <div class="modal-body">
-                        <form class="form-horizontal" action="" method=post enctype=multipart/form-data>
+                    <div class="modal-body" id="modal-body">
+                        <form class="form-horizontal" id="modal-form" action="" method=post enctype=multipart/form-data>
                             <p>Add new analysis</p>
                             {% for system in available_plugins.unused | sort %}
                                 <label class="checkbox-inline" data-toggle="tooltip" title="{{ analysis_plugin_dict[system][0] | safe }}" style="display: block">
@@ -294,6 +313,19 @@ <h5 class="modal-title">Add analysis to file</h5>
                 </div>
             </div>
         </div>
+        <script>
+            document.addEventListener('DOMContentLoaded', () => {
+                // overwrites the submit action of the "Run additional analysis" modal button
+                // and waits for the result before refreshing the page
+                const form = document.getElementById('modal-form');
+                const modalBody = document.getElementById('modal-body');
+                form.addEventListener('submit', (event) => {
+                    event.preventDefault();
+                    const formData = new FormData(form);
+                    waitForAnalysis(form.action, modalBody, formData);
+                });
+            });
+        </script>
 
         {# Showing analysis section #}
 
@@ -313,7 +345,7 @@ <h5 class="modal-title">Add analysis to file</h5>
                     <div id="summary-div">
                         <div class="mb-3" id="loading-summary-gif" style="display: none; border: 1px solid #dddddd; padding: 5px; text-align: center">
                             <img src="{{ url_for("static", filename = "Pacman.gif") }}" alt="loading gif">
-                            <p>Loading summary for included files ..</p>
+                            <p>Loading summary for included files...</p>
                         </div>
                     </div>
                     <script type="text/javascript" src="{{ url_for('static', filename='js/show_analysis_summary.js') }}"></script>