From d2a1bc387c242166f02fdcaa07a57d60d7fa6606 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mu=CC=88ller?= Date: Thu, 18 Jul 2024 18:10:09 +0200 Subject: [PATCH 1/5] Prevent a race condition causing missing data When booting up Jupiter, the foreground actor triggers an initial repository scan in the background actor. It then sends a FileEvent for every file in the repo using a tokio broadcast. The loader actor subscribes to this broadcast and in turn triggers a reload of every loader. The loader actor subscribed to this broadcast too late, while some FileEvents were already being dispatched. The tokio broadcast drops these first events (as documented), which causes these files to not be loaded. This commit moves some code, so that the loader subscribes to the broadcast before the foreground actor starts the repo scan. Fixes: SIRI-994 --- jupiter-rs/src/repository/loader.rs | 3 ++- jupiter-rs/src/repository/mod.rs | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/jupiter-rs/src/repository/loader.rs b/jupiter-rs/src/repository/loader.rs index fcb4a46..da4d16e 100644 --- a/jupiter-rs/src/repository/loader.rs +++ b/jupiter-rs/src/repository/loader.rs @@ -256,11 +256,12 @@ pub fn actor( mut background_task_sender: mpsc::Sender, ) -> Queue { let (command_queue, mut commands_endpoint) = queue(); + let mut listener = repository.listener(); + spawn!(async move { use crate::commands::ResultExt; let mut loaders = HashMap::new(); - let mut listener = repository.listener(); let config = platform.require::(); let mut config_changed_flag = config.notifier(); let mut namespaces = load_namespaces(&config); diff --git a/jupiter-rs/src/repository/mod.rs b/jupiter-rs/src/repository/mod.rs index da0af44..4a3e9b9 100644 --- a/jupiter-rs/src/repository/mod.rs +++ b/jupiter-rs/src/repository/mod.rs @@ -289,6 +289,9 @@ pub fn create(platform: &Arc) -> Arc { /// perform the initial repository scan to determine what contents are already available. pub fn install(platform: Arc, repository: Arc) { let (background_task_queue, update_notifier) = background::actor(platform.clone()); + + let loader_queue = loader::actor(platform.clone(), repository, background_task_queue); + let command_queue = foreground::actor( platform.clone(), repository.clone(), @@ -296,8 +299,6 @@ pub fn install(platform: Arc, repository: Arc) { update_notifier, ); - let loader_queue = loader::actor(platform.clone(), repository, background_task_queue); - if let Some(commands) = platform.find::() { commands.register_command( "REPO.SCAN", From fd24aa65c7e9fc405b9045d46504e09a8c7caf91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mu=CC=88ller?= Date: Thu, 18 Jul 2024 18:15:11 +0200 Subject: [PATCH 2/5] Prevent a theoretical race condition The idb actor registers some own loaders, which might be missing if the initial repo scan is started before. This commit guarantees that the repo scan only happens after all loaders are registered. This problem is theoretical and was most likely not the cause of the missing data. Fixes: SIRI-994 --- jupiter-io/src/main.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/jupiter-io/src/main.rs b/jupiter-io/src/main.rs index 254b277..b6c905d 100644 --- a/jupiter-io/src/main.rs +++ b/jupiter-io/src/main.rs @@ -24,8 +24,8 @@ async fn main() { // Setup and enable the LRU cache... jupiter::lru::cache::install(platform.clone()); - // Setup and install a data repository... - jupiter::repository::install(platform.clone(), jupiter::repository::create(&platform)); + // Setup a data repository... + let repository = jupiter::repository::create(&platform); // Setup and enable InfoGraphDB... jupiter::idb::install(platform.clone()); @@ -33,5 +33,8 @@ async fn main() { // Setup pyrun... jupiter::pyrun::install(platform.clone()); + // Finally install the data repository after all other actors have registered their loaders + jupiter::repository::install(platform.clone(), repository); + platform.require::().event_loop().await; } From 38830db3b4659708dd0111d764553b74294c1def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mu=CC=88ller?= Date: Thu, 18 Jul 2024 19:48:44 +0200 Subject: [PATCH 3/5] Fix some clippy inspections Fixes: SIRI-994 --- jupiter-rs/src/repository/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jupiter-rs/src/repository/mod.rs b/jupiter-rs/src/repository/mod.rs index 4a3e9b9..0a6013a 100644 --- a/jupiter-rs/src/repository/mod.rs +++ b/jupiter-rs/src/repository/mod.rs @@ -290,7 +290,7 @@ pub fn create(platform: &Arc) -> Arc { pub fn install(platform: Arc, repository: Arc) { let (background_task_queue, update_notifier) = background::actor(platform.clone()); - let loader_queue = loader::actor(platform.clone(), repository, background_task_queue); + let loader_queue = loader::actor(platform.clone(), repository.clone(), background_task_queue.clone()); let command_queue = foreground::actor( platform.clone(), From aa15d6da1a77b5ca5b4b63551b68c14bb008c553 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mu=CC=88ller?= Date: Thu, 18 Jul 2024 19:50:38 +0200 Subject: [PATCH 4/5] Code format Fixes: SIRI-994 --- jupiter-rs/src/repository/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jupiter-rs/src/repository/mod.rs b/jupiter-rs/src/repository/mod.rs index 0a6013a..ce896a8 100644 --- a/jupiter-rs/src/repository/mod.rs +++ b/jupiter-rs/src/repository/mod.rs @@ -290,7 +290,11 @@ pub fn create(platform: &Arc) -> Arc { pub fn install(platform: Arc, repository: Arc) { let (background_task_queue, update_notifier) = background::actor(platform.clone()); - let loader_queue = loader::actor(platform.clone(), repository.clone(), background_task_queue.clone()); + let loader_queue = loader::actor( + platform.clone(), + repository.clone(), + background_task_queue.clone(), + ); let command_queue = foreground::actor( platform.clone(), From fdb6f46840933cbda5c855412212f0747004c9d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Mu=CC=88ller?= Date: Fri, 19 Jul 2024 08:12:48 +0200 Subject: [PATCH 5/5] Remove redundant clones Fixes: SIRI-994 --- jupiter-rs/src/repository/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jupiter-rs/src/repository/mod.rs b/jupiter-rs/src/repository/mod.rs index ce896a8..72296ac 100644 --- a/jupiter-rs/src/repository/mod.rs +++ b/jupiter-rs/src/repository/mod.rs @@ -298,8 +298,8 @@ pub fn install(platform: Arc, repository: Arc) { let command_queue = foreground::actor( platform.clone(), - repository.clone(), - background_task_queue.clone(), + repository, + background_task_queue, update_notifier, );