From be53f45873b51972f06e7ae7ad2b270a941565c7 Mon Sep 17 00:00:00 2001 From: Tim Daniel Metzler Date: Mon, 7 Oct 2024 18:10:34 +0200 Subject: [PATCH] Use asyncio.gather --- e2xauthoring/managers/taskmanager.py | 30 ++++++++++++++++++++---- e2xauthoring/managers/taskpoolmanager.py | 19 ++++++++++++--- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/e2xauthoring/managers/taskmanager.py b/e2xauthoring/managers/taskmanager.py index bb9d7e0..cb873f6 100644 --- a/e2xauthoring/managers/taskmanager.py +++ b/e2xauthoring/managers/taskmanager.py @@ -128,12 +128,23 @@ async def list(self, pool): tasks = [] path = os.path.join(self.base_path, pool) assert os.path.exists(path), f"No pool with the name {pool} exists." - for task_dir in self.listdir(os.path.join(self.base_path, pool)): - points, n_questions = await self.__get_task_info(task_dir, pool) - git_status = await asyncio.to_thread(self.git_status, pool, task_dir) + + coroutines = [] + task_dirs = self.listdir(os.path.join(self.base_path, pool)) + + for task_dir in task_dirs: + coroutines.append(self.__get_task_info(task_dir, pool)) + coroutines.append(asyncio.to_thread(self.git_status, pool, task_dir)) + + results = await asyncio.gather(*coroutines) + + for i, task_dir in enumerate(task_dirs): + points, n_questions = results[i * 2] + git_status = results[i * 2 + 1] if "repo" in git_status: del git_status["repo"] + tasks.append( Task( name=task_dir, @@ -143,15 +154,24 @@ async def list(self, pool): git_status=git_status, ) ) + return tasks async def list_all(self): pool_manager = TaskPoolManager(self.coursedir) tasks = [] pool_list = await pool_manager.list() - for pool in pool_list: - pool_tasks = await self.list(pool.name) + + # Collect all coroutines + coroutines = [self.list(pool.name) for pool in pool_list] + + # Run them concurrently + results = await asyncio.gather(*coroutines) + + # Flatten the list of results + for pool_tasks in results: tasks.extend(pool_tasks) + return tasks def copy(self, old_name: str, new_name: str, pool: str = ""): diff --git a/e2xauthoring/managers/taskpoolmanager.py b/e2xauthoring/managers/taskpoolmanager.py index a34d6a4..d6ecc74 100644 --- a/e2xauthoring/managers/taskpoolmanager.py +++ b/e2xauthoring/managers/taskpoolmanager.py @@ -61,13 +61,25 @@ async def list(self): if not os.path.exists(self.base_path): self.log.warning("The pool directory does not exist.") os.makedirs(self.base_path, exist_ok=True) + pool_dirs = await asyncio.to_thread(self.listdir, self.base_path) tasks = [] + coroutines = [] + for pool_dir in pool_dirs: - n_tasks = await self.__get_n_tasks(pool_dir) - is_repo = await asyncio.to_thread( - is_version_controlled, os.path.join(self.base_path, pool_dir) + coroutines.append(self.__get_n_tasks(pool_dir)) + coroutines.append( + asyncio.to_thread( + is_version_controlled, os.path.join(self.base_path, pool_dir) + ) ) + + results = await asyncio.gather(*coroutines) + + for i, pool_dir in enumerate(pool_dirs): + n_tasks = results[i * 2] + is_repo = results[i * 2 + 1] + tasks.append( TaskPool( name=pool_dir, @@ -75,4 +87,5 @@ async def list(self): is_repo=is_repo, ) ) + return tasks