Skip to content

Commit

Permalink
Use asyncio.gather
Browse files Browse the repository at this point in the history
  • Loading branch information
tmetzl committed Oct 7, 2024
1 parent cef368d commit be53f45
Show file tree
Hide file tree
Showing 2 changed files with 41 additions and 8 deletions.
30 changes: 25 additions & 5 deletions e2xauthoring/managers/taskmanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,12 +128,23 @@ async def list(self, pool):
tasks = []
path = os.path.join(self.base_path, pool)
assert os.path.exists(path), f"No pool with the name {pool} exists."
for task_dir in self.listdir(os.path.join(self.base_path, pool)):
points, n_questions = await self.__get_task_info(task_dir, pool)
git_status = await asyncio.to_thread(self.git_status, pool, task_dir)

coroutines = []
task_dirs = self.listdir(os.path.join(self.base_path, pool))

for task_dir in task_dirs:
coroutines.append(self.__get_task_info(task_dir, pool))
coroutines.append(asyncio.to_thread(self.git_status, pool, task_dir))

results = await asyncio.gather(*coroutines)

for i, task_dir in enumerate(task_dirs):
points, n_questions = results[i * 2]
git_status = results[i * 2 + 1]

if "repo" in git_status:
del git_status["repo"]

tasks.append(
Task(
name=task_dir,
Expand All @@ -143,15 +154,24 @@ async def list(self, pool):
git_status=git_status,
)
)

return tasks

async def list_all(self):
pool_manager = TaskPoolManager(self.coursedir)
tasks = []
pool_list = await pool_manager.list()
for pool in pool_list:
pool_tasks = await self.list(pool.name)

# Collect all coroutines
coroutines = [self.list(pool.name) for pool in pool_list]

# Run them concurrently
results = await asyncio.gather(*coroutines)

# Flatten the list of results
for pool_tasks in results:
tasks.extend(pool_tasks)

return tasks

def copy(self, old_name: str, new_name: str, pool: str = ""):
Expand Down
19 changes: 16 additions & 3 deletions e2xauthoring/managers/taskpoolmanager.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,18 +61,31 @@ async def list(self):
if not os.path.exists(self.base_path):
self.log.warning("The pool directory does not exist.")
os.makedirs(self.base_path, exist_ok=True)

pool_dirs = await asyncio.to_thread(self.listdir, self.base_path)
tasks = []
coroutines = []

for pool_dir in pool_dirs:
n_tasks = await self.__get_n_tasks(pool_dir)
is_repo = await asyncio.to_thread(
is_version_controlled, os.path.join(self.base_path, pool_dir)
coroutines.append(self.__get_n_tasks(pool_dir))
coroutines.append(
asyncio.to_thread(
is_version_controlled, os.path.join(self.base_path, pool_dir)
)
)

results = await asyncio.gather(*coroutines)

for i, pool_dir in enumerate(pool_dirs):
n_tasks = results[i * 2]
is_repo = results[i * 2 + 1]

tasks.append(
TaskPool(
name=pool_dir,
n_tasks=n_tasks,
is_repo=is_repo,
)
)

return tasks

0 comments on commit be53f45

Please sign in to comment.