From ce738b0bd1fef7c5aec8464fdab0c98dfd3384da Mon Sep 17 00:00:00 2001 From: Tim Daniel Metzler Date: Mon, 7 Oct 2024 15:40:35 +0200 Subject: [PATCH 1/5] Make TaskManager async --- e2xauthoring/managers/taskmanager.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/e2xauthoring/managers/taskmanager.py b/e2xauthoring/managers/taskmanager.py index 4e109fe..54eb1f8 100644 --- a/e2xauthoring/managers/taskmanager.py +++ b/e2xauthoring/managers/taskmanager.py @@ -1,3 +1,4 @@ +import asyncio import os import shutil @@ -17,7 +18,7 @@ class TaskManager(BaseManager): "pools", help="The relative directory where the pools are stored" ) - def __get_task_info(self, task, pool): + async def __get_task_info(self, task, pool): base_path = os.path.join(self.base_path, pool) notebooks = [ file @@ -29,7 +30,11 @@ def __get_task_info(self, task, pool): questions = 0 for notebook in notebooks: - nb = nbformat.read(os.path.join(base_path, task, notebook), as_version=4) + nb = await asyncio.to_thread( + nbformat.read, + os.path.join(base_path, task, notebook), + as_version=nbformat.NO_CONVERT, + ) for cell in nb.cells: if "nbgrader" in cell.metadata and cell.metadata.nbgrader.grade: points += cell.metadata.nbgrader.points @@ -74,11 +79,12 @@ def git_diff(self, pool, task, file): .replace("\n", "
"), ) - def get(self, pool: str, name: str): + async def get(self, pool: str, name: str): path = os.path.join(self.base_path, pool, name) assert os.path.exists(path), "The task does not exists" - points, n_questions = self.__get_task_info(name, pool) - git_status = self.git_status(pool, name) + points, n_questions = await self.__get_task_info(name, pool) + git_status = await asyncio.to_thread(self.git_status, pool, name) + if "repo" in git_status: del git_status["repo"] return Task( @@ -118,13 +124,14 @@ def remove(self, pool, name): ), f"No task with the name {name} from pool {pool} exists." shutil.rmtree(path) - def list(self, pool): + async def list(self, pool): tasks = [] path = os.path.join(self.base_path, pool) assert os.path.exists(path), f"No pool with the name {pool} exists." for task_dir in self.listdir(os.path.join(self.base_path, pool)): - points, n_questions = self.__get_task_info(task_dir, pool) - git_status = self.git_status(pool, task_dir) + points, n_questions = await self.__get_task_info(task_dir, pool) + git_status = await asyncio.to_thread(self.git_status, pool, task_dir) + if "repo" in git_status: del git_status["repo"] tasks.append( @@ -138,11 +145,12 @@ def list(self, pool): ) return tasks - def list_all(self): + async def list_all(self): pool_manager = TaskPoolManager(self.coursedir) tasks = [] for pool in pool_manager.list(): - tasks.extend(self.list(pool.name)) + pool_tasks = await self.list(pool.name) + tasks.extend(pool_tasks) return tasks def copy(self, old_name: str, new_name: str, pool: str = ""): From ff5e38c659f57364219bef53053f78e20ab707c6 Mon Sep 17 00:00:00 2001 From: Tim Daniel Metzler Date: Mon, 7 Oct 2024 15:43:36 +0200 Subject: [PATCH 2/5] Create pool and template dirs if they don't exist --- e2xauthoring/managers/taskpoolmanager.py | 4 +++- e2xauthoring/managers/templatemanager.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/e2xauthoring/managers/taskpoolmanager.py b/e2xauthoring/managers/taskpoolmanager.py index 4ae70b9..341e30e 100644 --- a/e2xauthoring/managers/taskpoolmanager.py +++ b/e2xauthoring/managers/taskpoolmanager.py @@ -55,7 +55,9 @@ def remove(self, name): shutil.rmtree(path) def list(self): - assert os.path.exists(self.base_path), "Pool directory does not exist." + if not os.path.exists(self.base_path): + self.log.warning("The pool directory does not exist.") + os.makedirs(self.base_path, exist_ok=True) return [ TaskPool( name=pool_dir, diff --git a/e2xauthoring/managers/templatemanager.py b/e2xauthoring/managers/templatemanager.py index afeebdc..681a1b4 100644 --- a/e2xauthoring/managers/templatemanager.py +++ b/e2xauthoring/managers/templatemanager.py @@ -49,7 +49,9 @@ def remove(self, name: str): shutil.rmtree(path) def list(self): - assert os.path.exists(self.base_path), "Template directory not found." + if not os.path.exists(self.base_path): + self.log.warning("The template directory does not exist.") + os.makedirs(self.base_path, exist_ok=True) templates = [ Template(name=template_dir) for template_dir in self.listdir(self.base_path) ] From 0804f8d617b12d66208f564158469196b95a3066 Mon Sep 17 00:00:00 2001 From: Tim Daniel Metzler Date: Mon, 7 Oct 2024 15:44:16 +0200 Subject: [PATCH 3/5] Make ApiManagerHandler work with async --- e2xauthoring/app/handlers/base.py | 60 +++++++++++++++++++++---------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/e2xauthoring/app/handlers/base.py b/e2xauthoring/app/handlers/base.py index 16269a8..952960f 100644 --- a/e2xauthoring/app/handlers/base.py +++ b/e2xauthoring/app/handlers/base.py @@ -1,3 +1,4 @@ +import asyncio import inspect from typing import List @@ -17,14 +18,26 @@ def status_msg(method): method: The method to decorate """ - def wrapper(*args, **kwargs): - try: - res = method(*args, **kwargs) - return SuccessMessage(data=res) - except Exception as e: - return ErrorMessage(error=getattr(e, "message", str(e))) + if asyncio.iscoroutinefunction(method): - return wrapper + async def async_wrapper(*args, **kwargs): + try: + res = await method(*args, **kwargs) + return SuccessMessage(data=res) + except Exception as e: + return ErrorMessage(error=getattr(e, "message", str(e))) + + return async_wrapper + else: + + def wrapper(*args, **kwargs): + try: + res = method(*args, **kwargs) + return SuccessMessage(data=res) + except Exception as e: + return ErrorMessage(error=getattr(e, "message", str(e))) + + return wrapper class ApiManageHandler(E2xApiHandler): @@ -59,16 +72,21 @@ def extract_arguments(self, method: str): params[arg] = argument return params - def perform_action(self, action: str, allowed_actions: List[str]): + async def perform_action(self, action: str, allowed_actions: List[str]): assert ( action is not None and action in allowed_actions ), f"Action {action} is not a valid action." method = getattr(self.__manager, action) arguments = self.extract_arguments(method) - return method(**arguments) + + # If the method is async, await it, otherwise call it synchronously + if inspect.iscoroutinefunction(method): + return await method(**arguments) + else: + return method(**arguments) @status_msg - def handle_request(self, request_type: str): + async def handle_request(self, request_type: str): action = self.get_argument( "action", default=None # self.__allowed_actions[request_type]["default"] ) @@ -77,26 +95,30 @@ def handle_request(self, request_type: str): "action", self.__allowed_actions[request_type]["default"] ) - return self.perform_action( + return await self.perform_action( action, self.__allowed_actions[request_type]["actions"] ) @web.authenticated @check_xsrf - def get(self): - self.finish(self.handle_request("get").json()) + async def get(self): + result = await self.handle_request("get") + self.finish(result.json()) @web.authenticated @check_xsrf - def delete(self): - self.finish(self.handle_request("delete").json()) + async def delete(self): + result = await self.handle_request("delete") + self.finish(result.json()) @web.authenticated @check_xsrf - def put(self): - self.finish(self.handle_request("put").json()) + async def put(self): + result = await self.handle_request("put") + self.finish(result.json()) @web.authenticated @check_xsrf - def post(self): - self.finish(self.handle_request("post").json()) + async def post(self): + result = await self.handle_request("post") + self.finish(result.json()) From 4b088d8b03704761faadecfc7abef15dfffdf769 Mon Sep 17 00:00:00 2001 From: Tim Daniel Metzler Date: Mon, 7 Oct 2024 15:44:27 +0200 Subject: [PATCH 4/5] Update package-lock.json --- package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package-lock.json b/package-lock.json index d8cf707..2abb410 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "e2xauthoring", - "version": "0.2.1", + "version": "0.2.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "e2xauthoring", - "version": "0.2.1", + "version": "0.2.2", "hasInstallScript": true, "license": "MIT", "workspaces": [ @@ -24431,7 +24431,7 @@ }, "packages/api": { "name": "@e2xauthoring/api", - "version": "0.2.1", + "version": "0.2.2", "license": "MIT", "devDependencies": { "webpack": "^5.73.0", @@ -24440,9 +24440,9 @@ }, "packages/app": { "name": "@e2xauthoring/app", - "version": "0.2.1", + "version": "0.2.2", "dependencies": { - "@e2xauthoring/api": "0.2.1", + "@e2xauthoring/api": "0.2.2", "@emotion/react": "^11.10.0", "@emotion/styled": "^11.10.0", "@fontsource/roboto": "^4.5.8", @@ -25836,7 +25836,7 @@ "@e2xauthoring/app": { "version": "file:packages/app", "requires": { - "@e2xauthoring/api": "0.2.1", + "@e2xauthoring/api": "0.2.2", "@emotion/react": "^11.10.0", "@emotion/styled": "^11.10.0", "@fontsource/roboto": "^4.5.8", From 0dbcc56a0809334f514f6071562faae666ff79a3 Mon Sep 17 00:00:00 2001 From: Tim Daniel Metzler Date: Mon, 7 Oct 2024 15:49:44 +0200 Subject: [PATCH 5/5] Make list pools async --- e2xauthoring/managers/taskpoolmanager.py | 42 +++++++++++++++--------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/e2xauthoring/managers/taskpoolmanager.py b/e2xauthoring/managers/taskpoolmanager.py index 341e30e..a34d6a4 100644 --- a/e2xauthoring/managers/taskpoolmanager.py +++ b/e2xauthoring/managers/taskpoolmanager.py @@ -1,3 +1,4 @@ +import asyncio import os import shutil @@ -13,14 +14,16 @@ class TaskPoolManager(BaseManager): "pools", help="The relative directory where the pools are stored" ) - def __get_n_tasks(self, name) -> int: - return len( - [ - d - for d in os.listdir(os.path.join(self.base_path, name)) - if not d.startswith(".") - ] - ) + async def __get_n_tasks(self, name) -> int: + base_path = os.path.join(self.base_path, name) + + # Offload os.listdir to a thread + directory_list = await asyncio.to_thread(os.listdir, base_path) + + # Filter out directories that start with a dot ('.') + task_count = len([d for d in directory_list if not d.startswith(".")]) + + return task_count def turn_into_repository(self, pool): path = os.path.join(self.base_path, pool) @@ -54,15 +57,22 @@ def remove(self, name): assert os.path.exists(path), f"The task pool {name} does not exist" shutil.rmtree(path) - def list(self): + async def list(self): if not os.path.exists(self.base_path): self.log.warning("The pool directory does not exist.") os.makedirs(self.base_path, exist_ok=True) - return [ - TaskPool( - name=pool_dir, - n_tasks=self.__get_n_tasks(pool_dir), - is_repo=is_version_controlled(os.path.join(self.base_path, pool_dir)), + pool_dirs = await asyncio.to_thread(self.listdir, self.base_path) + tasks = [] + for pool_dir in pool_dirs: + n_tasks = await self.__get_n_tasks(pool_dir) + is_repo = await asyncio.to_thread( + is_version_controlled, os.path.join(self.base_path, pool_dir) + ) + tasks.append( + TaskPool( + name=pool_dir, + n_tasks=n_tasks, + is_repo=is_repo, + ) ) - for pool_dir in self.listdir(self.base_path) - ] + return tasks