From 5f8446d38c8435de0822458f8abd830673d3a3ea Mon Sep 17 00:00:00 2001 From: Marcel Klehr Date: Tue, 29 Oct 2024 15:29:00 +0100 Subject: [PATCH 1/3] fix: download model in background task Signed-off-by: Marcel Klehr --- context_chat_backend/controller.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/context_chat_backend/controller.py b/context_chat_backend/controller.py index 4d8fd14..1f1d33e 100644 --- a/context_chat_backend/controller.py +++ b/context_chat_backend/controller.py @@ -19,7 +19,7 @@ from .models import LlmException from .ocs_utils import AppAPIAuthMiddleware from .setup_functions import ensure_config_file, repair_run, setup_env_vars -from .utils import JSONResponse, update_progress, value_of +from .utils import JSONResponse, value_of from .vectordb import BaseVectorDB, DbException # setup @@ -208,8 +208,7 @@ def _(): @app.post('/init') def _(bg_tasks: BackgroundTasks): nc = NextcloudApp() - fetch_models_task(nc, models_to_fetch, 0) - update_progress(app, 100) + bg_tasks.add_task(fetch_models_task, nc, models_to_fetch, 0) return JSONResponse(content={}, status_code=200) From bc97b0e744f3dac06dd529771458b6262ba751ed Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 31 Oct 2024 05:39:48 +0530 Subject: [PATCH 2/3] use nc_py_api's init, heartbeat and enabled endpoints Signed-off-by: Anupam Kumar --- context_chat_backend/controller.py | 46 ++++++++++-------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/context_chat_backend/controller.py b/context_chat_backend/controller.py index 1f1d33e..f036249 100644 --- a/context_chat_backend/controller.py +++ b/context_chat_backend/controller.py @@ -1,16 +1,16 @@ import os import threading +from collections.abc import Callable from contextlib import asynccontextmanager from functools import wraps from logging import error as log_error from threading import Event -from typing import Annotated, Any, Callable +from typing import Annotated, Any -from fastapi import BackgroundTasks, Body, FastAPI, Request, UploadFile +from fastapi import Body, FastAPI, Request, UploadFile from langchain.llms.base import LLM -from nc_py_api import NextcloudApp -from nc_py_api.ex_app import persistent_storage -from nc_py_api.ex_app.integration_fastapi import fetch_models_task +from nc_py_api import AsyncNextcloudApp, NextcloudApp +from nc_py_api.ex_app import persistent_storage, set_handlers from pydantic import BaseModel, ValidationInfo, field_validator from .chain import ContextException, LLMOutput, ScopeType, embed_sources, process_context_query, process_query @@ -35,11 +35,19 @@ } app_enabled = Event() -# disabled for now -# scheduler = AsyncIOScheduler() +def enabled_handler(enabled: bool, _: NextcloudApp | AsyncNextcloudApp) -> str: + if enabled: + app_enabled.set() + else: + app_enabled.clear() + + print('App', 'enabled' if enabled else 'disabled', flush=True) + return '' + @asynccontextmanager async def lifespan(app: FastAPI): + set_handlers(app, enabled_handler, models_to_fetch=models_to_fetch) nc = NextcloudApp() if nc.enabled_state: app_enabled.set() @@ -187,30 +195,6 @@ def _(userId: str, sourceNames: str): def _(): return JSONResponse(content={'enabled': app_enabled.is_set()}, status_code=200) -@app.put('/enabled') -def _(enabled: bool): - - if enabled: - app_enabled.set() - else: - app_enabled.clear() - - print('App', 'enabled' if enabled else 'disabled', flush=True) - return JSONResponse(content={'error': ''}, status_code=200) - - -@app.get('/heartbeat') -def _(): - print('heartbeat_handler: result=ok') - return JSONResponse(content={'status': 'ok'}, status_code=200) - - -@app.post('/init') -def _(bg_tasks: BackgroundTasks): - nc = NextcloudApp() - bg_tasks.add_task(fetch_models_task, nc, models_to_fetch, 0) - return JSONResponse(content={}, status_code=200) - @app.post('/deleteSources') @enabled_guard(app) From 69bd0e8ce4f3819cf9773af247d057abf5deaa0a Mon Sep 17 00:00:00 2001 From: Anupam Kumar Date: Thu, 31 Oct 2024 05:40:18 +0530 Subject: [PATCH 3/3] use commit hash instead of main for em model Signed-off-by: Anupam Kumar --- context_chat_backend/chain/ingest/doc_loader.py | 4 +--- context_chat_backend/controller.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/context_chat_backend/chain/ingest/doc_loader.py b/context_chat_backend/chain/ingest/doc_loader.py index 13f7206..81033c3 100644 --- a/context_chat_backend/chain/ingest/doc_loader.py +++ b/context_chat_backend/chain/ingest/doc_loader.py @@ -1,5 +1,4 @@ import re -import gc import tempfile import traceback from collections.abc import Callable @@ -138,5 +137,4 @@ def decode_source(source: UploadFile) -> str | None: log_error(f'Error decoding source file ({source.filename})') return None finally: - source.file.close() # Ensure file is closed after processing - gc.collect() # Force garbage collection to free up memory + source.file.close() # Ensure file is closed after processing diff --git a/context_chat_backend/controller.py b/context_chat_backend/controller.py index f036249..b076f5a 100644 --- a/context_chat_backend/controller.py +++ b/context_chat_backend/controller.py @@ -29,7 +29,7 @@ ensure_config_file() models_to_fetch = { - "https://huggingface.co/Ralriki/multilingual-e5-large-instruct-GGUF/resolve/main/multilingual-e5-large-instruct-q6_k.gguf": { + "https://huggingface.co/Ralriki/multilingual-e5-large-instruct-GGUF/resolve/8738f8d3d8f311808479ecd5756607e24c6ca811/multilingual-e5-large-instruct-q6_k.gguf": { # noqa: E501 "save_path": os.path.join(persistent_storage(), 'model_files', "multilingual-e5-large-instruct-q6_k.gguf") } }