diff --git a/app/api/api/settings.py b/app/api/api/settings.py index b49debb0..b6b375bb 100644 --- a/app/api/api/settings.py +++ b/app/api/api/settings.py @@ -56,6 +56,7 @@ "model_control", "vector_db_control", "corsheaders", + "logs_control", ] MIDDLEWARE = [ diff --git a/app/api/api/urls.py b/app/api/api/urls.py index 28e6d331..28ee8856 100644 --- a/app/api/api/urls.py +++ b/app/api/api/urls.py @@ -30,4 +30,5 @@ path("models/", include("model_control.urls")), path('reset_board/', include('docker_control.urls')), path("collections/", include("vector_db_control.urls")), + path("logs/", include("logs_control.urls")), ] diff --git a/app/api/logs_control/__init__.py b/app/api/logs_control/__init__.py new file mode 100644 index 00000000..deca65cf --- /dev/null +++ b/app/api/logs_control/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC \ No newline at end of file diff --git a/app/api/logs_control/urls.py b/app/api/logs_control/urls.py new file mode 100644 index 00000000..47f333ff --- /dev/null +++ b/app/api/logs_control/urls.py @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC + +from django.urls import path +from .views import ListLogsView, GetLogView + +urlpatterns = [ + path('', ListLogsView.as_view(), name='list_logs'), + path('/', GetLogView.as_view(), name='get_log'), # Use for multi-directory paths +] \ No newline at end of file diff --git a/app/api/logs_control/views.py b/app/api/logs_control/views.py new file mode 100644 index 00000000..428a5a2b --- /dev/null +++ b/app/api/logs_control/views.py @@ -0,0 +1,81 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC + +import os +from urllib.parse import unquote +from django.http import JsonResponse, HttpResponse, Http404 +from rest_framework.views import APIView +from shared_config.logger_config import get_logger + +# Setting up logger +logger = get_logger(__name__) + +# Use environment variable for the base storage volume +LOGS_ROOT = os.getenv("INTERNAL_PERSISTENT_STORAGE_VOLUME", "/path/to/fallback") + +class ListLogsView(APIView): + """ + Lists all available directories and log files within the base logs directory + """ + def get(self, request, *args, **kwargs): + logger.info("ListLogsView endpoint hit") + try: + logs_tree = self.build_logs_tree(LOGS_ROOT) + logger.info(f"Log tree built: {logs_tree}") + return JsonResponse({'logs': logs_tree}, status=200) + except Exception as e: + logger.error(f"Error listing logs: {e}") + return JsonResponse({'error': str(e)}, status=500) + + def build_logs_tree(self, directory): + """ + Recursively build a tree of directories and files. + """ + tree = [] + for entry in os.listdir(directory): + path = os.path.join(directory, entry) + if os.path.isdir(path): + tree.append({ + "name": entry, + "type": "directory", + "children": self.build_logs_tree(path) + }) + elif entry.endswith(".log"): + tree.append({ + "name": entry, + "type": "file" + }) + return tree + + +class GetLogView(APIView): + """ + Retrieves the content of a specific log file from the logs directory. + """ + def get(self, request, filename, *args, **kwargs): + + decoded_filename = unquote(filename) + + file_path = os.path.normpath(os.path.join(LOGS_ROOT, decoded_filename)) + + # Security check: Ensure the resolved path is within LOGS_ROOT + if not file_path.startswith(os.path.abspath(LOGS_ROOT)): + logger.error(f"Invalid log file path: {file_path}") + raise Http404("Invalid file path.") + + logger.info(f"Looking for log file at: {file_path}") + + + if os.path.exists(file_path) and os.path.isfile(file_path): + try: + with open(file_path, 'r') as file: + content = file.read() + logger.info(f"Successfully retrieved content for log: {decoded_filename}") + return HttpResponse(content, content_type='text/plain') + except Exception as e: + logger.error(f"Error reading log file {decoded_filename}: {e}") + return JsonResponse({'error': str(e)}, status=500) + else: + logger.error(f"Log file {decoded_filename} not found at {file_path}") + raise Http404(f"Log file {decoded_filename} not found.") + diff --git a/app/api/model_control/model_utils.py b/app/api/model_control/model_utils.py index dfc846ed..3c7669fe 100644 --- a/app/api/model_control/model_utils.py +++ b/app/api/model_control/model_utils.py @@ -34,21 +34,34 @@ def get_all_records(): data = get_all_records() return data - def stream_response_from_external_api(url, json_data): logger.info(f"stream_response_from_external_api to: url={url}") try: headers = {"Authorization": f"Bearer {encoded_jwt}"} + logger.info(f"stream_response_from_external_api headers:={headers}") + logger.info(f"stream_response_from_external_api json_data:={json_data}") + json_data["temperature"] = 1 + json_data["max_tokens"] = 128 + logger.info(f"added extra token and temp!:={json_data}") + with requests.post( url, json=json_data, headers=headers, stream=True, timeout=None ) as response: + logger.info(f"stream_response_from_external_api response:={response}") response.raise_for_status() logger.info(f"response.headers:={response.headers}") logger.info(f"response.encoding:={response.encoding}") # only allow HTTP 1.1 chunked encoding assert response.headers.get("transfer-encoding") == "chunked" - # Note: chunk_size=None must be passed or it will chunk single chars + + # Stream chunks for chunk in response.iter_content(chunk_size=None, decode_unicode=True): + logger.info(f"stream_response_from_external_api chunk:={chunk}") yield chunk + + # Append the custom end marker after the last chunk + yield "<>" # Custom marker to signal end of stream + + logger.info("stream_response_from_external done") except requests.RequestException as e: yield str(e) diff --git a/app/api/shared_config/model_config.py b/app/api/shared_config/model_config.py index 526712ac..c62856ed 100644 --- a/app/api/shared_config/model_config.py +++ b/app/api/shared_config/model_config.py @@ -25,9 +25,7 @@ class ModelImpl: model_id: str image_name: str image_tag: str - device_configurations: Set[ - "DeviceConfigurations" - ] # Assuming DeviceConfigurations is an enum or similar + device_configurations: Set["DeviceConfigurations"] docker_config: Dict[str, Any] user_uid: int # user inside docker container uid (for file permissions) user_gid: int # user inside docker container gid (for file permissions) @@ -41,6 +39,10 @@ def __post_init__(self): self.docker_config["environment"]["HF_HOME"] = Path( backend_config.model_container_cache_root ).joinpath("huggingface") + + # Set environment variable if N150 or N300x4 is in the device configurations + if DeviceConfigurations.N150 in self.device_configurations or DeviceConfigurations.N300x4 in self.device_configurations: + self.docker_config["environment"]["WH_ARCH_YAML"] = "wormhole_b0_80_arch_eth_dispatch.yaml" @property def image_version(self) -> str: @@ -150,9 +152,34 @@ def base_docker_config(): service_port=7000, service_route="/inference/falcon7b", ), + ModelImpl( + model_name="Llama3.1-70bv0.0.1-instruct", + model_id="id_tt-metal-llama-3.1-70b-instructv0.0.1", + image_name="ghcr.io/tenstorrent/tt-inference-server/tt-metal-llama3-70b-src-base-inference", + image_tag="v0.0.1-tt-metal-v0.52.0-rc31-9d3be887987b", + device_configurations={DeviceConfigurations.N300x4}, + docker_config=base_docker_config(), + user_uid=1000, + user_gid=1000, + shm_size="32G", + service_port=7000, + service_route="/inference/llama3-70b", + ), + ModelImpl( + model_name="Mistral7B-instruct-v0.2", + model_id="id_tt-metal-mistral-7bv0.0.2", + image_name="ghcr.io/tenstorrent/tt-inference-server/tt-metal-mistral-7b-src-base", + image_tag="v0.0.3-tt-metal-v0.52.0-rc33", + device_configurations={DeviceConfigurations.N300x4}, + docker_config=base_docker_config(), + user_uid=1000, + user_gid=1000, + shm_size="32G", + service_port=7000, + service_route="/inference/mistral7b", + ) ] - def validate_model_implemenation_config(impl): # no / in model_id strings, model_id will be used in path names assert not "/" in impl.model_id diff --git a/app/frontend/index.html b/app/frontend/index.html index 383f2f4e..d2de22e2 100644 --- a/app/frontend/index.html +++ b/app/frontend/index.html @@ -1,10 +1,10 @@ - + - llm studio v0.0 + llm studio v1.0
diff --git a/app/frontend/package-lock.json b/app/frontend/package-lock.json index 72ad382c..4fb719a1 100644 --- a/app/frontend/package-lock.json +++ b/app/frontend/package-lock.json @@ -26,7 +26,7 @@ "@radix-ui/react-separator": "^1.0.3", "@radix-ui/react-slot": "^1.1.0", "@radix-ui/react-tooltip": "^1.0.7", - "@tanstack/react-query": "^5.35.1", + "@tanstack/react-query": "^5.56.2", "axios": "^1.6.8", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", @@ -2234,9 +2234,9 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.19.0.tgz", - "integrity": "sha512-JlPfZ/C7yn5S5p0yKk7uhHTTnFlvTgLetl2VxqE518QgyM7C9bSfFTYvB/Q/ftkq0RIPY4ySxTz+/wKJ/dXC0w==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.21.3.tgz", + "integrity": "sha512-MmKSfaB9GX+zXl6E8z4koOr/xU63AMVleLEa64v7R0QF/ZloMs5vcD1sHgM64GXXS1csaJutG+ddtzcueI/BLg==", "cpu": [ "arm" ], @@ -2247,9 +2247,9 @@ ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.19.0.tgz", - "integrity": "sha512-RDxUSY8D1tWYfn00DDi5myxKgOk6RvWPxhmWexcICt/MEC6yEMr4HNCu1sXXYLw8iAsg0D44NuU+qNq7zVWCrw==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.21.3.tgz", + "integrity": "sha512-zrt8ecH07PE3sB4jPOggweBjJMzI1JG5xI2DIsUbkA+7K+Gkjys6eV7i9pOenNSDJH3eOr/jLb/PzqtmdwDq5g==", "cpu": [ "arm64" ], @@ -2260,9 +2260,9 @@ ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.19.0.tgz", - "integrity": "sha512-emvKHL4B15x6nlNTBMtIaC9tLPRpeA5jMvRLXVbl/W9Ie7HhkrE7KQjvgS9uxgatL1HmHWDXk5TTS4IaNJxbAA==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.21.3.tgz", + "integrity": "sha512-P0UxIOrKNBFTQaXTxOH4RxuEBVCgEA5UTNV6Yz7z9QHnUJ7eLX9reOd/NYMO3+XZO2cco19mXTxDMXxit4R/eQ==", "cpu": [ "arm64" ], @@ -2273,9 +2273,9 @@ ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.19.0.tgz", - "integrity": "sha512-fO28cWA1dC57qCd+D0rfLC4VPbh6EOJXrreBmFLWPGI9dpMlER2YwSPZzSGfq11XgcEpPukPTfEVFtw2q2nYJg==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.21.3.tgz", + "integrity": "sha512-L1M0vKGO5ASKntqtsFEjTq/fD91vAqnzeaF6sfNAy55aD+Hi2pBI5DKwCO+UNDQHWsDViJLqshxOahXyLSh3EA==", "cpu": [ "x64" ], @@ -2286,9 +2286,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.19.0.tgz", - "integrity": "sha512-2Rn36Ubxdv32NUcfm0wB1tgKqkQuft00PtM23VqLuCUR4N5jcNWDoV5iBC9jeGdgS38WK66ElncprqgMUOyomw==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.21.3.tgz", + "integrity": "sha512-btVgIsCjuYFKUjopPoWiDqmoUXQDiW2A4C3Mtmp5vACm7/GnyuprqIDPNczeyR5W8rTXEbkmrJux7cJmD99D2g==", "cpu": [ "arm" ], @@ -2299,9 +2299,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.19.0.tgz", - "integrity": "sha512-gJuzIVdq/X1ZA2bHeCGCISe0VWqCoNT8BvkQ+BfsixXwTOndhtLUpOg0A1Fcx/+eA6ei6rMBzlOz4JzmiDw7JQ==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.21.3.tgz", + "integrity": "sha512-zmjbSphplZlau6ZTkxd3+NMtE4UKVy7U4aVFMmHcgO5CUbw17ZP6QCgyxhzGaU/wFFdTfiojjbLG3/0p9HhAqA==", "cpu": [ "arm" ], @@ -2312,9 +2312,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.19.0.tgz", - "integrity": "sha512-0EkX2HYPkSADo9cfeGFoQ7R0/wTKb7q6DdwI4Yn/ULFE1wuRRCHybxpl2goQrx4c/yzK3I8OlgtBu4xvted0ug==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.21.3.tgz", + "integrity": "sha512-nSZfcZtAnQPRZmUkUQwZq2OjQciR6tEoJaZVFvLHsj0MF6QhNMg0fQ6mUOsiCUpTqxTx0/O6gX0V/nYc7LrgPw==", "cpu": [ "arm64" ], @@ -2325,9 +2325,9 @@ ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.19.0.tgz", - "integrity": "sha512-GlIQRj9px52ISomIOEUq/IojLZqzkvRpdP3cLgIE1wUWaiU5Takwlzpz002q0Nxxr1y2ZgxC2obWxjr13lvxNQ==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.21.3.tgz", + "integrity": "sha512-MnvSPGO8KJXIMGlQDYfvYS3IosFN2rKsvxRpPO2l2cum+Z3exiExLwVU+GExL96pn8IP+GdH8Tz70EpBhO0sIQ==", "cpu": [ "arm64" ], @@ -2338,9 +2338,9 @@ ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.19.0.tgz", - "integrity": "sha512-N6cFJzssruDLUOKfEKeovCKiHcdwVYOT1Hs6dovDQ61+Y9n3Ek4zXvtghPPelt6U0AH4aDGnDLb83uiJMkWYzQ==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.21.3.tgz", + "integrity": "sha512-+W+p/9QNDr2vE2AXU0qIy0qQE75E8RTwTwgqS2G5CRQ11vzq0tbnfBd6brWhS9bCRjAjepJe2fvvkvS3dno+iw==", "cpu": [ "ppc64" ], @@ -2351,9 +2351,9 @@ ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.19.0.tgz", - "integrity": "sha512-2DnD3mkS2uuam/alF+I7M84koGwvn3ZVD7uG+LEWpyzo/bq8+kKnus2EVCkcvh6PlNB8QPNFOz6fWd5N8o1CYg==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.21.3.tgz", + "integrity": "sha512-yXH6K6KfqGXaxHrtr+Uoy+JpNlUlI46BKVyonGiaD74ravdnF9BUNC+vV+SIuB96hUMGShhKV693rF9QDfO6nQ==", "cpu": [ "riscv64" ], @@ -2364,9 +2364,9 @@ ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.19.0.tgz", - "integrity": "sha512-D6pkaF7OpE7lzlTOFCB2m3Ngzu2ykw40Nka9WmKGUOTS3xcIieHe82slQlNq69sVB04ch73thKYIWz/Ian8DUA==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.21.3.tgz", + "integrity": "sha512-R8cwY9wcnApN/KDYWTH4gV/ypvy9yZUHlbJvfaiXSB48JO3KpwSpjOGqO4jnGkLDSk1hgjYkTbTt6Q7uvPf8eg==", "cpu": [ "s390x" ], @@ -2377,9 +2377,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.19.0.tgz", - "integrity": "sha512-HBndjQLP8OsdJNSxpNIN0einbDmRFg9+UQeZV1eiYupIRuZsDEoeGU43NQsS34Pp166DtwQOnpcbV/zQxM+rWA==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.21.3.tgz", + "integrity": "sha512-kZPbX/NOPh0vhS5sI+dR8L1bU2cSO9FgxwM8r7wHzGydzfSjLRCFAT87GR5U9scj2rhzN3JPYVC7NoBbl4FZ0g==", "cpu": [ "x64" ], @@ -2390,9 +2390,9 @@ ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.19.0.tgz", - "integrity": "sha512-HxfbvfCKJe/RMYJJn0a12eiOI9OOtAUF4G6ozrFUK95BNyoJaSiBjIOHjZskTUffUrB84IPKkFG9H9nEvJGW6A==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.21.3.tgz", + "integrity": "sha512-S0Yq+xA1VEH66uiMNhijsWAafffydd2X5b77eLHfRmfLsRSpbiAWiRHV6DEpz6aOToPsgid7TI9rGd6zB1rhbg==", "cpu": [ "x64" ], @@ -2403,9 +2403,9 @@ ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.19.0.tgz", - "integrity": "sha512-HxDMKIhmcguGTiP5TsLNolwBUK3nGGUEoV/BO9ldUBoMLBssvh4J0X8pf11i1fTV7WShWItB1bKAKjX4RQeYmg==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.21.3.tgz", + "integrity": "sha512-9isNzeL34yquCPyerog+IMCNxKR8XYmGd0tHSV+OVx0TmE0aJOo9uw4fZfUuk2qxobP5sug6vNdZR6u7Mw7Q+Q==", "cpu": [ "arm64" ], @@ -2416,9 +2416,9 @@ ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.19.0.tgz", - "integrity": "sha512-xItlIAZZaiG/u0wooGzRsx11rokP4qyc/79LkAOdznGRAbOFc+SfEdfUOszG1odsHNgwippUJavag/+W/Etc6Q==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.21.3.tgz", + "integrity": "sha512-nMIdKnfZfzn1Vsk+RuOvl43ONTZXoAPUUxgcU0tXooqg4YrAqzfKzVenqqk2g5efWh46/D28cKFrOzDSW28gTA==", "cpu": [ "ia32" ], @@ -2429,9 +2429,9 @@ ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.19.0.tgz", - "integrity": "sha512-xNo5fV5ycvCCKqiZcpB65VMR11NJB+StnxHz20jdqRAktfdfzhgjTiJ2doTDQE/7dqGaV5I7ZGqKpgph6lCIag==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.21.3.tgz", + "integrity": "sha512-fOvu7PCQjAj4eWDEuD8Xz5gpzFqXzGlxHZozHP4b9Jxv9APtdxL6STqztDzMLuRXEc4UpXGGhx029Xgm91QBeA==", "cpu": [ "x64" ], @@ -2719,27 +2719,27 @@ } }, "node_modules/@tanstack/query-core": { - "version": "5.51.9", - "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.51.9.tgz", - "integrity": "sha512-HsAwaY5J19MD18ykZDS3aVVh+bAt0i7m6uQlFC2b77DLV9djo+xEN7MWQAQQTR8IM+7r/zbozTQ7P0xr0bHuew==", + "version": "5.56.2", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.56.2.tgz", + "integrity": "sha512-gor0RI3/R5rVV3gXfddh1MM+hgl0Z4G7tj6Xxpq6p2I03NGPaJ8dITY9Gz05zYYb/EJq9vPas/T4wn9EaDPd4Q==", "funding": { "type": "github", "url": "https://github.com/sponsors/tannerlinsley" } }, "node_modules/@tanstack/react-query": { - "version": "5.51.11", - "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.51.11.tgz", - "integrity": "sha512-4Kq2x0XpDlpvSnaLG+8pHNH60zEc3mBvb3B2tOMDjcPCi/o+Du3p/9qpPLwJOTliVxxPJAP27fuIhLrsRdCr7A==", + "version": "5.56.2", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.56.2.tgz", + "integrity": "sha512-SR0GzHVo6yzhN72pnRhkEFRAHMsUo5ZPzAxfTMvUxFIDVS6W9LYUp6nXW3fcHVdg0ZJl8opSH85jqahvm6DSVg==", "dependencies": { - "@tanstack/query-core": "5.51.9" + "@tanstack/query-core": "5.56.2" }, "funding": { "type": "github", "url": "https://github.com/sponsors/tannerlinsley" }, "peerDependencies": { - "react": "^18.0.0" + "react": "^18 || ^19" } }, "node_modules/@tsconfig/node10": { @@ -6535,9 +6535,9 @@ } }, "node_modules/picocolors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz", - "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.0.tgz", + "integrity": "sha512-TQ92mBOW0l3LeMeyLV6mzy/kWr8lkd/hp3mTg7wYK7zJhuBStmGMBG0BdeDZS/dZx1IukaX6Bk11zcln25o1Aw==" }, "node_modules/picomatch": { "version": "2.3.1", @@ -6567,9 +6567,9 @@ } }, "node_modules/postcss": { - "version": "8.4.41", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", - "integrity": "sha512-TesUflQ0WKZqAvg52PWL6kHgLKP6xB6heTOdoYM0Wt2UHyxNa4K25EZZMgKns3BH1RLVbZCREPpLY0rhnNoHVQ==", + "version": "8.4.47", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.47.tgz", + "integrity": "sha512-56rxCq7G/XfB4EkXq9Egn5GCqugWvDFjafDOThIdMBsI15iqPqR5r15TfSr1YPYeEI19YeaXMCbY6u88Y76GLQ==", "funding": [ { "type": "opencollective", @@ -6586,8 +6586,8 @@ ], "dependencies": { "nanoid": "^3.3.7", - "picocolors": "^1.0.1", - "source-map-js": "^1.2.0" + "picocolors": "^1.1.0", + "source-map-js": "^1.2.1" }, "engines": { "node": "^10 || ^12 || >=14" @@ -7230,9 +7230,9 @@ } }, "node_modules/rollup": { - "version": "4.19.0", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.19.0.tgz", - "integrity": "sha512-5r7EYSQIowHsK4eTZ0Y81qpZuJz+MUuYeqmmYmRMl1nwhdmbiYqt5jwzf6u7wyOzJgYqtCRMtVRKOtHANBz7rA==", + "version": "4.21.3", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.21.3.tgz", + "integrity": "sha512-7sqRtBNnEbcBtMeRVc6VRsJMmpI+JU1z9VTvW8D4gXIYQFz0aLcsE6rRkyghZkLfEgUZgVvOG7A5CVz/VW5GIA==", "dev": true, "dependencies": { "@types/estree": "1.0.5" @@ -7245,22 +7245,22 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.19.0", - "@rollup/rollup-android-arm64": "4.19.0", - "@rollup/rollup-darwin-arm64": "4.19.0", - "@rollup/rollup-darwin-x64": "4.19.0", - "@rollup/rollup-linux-arm-gnueabihf": "4.19.0", - "@rollup/rollup-linux-arm-musleabihf": "4.19.0", - "@rollup/rollup-linux-arm64-gnu": "4.19.0", - "@rollup/rollup-linux-arm64-musl": "4.19.0", - "@rollup/rollup-linux-powerpc64le-gnu": "4.19.0", - "@rollup/rollup-linux-riscv64-gnu": "4.19.0", - "@rollup/rollup-linux-s390x-gnu": "4.19.0", - "@rollup/rollup-linux-x64-gnu": "4.19.0", - "@rollup/rollup-linux-x64-musl": "4.19.0", - "@rollup/rollup-win32-arm64-msvc": "4.19.0", - "@rollup/rollup-win32-ia32-msvc": "4.19.0", - "@rollup/rollup-win32-x64-msvc": "4.19.0", + "@rollup/rollup-android-arm-eabi": "4.21.3", + "@rollup/rollup-android-arm64": "4.21.3", + "@rollup/rollup-darwin-arm64": "4.21.3", + "@rollup/rollup-darwin-x64": "4.21.3", + "@rollup/rollup-linux-arm-gnueabihf": "4.21.3", + "@rollup/rollup-linux-arm-musleabihf": "4.21.3", + "@rollup/rollup-linux-arm64-gnu": "4.21.3", + "@rollup/rollup-linux-arm64-musl": "4.21.3", + "@rollup/rollup-linux-powerpc64le-gnu": "4.21.3", + "@rollup/rollup-linux-riscv64-gnu": "4.21.3", + "@rollup/rollup-linux-s390x-gnu": "4.21.3", + "@rollup/rollup-linux-x64-gnu": "4.21.3", + "@rollup/rollup-linux-x64-musl": "4.21.3", + "@rollup/rollup-win32-arm64-msvc": "4.21.3", + "@rollup/rollup-win32-ia32-msvc": "4.21.3", + "@rollup/rollup-win32-x64-msvc": "4.21.3", "fsevents": "~2.3.2" } }, @@ -7433,9 +7433,9 @@ } }, "node_modules/source-map-js": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.0.tgz", - "integrity": "sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", "engines": { "node": ">=0.10.0" } @@ -8181,14 +8181,14 @@ } }, "node_modules/vite": { - "version": "5.3.4", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.3.4.tgz", - "integrity": "sha512-Cw+7zL3ZG9/NZBB8C+8QbQZmR54GwqIz+WMI4b3JgdYJvX+ny9AjJXqkGQlDXSXRP9rP0B4tbciRMOVEKulVOA==", + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.6.tgz", + "integrity": "sha512-IeL5f8OO5nylsgzd9tq4qD2QqI0k2CQLGrWD0rCN0EQJZpBK5vJAx0I+GDkMOXxQX/OfFHMuLIx6ddAxGX/k+Q==", "dev": true, "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.39", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" }, "bin": { "vite": "bin/vite.js" @@ -8207,6 +8207,7 @@ "less": "*", "lightningcss": "^1.21.0", "sass": "*", + "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" @@ -8224,6 +8225,9 @@ "sass": { "optional": true }, + "sass-embedded": { + "optional": true + }, "stylus": { "optional": true }, diff --git a/app/frontend/package.json b/app/frontend/package.json index cd62af72..aad719ee 100644 --- a/app/frontend/package.json +++ b/app/frontend/package.json @@ -29,7 +29,7 @@ "@radix-ui/react-separator": "^1.0.3", "@radix-ui/react-slot": "^1.1.0", "@radix-ui/react-tooltip": "^1.0.7", - "@tanstack/react-query": "^5.35.1", + "@tanstack/react-query": "^5.56.2", "axios": "^1.6.8", "class-variance-authority": "^0.7.0", "clsx": "^2.1.1", diff --git a/app/frontend/src/api/modelsDeployedApis.ts b/app/frontend/src/api/modelsDeployedApis.ts index f37e436c..9bfaf4dc 100644 --- a/app/frontend/src/api/modelsDeployedApis.ts +++ b/app/frontend/src/api/modelsDeployedApis.ts @@ -117,7 +117,7 @@ export const deleteModel = async (modelId: string): Promise => { `Model ID: ${truncatedModelId} has been reset successfully.`, ); } else { - customToast.error(`Model ID: ${truncatedModelId} reset failed.`); + customToast.error(`Board Reset failed.`); } console.log( diff --git a/app/frontend/src/components/ChatComponent.tsx b/app/frontend/src/components/ChatComponent.tsx deleted file mode 100644 index f50bef4d..00000000 --- a/app/frontend/src/components/ChatComponent.tsx +++ /dev/null @@ -1,422 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -// SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC -import React, { useEffect, useState, useRef } from "react"; -import { Card } from "./ui/card"; -import { Button } from "./ui/button"; -import * as ScrollArea from "@radix-ui/react-scroll-area"; -import { useLocation } from "react-router-dom"; -import { Spinner } from "./ui/spinner"; -import { User, ChevronDown } from "lucide-react"; -import { Textarea } from "./ui/textarea"; -import logo from "../assets/tt_logo.svg"; -import { - Breadcrumb, - BreadcrumbEllipsis, - BreadcrumbItem, - BreadcrumbLink, - BreadcrumbList, - BreadcrumbPage, - BreadcrumbSeparator, -} from "./ui/breadcrumb"; -import { - DropdownMenu, - DropdownMenuContent, - DropdownMenuItem, - DropdownMenuTrigger, -} from "./ui/dropdown-menu"; -import { fetchModels } from "../api/modelsDeployedApis"; -import ChatExamples from "./ChatExamples"; -import axios from "axios"; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from "./ui/select"; -import { useQuery } from "react-query"; -import { fetchCollections } from "@/src/pages/rag/"; - -interface InferenceRequest { - deploy_id: string; - text: string; - rag_context?: { documents: string[] }; -} - -interface RagDataSource { - id: string; - name: string; - metadata: Record; -} - -interface ChatMessage { - sender: "user" | "assistant"; - text: string; -} - -interface Model { - id: string; - name: string; -} - -const ChatComponent: React.FC = () => { - const location = useLocation(); - const [textInput, setTextInput] = useState(""); - - const [ragDatasource, setRagDatasource] = useState< - RagDataSource | undefined - >(); - - // Fetch collections - const { data: ragDataSources } = useQuery("collectionsList", { - queryFn: fetchCollections, - initialData: [], - }); - - const [chatHistory, setChatHistory] = useState([]); - const [modelID, setModelID] = useState(null); - const [modelName, setModelName] = useState(null); - const [isStreaming, setIsStreaming] = useState(false); - const viewportRef = useRef(null); - const bottomRef = useRef(null); - const [isScrollButtonVisible, setIsScrollButtonVisible] = useState(false); - const [modelsDeployed, setModelsDeployed] = useState([]); - - useEffect(() => { - if (location.state) { - setModelID(location.state.containerID); - setModelName(location.state.modelName); - } - - const loadModels = async () => { - try { - const models = await fetchModels(); - setModelsDeployed(models); - } catch (error) { - console.error("Error fetching models:", error); - } - }; - - loadModels(); - }, [location.state]); - - const scrollToBottom = () => { - if (viewportRef.current) { - viewportRef.current.scrollTo({ - top: viewportRef.current.scrollHeight, - behavior: "smooth", - }); - } - }; - - const handleScroll = () => { - if (viewportRef.current) { - const isAtBottom = - viewportRef.current.scrollHeight - viewportRef.current.scrollTop <= - viewportRef.current.clientHeight + 1; - setIsScrollButtonVisible(!isAtBottom); - } - }; - - const getRagContext = async (request: InferenceRequest) => { - const ragContext: { documents: string[] } = { - documents: [], - }; - - const response = await axios - .get(`/collections-api/${ragDatasource?.name}/query`, { - params: { query: request.text }, - }) - .catch((e) => { - console.error(`Error fetching RAG context ${e}`); - }); - - if (!response?.data) { - return ragContext; - } - - ragContext.documents = response.data.documents; - return ragContext; - }; - - const runInference = async (request: InferenceRequest) => { - try { - if (ragDatasource) { - request.rag_context = await getRagContext(request); - } - - setIsStreaming(true); - const response = await fetch(`/models-api/inference/`, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(request), - }); - - const reader = response.body?.getReader(); - - setChatHistory((prevHistory) => [ - ...prevHistory, - { sender: "user", text: textInput }, - ]); - setTextInput(""); - - let result = ""; - if (reader) { - let done = false; - while (!done) { - const { done: streamDone, value } = await reader.read(); - done = streamDone; - - if (value) { - const decoder = new TextDecoder(); - const chunk = decoder.decode(value); - result += chunk; - const cleanedResult = result.replace(/<\|endoftext\|>/g, ""); - setChatHistory((prevHistory) => { - const lastMessage = prevHistory[prevHistory.length - 1]; - if (lastMessage && lastMessage.sender === "assistant") { - const updatedHistory = [...prevHistory]; - updatedHistory[updatedHistory.length - 1] = { - ...lastMessage, - text: cleanedResult, - }; - return updatedHistory; - } else { - return [ - ...prevHistory, - { sender: "assistant", text: cleanedResult }, - ]; - } - }); - } - } - } - - setIsStreaming(false); - } catch (error) { - console.error("Error running inference:", error); - setIsStreaming(false); - } - }; - - const handleInference = () => { - if (textInput.trim() === "") return; - - const inferenceRequest: InferenceRequest = { - deploy_id: modelID!, - text: textInput, - }; - - if (textInput === "Tell me a fun fact.") { - setChatHistory((prevHistory) => [ - ...prevHistory, - { sender: "user", text: textInput }, - { sender: "assistant", text: "Did you know? Honey never spoils." }, - ]); - setTextInput(""); - scrollToBottom(); - return; - } - - runInference(inferenceRequest); - }; - - const RagContextSelector = ({ - collections, - onChange, - activeCollection, - }: { - collections: RagDataSource[]; - activeCollection?: RagDataSource; - onChange: (v: string) => void; - }) => ( -
-
Select RAG Datasource
- - -
- ); - - const handleKeyPress = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleInference(); - } - }; - - return ( -
- -
- - - - - Models Deployed - - - - / - - - - - - Toggle menu - - - {modelsDeployed.map((model) => ( - { - setModelID(model.id); - setModelName(model.name); - }} - > - {model.name} - - ))} - - - - - / - - - - {modelName} - - - - -
-
- { - const dataSource = ragDataSources.find((rds: RagDataSource) => { - return rds.name == v; - }); - if (dataSource) { - setRagDatasource(dataSource); - } - }} - activeCollection={ragDatasource} - /> - {chatHistory.length === 0 && ( - - )} - {chatHistory.length > 0 && ( -
- - - {chatHistory.map((message, index) => ( -
-
-
- {message.sender === "user" ? ( - - ) : ( - Tenstorrent Logo - )} -
-
-
- {message.text} -
-
- ))} -
- - - - - - {isScrollButtonVisible && ( - - )} -
- )} -
-