Anirud/v1 end to end testing

app/api/api/settings.py

@@ -56,6 +56,7 @@
     "model_control",
     "vector_db_control",
     "corsheaders",
+    "logs_control",
 ]
 
 MIDDLEWARE = [

app/api/api/urls.py

@@ -30,4 +30,5 @@
     path("models/", include("model_control.urls")),
     path('reset_board/', include('docker_control.urls')),
     path("collections/", include("vector_db_control.urls")),
+    path("logs/", include("logs_control.urls")),
 ]

app/api/logs_control/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC

app/api/logs_control/urls.py

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
+
+from django.urls import path
+from .views import ListLogsView, GetLogView
+
+urlpatterns = [
+    path('', ListLogsView.as_view(), name='list_logs'),
+    path('<path:filename>/', GetLogView.as_view(), name='get_log'),  # Use <path:filename> for multi-directory paths
+]

app/api/logs_control/views.py

@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC
+
+import os
+from urllib.parse import unquote
+from django.http import JsonResponse, HttpResponse, Http404
+from rest_framework.views import APIView
+from shared_config.logger_config import get_logger
+
+# Setting up logger
+logger = get_logger(__name__)
+
+# Use environment variable for the base storage volume
+LOGS_ROOT = os.getenv("INTERNAL_PERSISTENT_STORAGE_VOLUME", "/path/to/fallback")
+
+class ListLogsView(APIView):
+    """
+    Lists all available directories and log files within the base logs directory
+    """
+    def get(self, request, *args, **kwargs):
+        logger.info("ListLogsView endpoint hit")
+        try:
+            logs_tree = self.build_logs_tree(LOGS_ROOT)
+            logger.info(f"Log tree built: {logs_tree}")
+            return JsonResponse({'logs': logs_tree}, status=200)
+        except Exception as e:
+            logger.error(f"Error listing logs: {e}")
+            return JsonResponse({'error': str(e)}, status=500)
+
+    def build_logs_tree(self, directory):
+        """
+        Recursively build a tree of directories and files.
+        """
+        tree = []
+        for entry in os.listdir(directory):
+            path = os.path.join(directory, entry)
+            if os.path.isdir(path):
+                tree.append({
+                    "name": entry,
+                    "type": "directory",
+                    "children": self.build_logs_tree(path)
+                })
+            elif entry.endswith(".log"):
+                tree.append({
+                    "name": entry,
+                    "type": "file"
+                })
+        return tree
+
+
+class GetLogView(APIView):
+    """
+    Retrieves the content of a specific log file from the logs directory.
+    """
+    def get(self, request, filename, *args, **kwargs):
+
+        decoded_filename = unquote(filename)
+
+        file_path = os.path.normpath(os.path.join(LOGS_ROOT, decoded_filename))
+
+        # Security check: Ensure the resolved path is within LOGS_ROOT
+        if not file_path.startswith(os.path.abspath(LOGS_ROOT)):
+            logger.error(f"Invalid log file path: {file_path}")
+            raise Http404("Invalid file path.")
+
+        logger.info(f"Looking for log file at: {file_path}")
+
+
+        if os.path.exists(file_path) and os.path.isfile(file_path):
+            try:
+                with open(file_path, 'r') as file:
+                    content = file.read()
+                logger.info(f"Successfully retrieved content for log: {decoded_filename}")
+                return HttpResponse(content, content_type='text/plain')
+            except Exception as e:
+                logger.error(f"Error reading log file {decoded_filename}: {e}")
+                return JsonResponse({'error': str(e)}, status=500)
+        else:
+            logger.error(f"Log file {decoded_filename} not found at {file_path}")
+            raise Http404(f"Log file {decoded_filename} not found.")
+

app/api/model_control/model_utils.py

@@ -34,21 +34,34 @@ def get_all_records():
     data = get_all_records()
     return data
 
-
 def stream_response_from_external_api(url, json_data):
     logger.info(f"stream_response_from_external_api to: url={url}")
     try:
         headers = {"Authorization": f"Bearer {encoded_jwt}"}
+        logger.info(f"stream_response_from_external_api headers:={headers}")
+        logger.info(f"stream_response_from_external_api json_data:={json_data}")
+        json_data["temperature"] = 1
+        json_data["max_tokens"] = 128
+        logger.info(f"added extra token and temp!:={json_data}")
+
         with requests.post(
             url, json=json_data, headers=headers, stream=True, timeout=None
         ) as response:
+            logger.info(f"stream_response_from_external_api response:={response}")
             response.raise_for_status()
             logger.info(f"response.headers:={response.headers}")
             logger.info(f"response.encoding:={response.encoding}")
             # only allow HTTP 1.1 chunked encoding
             assert response.headers.get("transfer-encoding") == "chunked"
-            # Note: chunk_size=None must be passed or it will chunk single chars
+
+            # Stream chunks
             for chunk in response.iter_content(chunk_size=None, decode_unicode=True):
+                logger.info(f"stream_response_from_external_api chunk:={chunk}")
                 yield chunk
+
+            # Append the custom end marker after the last chunk
+            yield "<<END_OF_STREAM>>"  # Custom marker to signal end of stream
+
+            logger.info("stream_response_from_external done")
     except requests.RequestException as e:
         yield str(e)

app/api/shared_config/model_config.py

@@ -25,9 +25,7 @@ class ModelImpl:
     model_id: str
     image_name: str
     image_tag: str
-    device_configurations: Set[
-        "DeviceConfigurations"
-    ]  # Assuming DeviceConfigurations is an enum or similar
+    device_configurations: Set["DeviceConfigurations"]
     docker_config: Dict[str, Any]
     user_uid: int  # user inside docker container uid (for file permissions)
     user_gid: int  # user inside docker container gid (for file permissions)
@@ -41,6 +39,10 @@ def __post_init__(self):
         self.docker_config["environment"]["HF_HOME"] = Path(
             backend_config.model_container_cache_root
         ).joinpath("huggingface")
+
+        # Set environment variable if N150 or N300x4 is in the device configurations
+        if DeviceConfigurations.N150 in self.device_configurations or DeviceConfigurations.N300x4 in self.device_configurations:
+            self.docker_config["environment"]["WH_ARCH_YAML"] = "wormhole_b0_80_arch_eth_dispatch.yaml"
 
     @property
     def image_version(self) -> str:
@@ -150,9 +152,34 @@ def base_docker_config():
         service_port=7000,
         service_route="/inference/falcon7b",
     ),
+        ModelImpl(
+        model_name="Llama3.1-70bv0.0.1-instruct",
+        model_id="id_tt-metal-llama-3.1-70b-instructv0.0.1",
+        image_name="ghcr.io/tenstorrent/tt-inference-server/tt-metal-llama3-70b-src-base-inference",
+        image_tag="v0.0.1-tt-metal-v0.52.0-rc31-9d3be887987b",
+        device_configurations={DeviceConfigurations.N300x4},
+        docker_config=base_docker_config(),
+        user_uid=1000,
+        user_gid=1000,
+        shm_size="32G",
+        service_port=7000,
+        service_route="/inference/llama3-70b",
+    ),
+    ModelImpl(
+        model_name="Mistral7B-instruct-v0.2",
+        model_id="id_tt-metal-mistral-7bv0.0.2",
+        image_name="ghcr.io/tenstorrent/tt-inference-server/tt-metal-mistral-7b-src-base",
+        image_tag="v0.0.3-tt-metal-v0.52.0-rc33",
+        device_configurations={DeviceConfigurations.N300x4},
+        docker_config=base_docker_config(),
+        user_uid=1000,
+        user_gid=1000,
+        shm_size="32G",
+        service_port=7000,
+        service_route="/inference/mistral7b",
+    )
 ]
 
-
 def validate_model_implemenation_config(impl):
     # no / in model_id strings, model_id will be used in path names
     assert not "/" in impl.model_id

app/frontend/index.html

@@ -1,10 +1,10 @@
-<!DOCTYPE html>
+<!doctype html>
 <html lang="en">
   <head>
     <meta charset="UTF-8" />
     <link rel="icon" type="image/svg+xml" href="./src/assets/tt_logo.svg" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>llm studio v0.0</title>
+    <title>llm studio v1.0</title>
   </head>
   <body>
     <div id="root"></div>