diff --git a/.github/workflows/backend-license-checker.yml b/.github/workflows/backend-license-checker.yml new file mode 100644 index 00000000..d7cb5ae1 --- /dev/null +++ b/.github/workflows/backend-license-checker.yml @@ -0,0 +1,144 @@ +name: Backend-Models SPDX Licenses Checker + +on: + workflow_dispatch: + workflow_call: + pull_request: + branches: + - "main" + - "staging" + types: + - opened + - reopened + - synchronize + - assigned + - review_requested + +jobs: + check-spdx-licenses: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5.0.0 + with: + cache: "pip" + python-version: "3.8" + + # Install the copyright checking tool + - name: Install copyright check tool + run: pip install git+https://github.com/espressif/check-copyright.git@master + + # Check SPDX licenses + - name: Check SPDX licenses + id: check_spdx_licenses + run: | + set +e + output=$(python -m check_copyright --verbose --dry-run --config ./check_copyright_config.yaml . 2>&1) + exit_code=$? + clean_output=$(echo "$output" | sed 's/\x1b\[[0-9;]*m//g') + echo "CLEAN_OUTPUT<> $GITHUB_ENV + echo "$clean_output" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "EXIT_CODE=$exit_code" >> $GITHUB_ENV + exit 0 + + - name: Debug Extracted Files + run: | + echo "Extracted Files:" + echo "$CLEAN_OUTPUT" + + # Extract files needing SPDX header addition + - name: Extract Files + id: extract_files + run: | + set +e + files=$(echo "$CLEAN_OUTPUT" | awk ' + /Files which failed the copyright check:/, /Additional information about this hook and copyright headers may be found here:/ { + if ($0 !~ /Files which failed the copyright check:/ && $0 !~ /Additional information about this hook and copyright headers may be found here:/) print + } + /Some files are without a copyright note and a license header needs to be added:/, /Additional information about this hook and copyright headers may be found here:/ { + if ($0 !~ /Some files are without a copyright note and a license header needs to be added:/ && $0 !~ /Additional information about this hook and copyright headers may be found here:/) print + }' | sed 's/^ *//' | sed '/^$/d' | grep '^.') + echo "FILES<> $GITHUB_ENV + echo "$files" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "Extracted Files: $files" + exit 0 + + - name: Debug Extracted Files + run: | + echo "Extracted Files:" + echo "$FILES" + + # Run the `add_spdx_header.py` script to fix missing SPDX headers + - name: Run SPDX Header Script + if: env.FILES != '' + run: | + set +e + python add_spdx_header.py # Run the script to add SPDX headers to the missing files + exit 0 + + # Commit the changes made by `add_spdx_header.py` + - name: Commit changes + if: env.FILES != '' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_user_name: SPDX-Bot + commit_user_email: bot@example.com + commit_message: '🚨✨AUTOMATED COMMIT | Added missing SPDX license headers automatically' + branch: ${{ github.head_ref }} + + # Extract and clean file paths + - name: Extract File Paths + if: env.FILES != '' + run: | + files_clean=$(echo "$FILES" | awk '/^Modified files:/ {ignore = 1} /^Above is a list of files/ {ignore = 0} !ignore && /^\.\// {print}' | sort | uniq) + echo "FILES_CLEAN<> $GITHUB_ENV + echo "$files_clean" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "Extracted Paths: $files_clean" + + - name: Debug Extracted Paths + run: | + echo "Extracted Paths:" + echo "$FILES_CLEAN" + echo "---------------------------------" + + # Post a comment on the PR or Issue if there are SPDX issues + - name: Comment on PR or Issue + if: env.FILES_CLEAN != '' + uses: actions/github-script@v7 + with: + script: | + const formattedOutput = process.env.FILES_CLEAN; + const issueNumber = context.payload.pull_request.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + if (formattedOutput) { + const commentBody = `Our automated SPDX license verification process has discovered that the following files are missing a license header:\n\`\`\`\n${formattedOutput}\n\`\`\`\nPlease ensure each indicated file includes a valid SPDX license identifier. This is essential for maintaining licensing compliance. Thank you.`; + await github.rest.issues.createComment({ + issue_number: issueNumber, + owner: owner, + repo: repo, + body: commentBody + }); + } else { + console.log("No SPDX license issues found."); + } + + - name: Run SPDX Header Script Again + run: | + set +e + echo "Running SPDX header script again on all files in the repository" + python ./add_spdx_header.py + exit 0 + + # Commit the changes made by the second run of `add_spdx_header.py` + - name: Commit changes from second run + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_user_name: SPDX-Bot + commit_user_email: bot@example.com + commit_message: '🚨✨AUTOMATED COMMIT | Final SPDX license header additions' + branch: ${{ github.head_ref }} \ No newline at end of file diff --git a/.github/workflows/frontend-lint-license-checker.yml b/.github/workflows/frontend-lint-license-checker.yml new file mode 100644 index 00000000..2bcdbd94 --- /dev/null +++ b/.github/workflows/frontend-lint-license-checker.yml @@ -0,0 +1,176 @@ +name: Front-End Linter SPDX Licenses Checker + +on: + push: + branches: + - main + pull_request: + branches: + - "main" + - "staging" + types: + - opened + - reopened + - synchronize + - assigned + - review_requested + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: '16' + + - name: Install dependencies + working-directory: app/frontend + run: npm install + + - name: Run ESLint with Auto-fix + working-directory: app/frontend + run: | + set +e + npm run lint || true # Run lint and allow the workflow to continue even if there are warnings/errors + + # Check if GPG key exists + - name: Check if GPG key exists + run: | + if [ -n "${{ secrets.GPG_PRIVATE_KEY }}" ]; then + echo "GPG_KEY_EXISTS=true" >> $GITHUB_ENV + else + echo "GPG_KEY_EXISTS=false" >> $GITHUB_ENV + fi + + # Import GPG key if it exists + - name: Import GPG key if it exists + if: env.GPG_KEY_EXISTS == 'true' + uses: crazy-max/ghaction-import-gpg@v6 + with: + gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} + passphrase: ${{ secrets.GPG_PASSPHRASE }} + git_user_signingkey: true + git_commit_gpgsign: true + + # Auto-commit changes using stefanzweifel/git-auto-commit-action + - name: Commit changes + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_user_name: CleanBot + commit_user_email: your-email@example.com + commit_message: '🚨✨ AUTOMATED COMMIT | Applied Code Formatting, Cleanup and added SPDX license headers' + commit_options: ${{ env.GPG_KEY_EXISTS == 'true' && '-S' || '' }} + branch: ${{ github.head_ref }} + + # Run ESLint and Capture Output + - name: Run ESLint and Capture Output + working-directory: app/frontend + id: run_eslint + run: | + set +e + output=$(npm run lint --silent 2>&1) + exit_code=$? + echo "$output" + clean_output=$(echo "$output" | sed 's/\x1b\[[0-9;]*m//g') # Remove ANSI escape codes + echo "CLEAN_OUTPUT<> $GITHUB_ENV + echo "$clean_output" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + + grouped_errors="" + current_file="" + errors_for_file="" + has_errors=false + missing_lc_headers=false + lc_flagged_files="" + + while IFS= read -r line; do + if echo "$line" | grep -q '^\s*\/'; then + if [ "$has_errors" = true ]; then + grouped_errors+="$current_file\n$errors_for_file\n------------------------------------------------------------------------------------------------------------------------------------------\n" + errors_for_file="" + has_errors=false + fi + current_file=$(echo "$line" | sed 's/\n//g') + elif echo "$line" | grep -q 'error'; then + errors_for_file+=" $line\n" + has_errors=true + + if echo "$line" | grep -q 'missing header'; then + missing_lc_headers=true + errors_for_file+="!Flagged: LC header missing\n" + lc_flagged_files+="${current_file}\n" + fi + fi + done <<< "$clean_output" + + if [ "$has_errors" = true ]; then + grouped_errors+="$current_file\n$errors_for_file\n" + fi + + if [ -n "$grouped_errors" ]; then + echo "Errors found." + echo "GROUPED_ERRORS<> $GITHUB_ENV + echo -e "$grouped_errors" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + echo "HAS_ERRORS=true" >> $GITHUB_ENV + else + echo "No relevant errors found." + echo "HAS_ERRORS=false" >> $GITHUB_ENV + fi + + if [ "$missing_lc_headers" = true ]; then + echo "LC headers missing in one or more files." + echo "MISSING_LC_HEADERS=true" >> $GITHUB_ENV + echo "LC_FLAGGED_FILES<> $GITHUB_ENV + echo -e "$(echo -e "$lc_flagged_files" | sed '/^\s*$/d')" >> $GITHUB_ENV + echo "EOF" >> $GITHUB_ENV + else + echo "MISSING_LC_HEADERS=false" >> $GITHUB_ENV + fi + + exit 0 + + # Comment on PR with ESLint Errors and LC Headers + - name: Comment on PR with ESLint Errors and LC Headers + if: env.HAS_ERRORS == 'true' + uses: actions/github-script@v7 + with: + script: | + const lintErrors = process.env.GROUPED_ERRORS; + const issueNumber = context.payload.pull_request.number; + const owner = context.repo.owner; + const repo = context.repo.repo; + + let commentBody = ""; + + if (process.env.MISSING_LC_HEADERS === 'true') { + const flaggedFiles = process.env.LC_FLAGGED_FILES.trim(); + commentBody += `## 🚨 SPDX-License Header Errors\n\nThe following files are missing the required license headers:\n\n\`\`\`\n${flaggedFiles}\n\`\`\`\nPlease ensure each of these files includes a valid SPDX license identifier. This is essential for maintaining licensing compliance. Thank you!`; + } + + if (lintErrors) { + commentBody += `## Frontend Project: ESLint Errors\n\`\`\`\n${lintErrors}\n\`\`\`\nPlease review and resolve the ESLint errors.`; + } + + if (commentBody.trim() && issueNumber) { + await github.rest.issues.createComment({ + issue_number: issueNumber, + owner: owner, + repo: repo, + body: commentBody + }); + } else { + console.log("No relevant errors to report."); + } + - name: Fail the Workflow if LC Headers Are Missing + if: env.MISSING_LC_HEADERS == 'true' + run: | + echo "Failing the workflow because LC headers are missing." + exit 1 \ No newline at end of file diff --git a/LICENSE b/LICENSE index c0d707ec..23e63805 100644 --- a/LICENSE +++ b/LICENSE @@ -1,17 +1,3 @@ - Copyright (c) 2024 Tenstorrent AI ULC - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ @@ -189,8 +175,22 @@ END OF TERMS AND CONDITIONS + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ------------------------------------------------------------------------------- + Copyright (c) 2024 Tenstorrent AI ULC +------------------------------------------------------------------------------- Third-Party Dependencies: The following dependencies are utilized by this project but are not explicitly diff --git a/add_spdx_header.py b/add_spdx_header.py index 6e1c6036..4d9011e9 100644 --- a/add_spdx_header.py +++ b/add_spdx_header.py @@ -1,9 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC + from pathlib import Path +from datetime import datetime -# SPDX header content -SPDX_HEADER = """# SPDX-License-Identifier: Apache-2.0 +current_year = datetime.now().year + +SPDX_HEADER = f"""# SPDX-License-Identifier: Apache-2.0 # -# SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC +# SPDX-FileCopyrightText: © {current_year} Tenstorrent AI ULC """ def add_spdx_header(file_path): @@ -12,7 +18,6 @@ def add_spdx_header(file_path): """ with open(file_path, "r+") as file: content = file.read() - # print(content) if "SPDX-License-Identifier" not in content: file.seek(0, 0) file.write(SPDX_HEADER + "\n" + content) @@ -24,14 +29,10 @@ def add_spdx_header(file_path): repo_root / "tt-studio/app/api", repo_root / "tt-studio/models", ] - - # print(f"Processing directories: {directories_to_process}") # Walk through the directories and add the header to relevant files for directory in directories_to_process: for file_path in directory.rglob("*"): - # print(file_path) # Check if the file is a Python file, Bash script, or Dockerfile if file_path.suffix in (".py", ".sh") or file_path.name == "Dockerfile": - print(f"Adding SPDX header to: {file_path}") - add_spdx_header(file_path) + add_spdx_header(file_path) \ No newline at end of file diff --git a/app/frontend/package.json b/app/frontend/package.json index a89bc6a0..cd62af72 100644 --- a/app/frontend/package.json +++ b/app/frontend/package.json @@ -6,7 +6,7 @@ "scripts": { "dev": "vite ", "build": "tsc && vite build", - "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0", + "lint": "eslint . --ext ts,tsx --report-unused-disable-directives --max-warnings 0 --fix", "preview": "vite preview", "generate-license": "npx generate-license-file --input package.json --output ./third-party-licenses.txt" }, diff --git a/app/frontend/src/App.tsx b/app/frontend/src/App.tsx index be57984f..2fa6dbd4 100644 --- a/app/frontend/src/App.tsx +++ b/app/frontend/src/App.tsx @@ -5,7 +5,6 @@ import { ThemeProvider } from "./providers/ThemeProvider"; import AppRouter from "./routes/index.tsx"; import { QueryClient, QueryClientProvider } from "react-query"; function App() { - const client = new QueryClient({ defaultOptions: {}, }); diff --git a/app/frontend/src/components/ConfirmDialog.tsx b/app/frontend/src/components/ConfirmDialog.tsx index 985d0c3a..be953c50 100644 --- a/app/frontend/src/components/ConfirmDialog.tsx +++ b/app/frontend/src/components/ConfirmDialog.tsx @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC import { AlertDialog, AlertDialogAction, @@ -17,10 +19,10 @@ export function ConfirmDialog({ cancelText, confirmText, }: { - cancelText?: string, - confirmText?: string, - dialogTitle: string, - dialogDescription: string, + cancelText?: string; + confirmText?: string; + dialogTitle: string; + dialogDescription: string; alertTrigger: React.ReactNode; onConfirm: React.MouseEventHandler; }) { @@ -30,19 +32,20 @@ export function ConfirmDialog({ {dialogTitle} - - {dialogDescription} - + {dialogDescription} - {cancelText || 'Cancel'} + + {cancelText || "Cancel"}{" "} + {confirmText || 'Continue'} + onClick={onConfirm} + > + {confirmText || "Continue"} + ); -} \ No newline at end of file +} diff --git a/app/frontend/src/components/SideBar.tsx b/app/frontend/src/components/SideBar.tsx index ebaf8404..b5ca54a4 100644 --- a/app/frontend/src/components/SideBar.tsx +++ b/app/frontend/src/components/SideBar.tsx @@ -19,8 +19,9 @@ const Sidebar = forwardRef((_, ref) => { }; const getHelpContent = () => { - const baseStyles = `shadow-lg rounded-lg p-6 my-4 ${theme === "dark" ? "bg-TT-slate text-white" : "bg-white text-tt-black" - }`; + const baseStyles = `shadow-lg rounded-lg p-6 my-4 ${ + theme === "dark" ? "bg-TT-slate text-white" : "bg-white text-tt-black" + }`; return ( { @@ -73,19 +74,25 @@ const Sidebar = forwardRef((_, ref) => {

- The "RAG Management" page allows you to create additional Datasources - to augment the responses provided by the models you have deployed. You can select - a datasource to use from the dropdown menu at the top of the chat window. + The "RAG Management" page allows you to create additional + Datasources to augment the responses provided by the models you + have deployed. You can select a datasource to use from the + dropdown menu at the top of the chat window.

  • - Create New RAG Datasource: RAG datasources must have a unique name of at least two characters and no spaces. - Enter the name of your datasource and click the create button to create the datasource.
    • + Create New RAG Datasource: RAG datasources + must have a unique name of at least two characters and no + spaces. Enter the name of your datasource and click the create + button to create the datasource.{" "} +
  • - Upload Document: Select a document to upload and embed into the RAG datasource. + Upload Document: Select a document to upload + and embed into the RAG datasource.
  • - Delete: Delete your RAG datasource. This action is permanent and irreversible. + Delete: Delete your RAG datasource. This + action is permanent and irreversible.
@@ -157,9 +164,11 @@ const Sidebar = forwardRef((_, ref) => { return (
@@ -180,8 +189,9 @@ const Sidebar = forwardRef((_, ref) => {
diff --git a/app/frontend/src/pages/rag/RagManagement.tsx b/app/frontend/src/pages/rag/RagManagement.tsx index 9a14fcb6..c78b4ad4 100644 --- a/app/frontend/src/pages/rag/RagManagement.tsx +++ b/app/frontend/src/pages/rag/RagManagement.tsx @@ -148,7 +148,7 @@ export default function RagManagement() { }; const renderRow = ({ - theme, + // theme, item, isUploading, onDelete, @@ -191,7 +191,6 @@ export default function RagManagement() { disabled={isUploading} className="bg-blue-500 dark:bg-blue-700 hover:bg-blue-600 dark:hover:bg-blue-600 text-white rounded-lg" onClick={() => onUploadClick(item)} - className="bg-blue-500 dark:bg-blue-700 hover:bg-blue-600 dark:hover:bg-blue-600 text-white rounded-lg" > Upload Document diff --git a/app/frontend/src/routes/index.tsx b/app/frontend/src/routes/index.tsx index d9fef55d..faa9f9dd 100644 --- a/app/frontend/src/routes/index.tsx +++ b/app/frontend/src/routes/index.tsx @@ -12,15 +12,15 @@ import RagManagement from "../pages/rag/RagManagement"; const AppRouter = () => { return ( - - - - } /> - } /> - } /> - } /> - - + + + + } /> + } /> + } /> + } /> + + ); }; diff --git a/app/frontend/vite.config.ts b/app/frontend/vite.config.ts index 5bdf94af..63b6b2c1 100644 --- a/app/frontend/vite.config.ts +++ b/app/frontend/vite.config.ts @@ -24,19 +24,36 @@ const proxyConfig: Record = Object.fromEntries( secure: true, // debug logging configure: (proxy: HttpProxy.Server) => { - proxy.on("error", (err: Error, _req: IncomingMessage, _res: ServerResponse) => { - console.log("proxy error", err); - }); - proxy.on("proxyReq", (proxyReq: ClientRequest, req: IncomingMessage, _res: ServerResponse) => { - console.log("Sending Request to the Target:", req.method, req.url); - }); - proxy.on("proxyRes", (proxyRes: IncomingMessage, req: IncomingMessage, _res: ServerResponse) => { - console.log( - "Received Response from the Target:", - proxyRes.statusCode, - req.url, - ); - }); + proxy.on( + "error", + (err: Error, _req: IncomingMessage, _res: ServerResponse) => { + console.log("proxy error", err); + }, + ); + proxy.on( + "proxyReq", + ( + proxyReq: ClientRequest, + req: IncomingMessage, + _res: ServerResponse, + ) => { + console.log("Sending Request to the Target:", req.method, req.url); + }, + ); + proxy.on( + "proxyRes", + ( + proxyRes: IncomingMessage, + req: IncomingMessage, + _res: ServerResponse, + ) => { + console.log( + "Received Response from the Target:", + proxyRes.statusCode, + req.url, + ); + }, + ); }, rewrite: (path: string) => path.replace(new RegExp(`^/${proxyPath}`), `/${actualPath}`), diff --git a/models/tt-metal-falcon-7b/src/_mock_inference_api_server.py b/models/tt-metal-falcon-7b/src/_mock_inference_api_server.py index a31f6f69..a1b61c14 100644 --- a/models/tt-metal-falcon-7b/src/_mock_inference_api_server.py +++ b/models/tt-metal-falcon-7b/src/_mock_inference_api_server.py @@ -2,6 +2,8 @@ # # SPDX-FileCopyrightText: © 2024 Tenstorrent AI ULC + + import os from time import sleep from unittest.mock import Mock, patch