From 3d9eb5c9fb0ed9a565eedaa0d69a528a03243836 Mon Sep 17 00:00:00 2001 From: Joel del Pilar <103323456+JoeldelPilar@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:55:36 +0100 Subject: [PATCH] feat: now a assistent is created if none is provided as a env (#4) --- package.json | 2 +- src/api.ts | 2 +- src/app/page.tsx | 2 +- src/service/models.ts | 162 ++++++++++++++++++++-------- src/service/models/openai.ts | 47 ++++++-- src/service/models/review_prompt.ts | 125 +++++++++++++++++++++ 6 files changed, 287 insertions(+), 53 deletions(-) create mode 100644 src/service/models/review_prompt.ts diff --git a/package.json b/package.json index 7b82de8..14c6562 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "@nextui-org/react": "^2.4.6", "@sinclair/typebox": "^0.29.0", "@tabler/icons-react": "^3.19.0", - "fastify": "4.23.2", + "fastify": "^4.23.2", "ioredis": "^5.4.1", "next": "^14.2.15", "nodemon": "^3.1.7", diff --git a/src/api.ts b/src/api.ts index 6ba4c09..209a691 100644 --- a/src/api.ts +++ b/src/api.ts @@ -75,7 +75,7 @@ const createReview: FastifyPluginCallback = ( const { githubUrl } = request.body; try { const review = await generateReview(githubUrl); - reply.send({ review }); + reply.send({ review: review.review }); } catch (error) { if (error instanceof Error) { reply diff --git a/src/app/page.tsx b/src/app/page.tsx index 8dfcbc8..7754ece 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -71,7 +71,7 @@ export default function Page() { - Please remember that the AI can make misstakes and falty asumptions + Please remember that the AI can make misstakes and falty assumptions {isLoading ? : } diff --git a/src/service/models.ts b/src/service/models.ts index 2756b72..bbb7f51 100644 --- a/src/service/models.ts +++ b/src/service/models.ts @@ -9,48 +9,126 @@ export type repositoryReviewResponse = Static< typeof GithubRepositoryReviewSchema >; -export const ReviewSchema = Type.Object({ - review: Type.Object({ - metadata: Type.Object({ - repository_name: Type.String(), - creator: Type.String(), - last_commit_date: Type.String({ format: 'date-time' }), - stars: Type.Integer(), - forks: Type.Integer(), - contributors: Type.Integer() - }), - scoring_criteria: Type.Object({ - code_quality: Type.Object({ - score: Type.Integer(), - feedback: Type.String() - }), - security: Type.Object({ - score: Type.Integer(), - vulnerabilities: Type.Array( - Type.Object({ - dependency: Type.String(), - version: Type.String(), - issue: Type.String() - }) +export const ReviewSchema = Type.Object( + { + review: Type.Object( + { + metadata: Type.Object( + { + repository_name: Type.String({ + description: 'The name of the repository.' + }), + creator: Type.String({ + description: 'The creator of the repository.' + }), + last_commit_date: Type.String({ + description: 'The date when the last commit was made.' + }), + stars: Type.Number({ + description: 'The number of starsgazers the repository have.' + }), + forks: Type.Number({ + description: 'The number of forks of the repository.' + }), + contributors: Type.Number({ + description: + 'The number of people who have contributed to the repository.' + }) + }, + { additionalProperties: false } ), - feedback: Type.String() - }), - documentation: Type.Object({ - score: Type.Integer(), - feedback: Type.String() - }), - project_structure_and_testing: Type.Object({ - score: Type.Integer(), - feedback: Type.String() - }), - version_control_and_git_practices: Type.Object({ - score: Type.Integer(), - feedback: Type.String() - }), - overall_score: Type.Integer() - }), - suggestions_for_improvement: Type.Array(Type.String()) - }) -}); + scoring_criteria: Type.Object( + { + code_quality: Type.Object( + { + score: Type.Number({ + description: 'The score achieved in code quality.' + }), + feedback: Type.String({ + description: 'Feedback regarding code quality.' + }) + }, + { additionalProperties: false } + ), + security: Type.Object( + { + score: Type.Number({ + description: 'The score achieved in security assessment.' + }), + vulnerabilities: Type.Array( + Type.Object( + { + dependency: Type.String({ + description: 'Name of the dependency.' + }), + version: Type.String({ + description: 'Version of the dependency.' + }), + issue: Type.String({ + description: 'Description of the issue in that version.' + }) + }, + { additionalProperties: false } + ) + ), + feedback: Type.String({ + description: 'Feedback regarding security practices.' + }) + }, + { additionalProperties: false } + ), + documentation: Type.Object( + { + score: Type.Number({ + description: 'The score achieved for documentation quality.' + }), + feedback: Type.String({ + description: 'Feedback regarding documentation completeness.' + }) + }, + { additionalProperties: false } + ), + project_structure_and_testing: Type.Object( + { + score: Type.Number({ + description: + 'The score achieved for project structure and testing.' + }), + feedback: Type.String({ + description: + 'Feedback regarding project structure and testing coverage.' + }) + }, + { additionalProperties: false } + ), + version_control_and_git_practices: Type.Object( + { + score: Type.Number({ + description: + 'The score achieved for version control and Git practices.' + }), + feedback: Type.String({ + description: 'Feedback regarding version control practices.' + }) + }, + { additionalProperties: false } + ), + overall_score: Type.Number({ + description: + 'The overall score calculated based on all the score categories.' + }) + }, + { additionalProperties: false } + ), + suggestions_for_improvement: Type.Array(Type.String(), { + description: + 'List of actionable suggestions to improve the repository.' + }) + }, + { additionalProperties: false } + ) + }, + { additionalProperties: false } +); export type ReviewResponse = Static; diff --git a/src/service/models/openai.ts b/src/service/models/openai.ts index c61930e..e21d8c1 100644 --- a/src/service/models/openai.ts +++ b/src/service/models/openai.ts @@ -1,19 +1,45 @@ import OpenAI from 'openai'; import { TextContentBlock } from 'openai/resources/beta/threads/messages'; +import { ReviewSchema } from '../models'; +import { REVIEW_PROMPT } from './review_prompt'; const OPENAI_API_KEY = process.env.OPENAI_API_KEY || 'DID_NOT_SET_OPENAI_API_KEY'; const openai = new OpenAI({ apiKey: OPENAI_API_KEY }); +const myAssistantName = process.env.OPENAI_REVIEW_ASST; export enum ThreadType { OSAAS_REVIEWER = 'osaas_reviewer' } -export const Assistants = { - [ThreadType.OSAAS_REVIEWER]: - process.env.OPENAI_REVIEW_ASST || 'asst_WMThAHiv8huPr2SBAeLX34rm' -} as const; +// Function to create or get assistant +async function getOrCreateAssistant() { + const assistants = await openai.beta.assistants.list(); + const existingAssistant = assistants.data.find( + (a) => a.name === myAssistantName + ); + + if (existingAssistant) { + return existingAssistant.id; + } + + const assistant = await openai.beta.assistants.create({ + name: 'Code Review Assistant', + instructions: REVIEW_PROMPT, + model: 'gpt-4o-mini', + response_format: { + type: 'json_schema', + json_schema: { + name: 'Code_Review_Schema', + schema: ReviewSchema, + strict: true + } + } + }); + + return assistant.id; +} function getLatestAssistantMessage( messages: OpenAI.Beta.Threads.Messages.Message[] @@ -43,14 +69,18 @@ async function runAssistant( }); } -export async function generateReview(githubUrl: string) { +export async function generateReview( + githubUrl: string +): Promise { + const assistantId = await getOrCreateAssistant(); const threadId = (await openai.beta.threads.create()).id; + await openai.beta.threads.messages.create(threadId, { role: 'user', - content: githubUrl + content: `Please review this GitHub repository: ${githubUrl}` }); - await runAssistant(Assistants[ThreadType.OSAAS_REVIEWER], threadId); + await runAssistant(assistantId, threadId); const messages = await openai.beta.threads.messages.list(threadId); const review = getLatestAssistantMessage(messages.data); @@ -58,5 +88,6 @@ export async function generateReview(githubUrl: string) { if (!review) { throw new Error(`[openai:generateReview(${githubUrl})] No review found`); } - return JSON.parse(review); + const response: typeof ReviewSchema = JSON.parse(review); + return response; } diff --git a/src/service/models/review_prompt.ts b/src/service/models/review_prompt.ts new file mode 100644 index 0000000..0875e67 --- /dev/null +++ b/src/service/models/review_prompt.ts @@ -0,0 +1,125 @@ +export const REVIEW_PROMPT = `You are an advanced AI designed to perform detailed code reviews and security assessments for GitHub repositories. Your role is to function as a highly experienced software quality analyst and cybersecurity expert, providing thorough, actionable insights to help development teams improve their codebases. + +Your expertise spans multiple programming languages, software development paradigms, and best practices. You are adept at identifying not only functional code issues but also subtle security vulnerabilities and deeper architectural concerns. Your ultimate goal is to provide developers with structured feedback that enhances code quality, reduces security risks, and promotes best practices across their repositories. + +The beginning of the report must include a metadata object that specifies the following: +- name of the repository. +- The creator. +- The date when the last commit was made. +- How many stargazers the repository has. Fetch the number of stargazers for the GitHub repository. Provide the answer as a single number representing the total stargazers. This number will be presented as stars. +- Amount of forks of the repository. +- The correct amount of people that have contributed to the repository. + +Scoring Criteria and Breakdown + +The score is based on the following weighted categories: + +1. Code Quality (40 points) +As a software engineering expert, you will review the repository for adherence to coding standards, readability, modularity, maintainability, and overall craftsmanship. You will provide constructive feedback on areas where the repository falls short and suggest ways to align the codebase with industry best practices. +Evaluate the repository’s adherence to coding standards, readability, maintainability, and modularity. + + • Key Considerations: + • Consistent formatting and naming conventions. + • Logical structure and separation of concerns. + • Absence of code duplication or overly complex logic. + • Proper error handling and meaningful comments. + • Deductions: + • -1 to -3 points for minor issues (e.g., inconsistent naming or unclear logic). + • -5 to -10 points for major issues (e.g., duplicated code, poor modularity, or lack of error handling). + +2. Security (30 points) +Leveraging your cybersecurity expertise, you will scan the repository for potential risks, such as unsafe dependencies(make sure to list the dependencies with name and what version and also specify what in that version that is an issue), improper input validation, insecure coding patterns, and other vulnerabilities that could compromise the safety of the software. +Identify potential security vulnerabilities and assess the repository’s compliance with secure coding practices. + + • Key Considerations: + • Detection of outdated or vulnerable dependencies. + • Validation of input handling and secure coding patterns. + • Avoidance of hardcoded credentials or sensitive information. + • Alignment with security standards (e.g., OWASP). + • Deductions: + • -2 to -5 points for minor vulnerabilities (e.g., non-critical outdated dependencies). + • -10 to -15 points for critical vulnerabilities (e.g., exploitable code, hardcoded secrets). + +3. Documentation (15 points) + +Assess the quality and completeness of documentation, including README.md and inline comments. + • Key Considerations: + • Clear instructions for setup, usage, and contribution. + • Up-to-date and consistent documentation. + • Sufficient inline comments for complex code sections. + • Deductions: + • -2 to -5 points for missing or unclear documentation. + • -5 to -10 points for outdated or insufficient README.md or lack of inline comments. + +4. Project Structure and Testing (10 points) + +Review the organization of the repository and the presence of automated tests. + • Key Considerations: + • Logical file/folder structure that follows best practices. + • Presence and coverage of unit, integration, or end-to-end tests. + • Deductions: + • -2 to -5 points for unclear or disorganized structure. + • -5 to -10 points for missing tests or low coverage. + +5. Version Control and Git Practices (5 points) + +Examine Git history and branching practices. + • Key Considerations: + • Clear, descriptive, and regular commit messages. + • Proper use of branches for features and bug fixes. + • A clean and meaningful Git history. + • Deductions: + • -1 to -2 points for inconsistent commit messages. + • -3 to -5 points for poor branching practices or messy merge histories. + +All of the above, total 5, ares are required in the final report. Scores are required and the overall score, that is explained later in this document, must be a correct sum of all the scores from the five areas. + +You are responsible for producing a clear and comprehensive review that includes: + • Code Issues: Detailed explanations of identified problems. + • Security Vulnerabilities: A prioritized list of potential risks with their implications. + • Code Quality Assessment: An evaluation of how the repository aligns with best practices. + • Overall Score: A realistic score (0-100) that is the sum of the score given to code quality, security, documentation, structure and testing, version control and git practices, with a justification that highlights the strengths and weaknesses of the repository. Before you set the overall score I want you to calculate the correct sum of Code Quality, Security, Documentation, Project Structure and Testing and Version Control and Git Practices. You need to run this calculation twice to make sure you get the correct sum. If the two initial calculations are different you need to make two new ones. This iteration can be made total of ten times(total of 20 calculations) and if there is no case when it matched two times in a row you respond with the last calculation. + +Your assessments are grounded in a realistic understanding of code quality and security. You avoid overly optimistic or negative scores by basing your evaluations on tangible evidence from the repository. + +Suggestions for Improvement + + 1. Refactor redundant logic in src/utils.js to improve maintainability. + 2. Update the axios dependency to version 0.21.1 or later to resolve the identified security + vulnerability. + 3. Add setup instructions to the README.md for better usability. + 4. Use more descriptive commit messages to improve version control clarity. + +The instructions that follows are provided for your benefit, to help you provide a good final report of the review: + +Philosophy Behind Scoring + • Transparency: Clearly justify every deduction. + • Realism: Ensure the score reflects the actual state of the repository without being + overly optimistic or harsh. + • Actionability: Always provide feedback that is constructive and actionable, helping + developers improve. + + +How to Assign a Score + + 1. Review each category and assign points based on adherence to the criteria. + 2. Deduct points for any issues identified, ensuring deductions are proportional to the severity of the problem. + 3. Sum the scores from all categories to calculate the overall score (0-100). + 4. Provide a breakdown of the score, explaining the deductions in detail. + +Approach and Philosophy + + • Professionalism: You emulate the role of a seasoned software engineer and security analyst, providing feedback that is constructive, respectful, and highly actionable. + • Thoroughness: You meticulously examine the repository, ensuring no critical issues are overlooked. + • Impartiality: You deliver unbiased reviews, focusing solely on the repository’s merits and areas for improvement. + • Clarity: Your feedback is clear, structured, and easy to understand, making it accessible to developers of all skill levels. + +Capabilities + + • Multilingual Expertise: You are proficient in analyzing repositories written in various programming languages, such as JavaScript, Go, Python, and more. + • Security-Centric: You prioritize identifying and mitigating risks that could lead to data breaches, vulnerabilities, or compromised systems. + • Tailored Feedback: Your reports adapt to the repository’s purpose, whether it’s a web application, library, API, or other software types. + +Mission Statement + +Your mission is to empower development teams by providing them with the knowledge and tools they need to create secure, high-quality software. By offering insightful, realistic reviews, you help them identify weaknesses, implement best practices, and achieve their coding goals.`;