feat: add previous code to context (#1)

* feat: add evaluated code to previous steps context.

* test: update snapshots.

src/Copilot.test.ts

@@ -24,6 +24,8 @@ describe('Copilot', () => {
             }
         };
         jest.spyOn(console, 'error').mockImplementation(() => {});
+
+        (StepPerformer.prototype.perform as jest.Mock).mockResolvedValue({code: 'code', result: true});
     });
 
     afterEach(() => {
@@ -73,6 +75,7 @@ describe('Copilot', () => {
 
     describe('perform', () => {
         it('should call StepPerformer.perform with the given intent', async () => {
+
             Copilot.init(mockConfig);
             const instance = Copilot.getInstance();
             const intent = 'tap button';
@@ -83,7 +86,6 @@ describe('Copilot', () => {
         });
 
         it('should return the result from StepPerformer.perform', async () => {
-            (StepPerformer.prototype.perform as jest.Mock).mockResolvedValue(true);
             Copilot.init(mockConfig);
             const instance = Copilot.getInstance();
             const intent = 'tap button';
@@ -102,7 +104,11 @@ describe('Copilot', () => {
             await instance.performStep(intent1);
             await instance.performStep(intent2);
 
-            expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(intent2, [intent1]);
+            expect(StepPerformer.prototype.perform).toHaveBeenLastCalledWith(intent2, [{
+                step: intent1,
+                code: 'code',
+                result: true
+            }]);
         });
     });
 

src/Copilot.ts

@@ -3,7 +3,7 @@ import {PromptCreator} from "@/utils/PromptCreator";
 import {CodeEvaluator} from "@/utils/CodeEvaluator";
 import {SnapshotManager} from "@/utils/SnapshotManager";
 import {StepPerformer} from "@/actions/StepPerformer";
-import {Config} from "@/types";
+import {Config, PreviousStep} from "@/types";
 
 /**
  * The main Copilot class that provides AI-assisted testing capabilities for a given underlying testing framework.
@@ -16,7 +16,7 @@ export class Copilot {
     private readonly promptCreator: PromptCreator;
     private readonly codeEvaluator: CodeEvaluator;
     private readonly snapshotManager: SnapshotManager;
-    private previousSteps: string[] = [];
+    private previousSteps: PreviousStep[] = [];
     private stepPerformer: StepPerformer;
 
     private constructor(config: Config) {
@@ -57,8 +57,8 @@ export class Copilot {
      * @param step The step describing the operation to perform.
      */
     async performStep(step: string): Promise<any> {
-        const result = await this.stepPerformer.perform(step, this.previousSteps);
-        this.didPerformStep(step);
+        const {code, result} = await this.stepPerformer.perform(step, this.previousSteps);
+        this.didPerformStep(step, code, result);
 
         return result;
     }
@@ -71,8 +71,11 @@ export class Copilot {
         this.previousSteps = [];
     }
 
-    // todo: cache the previous steps' generated test code
-    private didPerformStep(step: string): void {
-        this.previousSteps = [...this.previousSteps, step];
+    private didPerformStep(step: string, code: string, result: any): void {
+        this.previousSteps = [...this.previousSteps, {
+            step,
+            code,
+            result
+        }];
     }
 }

src/actions/StepPerformer.test.ts

@@ -160,7 +160,12 @@ describe('StepPerformer', () => {
 
     it('should perform an intent successfully with previous intents', async () => {
         const intent = 'current intent';
-        const previousIntents = ['previous intent'];
+        const previousIntents = [{
+            step: 'previous intent',
+            code: 'previous code',
+            result: 'previous result',
+        }];
+
         setupMocks();
 
         const result = await stepPerformer.perform(intent, previousIntents);

src/actions/StepPerformer.ts

@@ -1,7 +1,7 @@
 import { PromptCreator } from '@/utils/PromptCreator';
 import { CodeEvaluator } from '@/utils/CodeEvaluator';
 import { SnapshotManager } from '@/utils/SnapshotManager';
-import { PromptHandler } from '@/types';
+import {CodeEvaluationResult, PreviousStep, PromptHandler} from '@/types';
 import * as fs from 'fs';
 import * as path from 'path';
 
@@ -20,7 +20,7 @@ export class StepPerformer {
         this.cacheFilePath = path.resolve(process.cwd(), 'copilot-cache', cacheFileName);
     }
 
-    private getCacheKey(step: string, previous: string[]): string {
+    private getCacheKey(step: string, previous: PreviousStep[]): string {
         return JSON.stringify({ step, previous });
     }
 
@@ -48,7 +48,7 @@ export class StepPerformer {
         }
     }
 
-    async perform(step: string, previous: string[] = []): Promise<any> {
+    async perform(step: string, previous: PreviousStep[] = []): Promise<CodeEvaluationResult> {
         // todo: replace with the user's logger
         console.log("\x1b[90m%s\x1b[0m%s", "Copilot performing: ", `"${step}"`);
 
@@ -87,10 +87,14 @@ export class StepPerformer {
         } catch (error) {
             // Extend 'previous' array with the failure message
             const failedAttemptMessage = promptResult
-                ? `Failed to perform "${step}", tried with "${promptResult}". Should we try a different approach? If can't, throw an error.`
-                : `Failed to perform "${step}", could not generate prompt result. Should we try a different approach? If can't, throw an error.`;
+                ? `Failed to evaluate "${step}", tried with generated code: "${promptResult}". Should we try a different approach? If can't, return a code that throws a descriptive error.`
+                : `Failed to perform "${step}", could not generate prompt result. Should we try a different approach? If can't, return a code that throws a descriptive error.`;
 
-            const newPrevious = [...previous, failedAttemptMessage];
+            const newPrevious = [...previous, {
+                step,
+                code: failedAttemptMessage,
+                result: undefined,
+            }];
 
             const retryCacheKey = this.getCacheKey(step, newPrevious);
 
@@ -112,6 +116,7 @@ export class StepPerformer {
                     retryPromptResult,
                     this.context,
                 );
+
                 // Cache the result under the original cache key
                 this.cache.set(cacheKey, retryPromptResult);
                 this.saveCacheToFile();

src/types.ts

@@ -129,3 +129,26 @@ export interface Config {
      */
     promptHandler: PromptHandler;
 }
+
+/**
+ * Represents a previous step that was performed in the test flow.
+ * @note This is used to keep track of the context and history of the test flow.
+ * @property step The description of the step.
+ * @property code The generated test code for the step.
+ * @property result The result of the step.
+ */
+export type PreviousStep = {
+    step: string;
+    code: string;
+    result: any;
+}
+
+/**
+ * Represents the result of a code evaluation operation.
+ * @property code The generated test code for the operation.
+ * @property result The result of the operation.
+ */
+export type CodeEvaluationResult = {
+    code: string;
+    result: any;
+}

src/utils/CodeEvaluator.test.ts

@@ -25,7 +25,11 @@ describe('CodeEvaluator', () => {
     it('should evaluate valid code with context successfully', async () => {
         const contextVariable = 43;
         const validCode = 'return contextVariable - 1;';
-        await expect(codeEvaluator.evaluate(validCode, { contextVariable })).resolves.toBe(42);
+
+        await expect(codeEvaluator.evaluate(validCode, { contextVariable })).resolves.toStrictEqual({
+            code: 'return contextVariable - 1;',
+            result: 42
+        });
     });
 
     it('should throw CodeEvaluationError for invalid code', async () => {
@@ -35,7 +39,11 @@ describe('CodeEvaluator', () => {
 
     it('should handle asynchronous code', async () => {
         const asyncCode = 'await new Promise(resolve => setTimeout(resolve, 100)); return "done";';
-        await expect(codeEvaluator.evaluate(asyncCode, {})).resolves.toBe('done');
+
+        await expect(codeEvaluator.evaluate(asyncCode, {})).resolves.toStrictEqual({
+            code: 'await new Promise(resolve => setTimeout(resolve, 100)); return "done";',
+            result: 'done'
+        });
     });
 
     it('should throw CodeEvaluationError with original error message', async () => {

src/utils/CodeEvaluator.ts

@@ -1,27 +1,29 @@
 import { CodeEvaluationError } from '@/errors/CodeEvaluationError';
+import {CodeEvaluationResult} from "@/types";
 
 export class CodeEvaluator {
-    async evaluate(code: string, context: any): Promise<any> {
+    async evaluate(rawCode: string, context: any): Promise<CodeEvaluationResult> {
+        const code = this.extractCodeBlock(rawCode);
         const asyncFunction = this.createAsyncFunction(code, context);
-        return await asyncFunction();
+        const result = await asyncFunction();
+
+        return { code, result }
     }
 
     private createAsyncFunction(code: string, context: any): Function {
-        const codeBlock = this.extractCodeBlock(code);
-
         // todo: this is a temp log for debugging, we'll need to pass a logging mechanism from the framework.
-        console.log("\x1b[90m%s\x1b[0m\x1b[92m%s\x1b[0m", "Copilot evaluating code block: ", `\`${codeBlock}\`\n`);
+        console.log("\x1b[90m%s\x1b[0m\x1b[92m%s\x1b[0m", "Copilot evaluating code block: ", `\`${code}\`\n`);
         try {
             const contextValues = Object.values(context);
 
             // Wrap the code in an immediately-invoked async function expression (IIFE), and inject context variables into the function
             return new Function(...Object.keys(context), `return (async () => { 
-              ${codeBlock}
+              ${code}
             })();`).bind(null, ...contextValues);
         } catch (error) {
             const underlyingErrorMessage = (error as Error)?.message;
             throw new CodeEvaluationError(
-                `Failed to execute test step: ${codeBlock}, error: ${underlyingErrorMessage}`
+                `Failed to execute test step code, error: ${underlyingErrorMessage}:\n\`\`\`\n${code}\n\`\`\``
             );
         }
     }

src/utils/PromptCreator.test.ts

@@ -1,5 +1,10 @@
 import { PromptCreator } from './PromptCreator';
-import { TestingFrameworkAPICatalog, TestingFrameworkAPICatalogCategory, TestingFrameworkAPICatalogItem } from "@/types";
+import {
+    PreviousStep,
+    TestingFrameworkAPICatalog,
+    TestingFrameworkAPICatalogCategory,
+    TestingFrameworkAPICatalogItem
+} from "@/types";
 
 const mockAPI: TestingFrameworkAPICatalog = {
     context: {},
@@ -62,16 +67,32 @@ describe('PromptCreator', () => {
 
     it('should include previous intents in the context', () => {
         const intent = 'tap button';
-        const previousIntents = ['navigate to login screen', 'enter username'];
+        const previousSteps: PreviousStep[] = [
+            {
+                step: 'navigate to login screen',
+                code: 'await element(by.id("login")).tap();',
+                result: undefined
+            },
+            {
+                step: 'enter username',
+                code: 'await element(by.id("username")).typeText("john_doe");',
+                result: undefined
+            }
+        ];
+
         const viewHierarchy = '<View><Button testID="submit" title="Submit" /></View>';
-        const prompt = promptCreator.createPrompt(intent, viewHierarchy, false, previousIntents);
+
+        const prompt = promptCreator.createPrompt(intent, viewHierarchy, false, previousSteps);
+
         expect(prompt).toMatchSnapshot();
     });
 
     it('should handle when no snapshot image is attached', () => {
         const intent = 'expect button to be visible';
         const viewHierarchy = '<View><Button testID="submit" title="Submit" /></View>';
+
         const prompt = promptCreator.createPrompt(intent, viewHierarchy, false, []);
+
         expect(prompt).toMatchSnapshot();
     });
 });

src/utils/PromptCreator.ts

@@ -1,4 +1,9 @@
-import { TestingFrameworkAPICatalog, TestingFrameworkAPICatalogCategory, TestingFrameworkAPICatalogItem } from "@/types";
+import {
+    PreviousStep,
+    TestingFrameworkAPICatalog,
+    TestingFrameworkAPICatalogCategory,
+    TestingFrameworkAPICatalogItem
+} from "@/types";
 
 export class PromptCreator {
     constructor(private apiCatalog: TestingFrameworkAPICatalog) {}
@@ -7,11 +12,11 @@ export class PromptCreator {
         intent: string,
         viewHierarchy: string,
         isSnapshotImageAttached: boolean,
-        previousIntents: string[]
+        previousSteps: PreviousStep[]
     ): string {
         return [
             this.createBasePrompt(),
-            this.createContext(intent, viewHierarchy, isSnapshotImageAttached, previousIntents),
+            this.createContext(intent, viewHierarchy, isSnapshotImageAttached, previousSteps),
             this.createAPIInfo(),
             this.createInstructions(intent, isSnapshotImageAttached)
         ]
@@ -33,7 +38,7 @@ export class PromptCreator {
         intent: string,
         viewHierarchy: string,
         isSnapshotImageAttached: boolean,
-        previousIntents: string[]
+        previousSteps: PreviousStep[]
     ): string[] {
         let context = [
             "## Context",
@@ -59,11 +64,19 @@ export class PromptCreator {
             );
         }
 
-        if (previousIntents.length > 0) {
+        if (previousSteps.length > 0) {
             context.push(
                 "### Previous intents",
                 "",
-                ...previousIntents.map((prevIntent, index) => `${index + 1}. ${prevIntent}`),
+                ...previousSteps.map((previousStep, index) => [
+                    `#### Step ${index + 1}`,
+                    `- Intent: "${previousStep.step}"`,
+                    `- Generated code:`,
+                    "```",
+                    previousStep.code,
+                    "```",
+                    ""
+                ]).flat(),
                 ""
             );
         }
@@ -117,7 +130,7 @@ export class PromptCreator {
     }
 
     private createInstructions(intent: string, isSnapshotImageAttached: boolean): string[] {
-        const instructions = [
+        return [
             "## Instructions",
             "",
             [
@@ -141,8 +154,6 @@ export class PromptCreator {
             "",
             "Please provide your response below:"
         ];
-
-        return instructions;
     }
 
     private createVisualAssertionsInstructionIfPossible(isSnapshotImageAttached: boolean): string[] {

src/utils/__snapshots__/PromptCreator.test.ts.snap

@@ -227,8 +227,20 @@ Generate the minimal executable code to perform the following intent: "tap butto
 
 ### Previous intents
 
-1. navigate to login screen
-2. enter username
+#### Step 1
+- Intent: "navigate to login screen"
+- Generated code:
+\`\`\`
+await element(by.id("login")).tap();
+\`\`\`
+
+#### Step 2
+- Intent: "enter username"
+- Generated code:
+\`\`\`
+await element(by.id("username")).typeText("john_doe");
+\`\`\`
+
 
 ## Available Testing Framework API