test: run watch cases concurrently

packages/rspack-test-tools/etc/test-tools.api.md

@@ -34,17 +34,31 @@ export class BasicCaseCreator<T extends ECompilerType> {
     // (undocumented)
     create(name: string, src: string, dist: string, temp?: string): ITester | undefined;
     // (undocumented)
+    protected createConcurrentEnv(): ITestEnv & IConcurrentTestEnv;
+    // (undocumented)
     protected createEnv(testConfig: TTestConfig<T>): ITestEnv;
     // (undocumented)
     protected createTester(name: string, src: string, dist: string, temp: string | void, testConfig: TTestConfig<T>): ITester;
     // (undocumented)
+    protected currentConcurrent: number;
+    // (undocumented)
     protected describe(name: string, tester: ITester, testConfig: TTestConfig<T>): void;
     // (undocumented)
+    protected describeConcurrent(name: string, tester: ITester, testConfig: TTestConfig<T>): void;
+    // (undocumented)
+    protected getMaxConcurrent(): number;
+    // (undocumented)
     protected _options: IBasicCaseCreatorOptions<T>;
     // (undocumented)
     protected readTestConfig(src: string): TTestConfig<T>;
     // (undocumented)
+    protected registerConcurrentTask(name: string, starter: () => void): () => void;
+    // (undocumented)
     protected skip(name: string, reason: string | boolean): void;
+    // (undocumented)
+    protected tasks: [string, () => void][];
+    // (undocumented)
+    protected tryRunTask(): void;
 }
 
 // @public (undocumented)
@@ -578,6 +592,8 @@ export interface IBasicCaseCreatorOptions<T extends ECompilerType> {
     // (undocumented)
     clean?: boolean;
     // (undocumented)
+    concurrent?: boolean | number;
+    // (undocumented)
     contextValue?: Record<string, unknown>;
     // (undocumented)
     describe?: boolean;
@@ -688,6 +704,14 @@ export interface ICompareOptions {
     snapshot?: string;
 }
 
+// @public (undocumented)
+interface IConcurrentTestEnv {
+    // (undocumented)
+    clear: () => void;
+    // (undocumented)
+    run: () => Promise<void>;
+}
+
 // @public (undocumented)
 export interface IConfigProcessorOptions<T extends ECompilerType> extends IMultiTaskProcessorOptions<T> {
 }
@@ -1573,12 +1597,13 @@ export type TTestConfig<T extends ECompilerType> = {
     beforeExecute?: () => void;
     afterExecute?: () => void;
     moduleScope?: (ms: IBasicModuleScope, stats?: TCompilerStatsCompilation<T>) => IBasicModuleScope;
-    checkStats?: (stepName: string, jsonStats: TCompilerStatsCompilation<T>, stringStats: String) => boolean;
+    checkStats?: (stepName: string, jsonStats: TCompilerStatsCompilation<T> | undefined, stringStats: String) => boolean;
     findBundle?: (index: number, options: TCompilerOptions<T>, stepName?: string) => string | string[];
     bundlePath?: string[];
     nonEsmThis?: (p: string | string[]) => Object;
     modules?: Record<string, Object>;
     timeout?: number;
+    concurrent?: boolean;
 };
 
 // @public (undocumented)

packages/rspack-test-tools/src/case/hot-step.ts

@@ -26,7 +26,8 @@ function getCreator(target: TTarget) {
 						configFiles: ["rspack.config.js", "webpack.config.js"]
 					})
 				],
-				runner: HotStepRunnerFactory
+				runner: HotStepRunnerFactory,
+				concurrent: true
 			})
 		);
 	}

packages/rspack-test-tools/src/case/hot.ts

@@ -26,7 +26,8 @@ function getCreator(target: TTarget) {
 						configFiles: ["rspack.config.js", "webpack.config.js"]
 					})
 				],
-				runner: HotRunnerFactory
+				runner: HotRunnerFactory,
+				concurrent: true
 			})
 		);
 	}

packages/rspack-test-tools/src/case/new-incremental.ts

@@ -37,6 +37,8 @@ function getHotCreator(target: TTarget, documentType: EDocumentType) {
 					})
 				],
 				runner: HotRunnerFactory
+				// TODO: enable concurrent then rspack will be hanged
+				// concurrent: true
 			})
 		);
 	}
@@ -101,7 +103,8 @@ const watchCreator = new BasicCaseCreator({
 						watchState
 					)
 		);
-	}
+	},
+	concurrent: true
 });
 
 export function createWatchNewIncrementalCase(

packages/rspack-test-tools/src/case/watch.ts

@@ -50,7 +50,8 @@ const creator = new BasicCaseCreator({
 						watchState
 					)
 		);
-	}
+	},
+	concurrent: true
 });
 
 export function createWatchCase(

packages/rspack-test-tools/src/compiler.ts

@@ -84,6 +84,10 @@ export class TestCompilerManager<T extends ECompilerType>
 			throw new Error("Compiler should be created before watch");
 		this.compilerInstance!.watch(
 			{
+				// IMPORTANT:
+				// This is a workaround for the issue that watchpack cannot detect the file change in time
+				// so we set the poll to 300ms to make it more sensitive to the file change
+				poll: 300,
 				aggregateTimeout: timeout
 			},
 			(error, newStats) => {

packages/rspack-test-tools/src/helper/util/currentWatchStep.js

@@ -1 +1 @@
-exports.step = undefined;
+exports.step = {};

packages/rspack-test-tools/src/processor/watch.ts

@@ -60,7 +60,8 @@ export class WatchProcessor<
 	async build(context: ITestContext) {
 		const compiler = this.getCompiler(context);
 		const currentWatchStepModule = require(currentWatchStepModulePath);
-		currentWatchStepModule.step = this._watchOptions.stepName;
+		currentWatchStepModule.step[this._options.name] =
+			this._watchOptions.stepName;
 		fs.mkdirSync(this._watchOptions.tempDir, { recursive: true });
 		copyDiff(
 			path.join(context.getSource(), this._watchOptions.stepName),
@@ -345,7 +346,8 @@ export class WatchStepProcessor<
 	async build(context: ITestContext) {
 		const compiler = this.getCompiler(context);
 		const currentWatchStepModule = require(currentWatchStepModulePath);
-		currentWatchStepModule.step = this._watchOptions.stepName;
+		currentWatchStepModule.step[this._options.name] =
+			this._watchOptions.stepName;
 		const task = new Promise((resolve, reject) => {
 			compiler.getEmitter().once(ECompilerEvent.Build, (e, stats) => {
 				if (e) return reject(e);

packages/rspack-test-tools/src/test/creator.ts

@@ -14,6 +14,11 @@ import type {
 } from "../type";
 import { Tester } from "./tester";
 
+interface IConcurrentTestEnv {
+	clear: () => void;
+	run: () => Promise<void>;
+}
+
 export interface IBasicCaseCreatorOptions<T extends ECompilerType> {
 	clean?: boolean;
 	describe?: boolean;
@@ -34,9 +39,15 @@ export interface IBasicCaseCreatorOptions<T extends ECompilerType> {
 		context: ITestContext
 	) => TRunnerFactory<ECompilerType>;
 	[key: string]: unknown;
+	concurrent?: boolean | number;
 }
 
+const DEFAULT_MAX_CONCURRENT = 5;
+
 export class BasicCaseCreator<T extends ECompilerType> {
+	protected currentConcurrent = 0;
+	protected tasks: [string, () => void][] = [];
+
 	constructor(protected _options: IBasicCaseCreatorOptions<T>) {}
 
 	create(name: string, src: string, dist: string, temp?: string) {
@@ -55,16 +66,97 @@ export class BasicCaseCreator<T extends ECompilerType> {
 		}
 
 		const tester = this.createTester(name, src, dist, temp, testConfig);
+		const concurrent =
+			testConfig.concurrent ?? this._options.concurrent ?? false;
 
 		if (this._options.describe) {
-			describe(name, () => this.describe(name, tester, testConfig));
+			if (concurrent) {
+				describe(name, () => this.describeConcurrent(name, tester, testConfig));
+			} else {
+				describe(name, () => this.describe(name, tester, testConfig));
+			}
 		} else {
-			this.describe(name, tester, testConfig);
+			if (concurrent) {
+				this.describeConcurrent(name, tester, testConfig);
+			} else {
+				this.describe(name, tester, testConfig);
+			}
 		}
 
 		return tester;
 	}
 
+	protected describeConcurrent(
+		name: string,
+		tester: ITester,
+		testConfig: TTestConfig<T>
+	) {
+		beforeAll(async () => {
+			await tester.prepare();
+		});
+
+		let starter = null;
+		let chain = new Promise<void>((resolve, reject) => {
+			starter = resolve;
+		});
+		const ender = this.registerConcurrentTask(name, starter!);
+		const env = this.createConcurrentEnv();
+		let bailout = false;
+		for (let index = 0; index < tester.total; index++) {
+			let stepSignalResolve = null;
+			let stepSignalReject = null;
+			const stepSignal = new Promise((resolve, reject) => {
+				stepSignalResolve = resolve;
+				stepSignalReject = reject;
+			});
+			const description =
+				typeof this._options.description === "function"
+					? this._options.description(name, index)
+					: `step ${index ? `[${index}]` : ""} should pass`;
+			it(
+				description,
+				async () => {
+					await stepSignal;
+				},
+				this._options.timeout || 180000
+			);
+
+			chain = chain.then(async () => {
+				try {
+					if (bailout) {
+						throw `Case "${name}" step ${index + 1} bailout because ${tester.step + 1} failed`;
+					}
+					env.clear();
+					await tester.compile();
+					await tester.check(env);
+					await env.run();
+					const context = tester.getContext();
+					if (!tester.next() && context.hasError()) {
+						bailout = true;
+						const errors = context
+							.getError()
+							.map(i => `${i.stack}`.split("\n").join("\t\n"))
+							.join("\n\n");
+						throw new Error(
+							`Case "${name}" failed at step ${tester.step + 1}:\n${errors}`
+						);
+					}
+					stepSignalResolve!();
+				} catch (e) {
+					stepSignalReject!(e);
+				}
+			});
+		}
+
+		chain.finally(() => {
+			ender();
+		});
+
+		afterAll(async () => {
+			await tester.resume();
+		});
+	}
+
 	protected describe(
 		name: string,
 		tester: ITester,
@@ -74,12 +166,12 @@ export class BasicCaseCreator<T extends ECompilerType> {
 			await tester.prepare();
 		});
 
+		let bailout = false;
 		for (let index = 0; index < tester.total; index++) {
 			const description =
 				typeof this._options.description === "function"
 					? this._options.description(name, index)
 					: `step ${index ? `[${index}]` : ""} should pass`;
-			let bailout = false;
 			it(
 				description,
 				async () => {
@@ -110,6 +202,72 @@ export class BasicCaseCreator<T extends ECompilerType> {
 		});
 	}
 
+	protected createConcurrentEnv(): ITestEnv & IConcurrentTestEnv {
+		const tasks: [string, () => Promise<void> | void][] = [];
+		const beforeTasks: (() => Promise<void> | void)[] = [];
+		const afterTasks: (() => Promise<void> | void)[] = [];
+		return {
+			clear: () => {
+				tasks.length = 0;
+				beforeTasks.length = 0;
+				afterTasks.length = 0;
+			},
+			run: async () => {
+				const runFn = async (
+					fn: (done?: (e?: Error) => void) => Promise<void> | void
+				) => {
+					if (fn.length) {
+						await new Promise<void>((resolve, reject) => {
+							fn(e => {
+								if (e) {
+									reject(e);
+								} else {
+									resolve();
+								}
+							});
+						});
+					} else {
+						const res = fn();
+						if (typeof res?.then === "function") {
+							await res;
+						}
+					}
+				};
+
+				for (const [description, fn] of tasks) {
+					for (const before of beforeTasks) {
+						await runFn(before);
+					}
+					try {
+						await runFn(fn);
+					} catch (e) {
+						throw new Error(
+							`Error: ${description} failed\n${(e as Error).stack}`
+						);
+					}
+					for (const after of afterTasks) {
+						await runFn(after);
+					}
+				}
+			},
+			expect,
+			it: (description: string, fn: () => Promise<void> | void) => {
+				expect(typeof description === "string");
+				expect(typeof fn === "function");
+				tasks.push([description, fn]);
+			},
+			beforeEach: (fn: () => Promise<void> | void) => {
+				expect(typeof fn === "function");
+				beforeTasks.push(fn);
+			},
+			afterEach: (fn: () => Promise<void> | void) => {
+				expect(typeof fn === "function");
+				afterTasks.push(fn);
+			},
+			jest
+		};
+	}
+
 	protected createEnv(testConfig: TTestConfig<T>): ITestEnv {
 		if (typeof this._options.runner === "function" && !testConfig.noTest) {
 			return createLazyTestEnv(10000);
@@ -178,4 +336,30 @@ export class BasicCaseCreator<T extends ECompilerType> {
 			})
 		});
 	}
+
+	protected tryRunTask() {
+		while (
+			this.tasks.length !== 0 &&
+			this.currentConcurrent < this.getMaxConcurrent()
+		) {
+			const [_name, starter] = this.tasks.shift()!;
+			this.currentConcurrent++;
+			starter();
+		}
+	}
+
+	protected getMaxConcurrent() {
+		return typeof this._options.concurrent === "number"
+			? this._options.concurrent
+			: DEFAULT_MAX_CONCURRENT;
+	}
+
+	protected registerConcurrentTask(name: string, starter: () => void) {
+		this.tasks.push([name, starter]);
+		this.tryRunTask();
+		return () => {
+			this.currentConcurrent--;
+			this.tryRunTask();
+		};
+	}
 }