Skip to content

Commit

Permalink
[scout] validate config has tests before starting servers (#211918)
Browse files Browse the repository at this point in the history
## Summary

There is no need to start servers (~1.5 min run time) if there are no
tests matching filters or maybe config itself has all tests skipped.

This PR uses Playwright cli with `--list` flag to quickly validate
playwright config and exit with status code `2` (`1` is reserved for
errors during servers start or test failures). it also useful to know in
advance how many tests were about to run:

case 1: tests found

```
$ node scripts/scout.js run-tests --config x-pack/platform/plugins/private/discover_enhanced/ui_tests/parallel.playwright.config.ts --serverless=security
 info scout: Test server configuration saved at /Users/dmle/github/kibana/.scout/servers/local.json
 info scout: Validate Playwright config has tests
 info scout: Total: 5 tests in 2 files
 info Verifying Docker is installed.
   │ info Docker version 20.10.14, build a224086349
...
```

case 2: no tests found
```
$ node scripts/scout.js run-tests --config x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts --stateful
 info scout: Test server configuration saved at /Users/dmle/github/kibana/.scout/servers/local.json
 info scout: Validate Playwright config has tests
ERROR scout: No tests found in [x-pack/solutions/observability/plugins/observability_onboarding/ui_tests/playwright.config.ts]
```

(cherry picked from commit 30c4275)
  • Loading branch information
dmlemeshko committed Feb 20, 2025
1 parent 83aa968 commit 3b21172
Showing 1 changed file with 19 additions and 7 deletions.
26 changes: 19 additions & 7 deletions packages/kbn-scout/src/playwright/runner/run_tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
*/

import { resolve } from 'path';
import { exec } from 'child_process';
import { promisify } from 'util';
const execPromise = promisify(exec);

import { ToolingLog } from '@kbn/tooling-log';
import { withProcRunner } from '@kbn/dev-proc-runner';
Expand All @@ -28,7 +31,21 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {
const playwrightGrepTag = getPlaywrightGrepTag(config);
const playwrightConfigPath = options.configPath;

const cmd = resolve(REPO_ROOT, './node_modules/.bin/playwright');
const cmdArgs = ['test', `--config=${playwrightConfigPath}`, `--grep=${playwrightGrepTag}`];

await withProcRunner(log, async (procs) => {
log.info(`scout: Validate Playwright config has tests`);
try {
// '--list' flag tells Playwright to collect all the tests, but do not run it
const result = await execPromise(`${cmd} ${cmdArgs.join(' ')} --list`);
const lastLine = result.stdout.trim().split('\n').pop();
log.info(`scout: ${lastLine}`);
} catch (err) {
log.error(`scout: No tests found in [${playwrightConfigPath}]`);
process.exit(2); // code "2" means no tests found
}

const abortCtrl = new AbortController();

const onEarlyExit = (msg: string) => {
Expand Down Expand Up @@ -60,13 +77,8 @@ export async function runTests(log: ToolingLog, options: RunTestsOptions) {

// Running 'npx playwright test --config=${playwrightConfigPath}'
await procs.run(`playwright`, {
cmd: resolve(REPO_ROOT, './node_modules/.bin/playwright'),
args: [
'test',
`--config=${playwrightConfigPath}`,
`--grep=${playwrightGrepTag}`,
...(options.headed ? ['--headed'] : []),
],
cmd,
args: [...cmdArgs, ...(options.headed ? ['--headed'] : [])],
cwd: resolve(REPO_ROOT),
env: {
...process.env,
Expand Down

0 comments on commit 3b21172

Please sign in to comment.