running several playwright load tests at the same time (or in sequence)

[zoe-jobson-IL]

We have a number of playwright load test that we want to reuse for a nightly performance test, so far I have this

before:
  flow:
    - function: ensureXlOrgIsReady # default to smallest org
    - function: fetchAllUsers
scenarios:
  - engine: playwright
    testFunction: "platformNavigation"
  - engine: playwright
    testFunction: "appsec"
  - engine: playwright
    testFunction: "bookmarking"
  - engine: playwright
    testFunction: "crisisSim"
  - engine: playwright
    testFunction: "leaderboard"
  - engine: playwright
    testFunction: "mitre"
  - engine: playwright
    testFunction: "workforce"

and this

export { scenario as appsec } from "@/tests/artillery/app-sec.art";
export { scenario as bookmarking } from "@/tests/artillery/bookmarking.art";
export { scenario as crisisSim } from "@/tests/artillery/crisis-sim-multi-feature.art";
export { scenario as leaderboard } from "@/tests/artillery/leaderboard.art";
export { scenario as mitre } from "@/tests/artillery/mitre-click-on-sub-technique.art";
export { scenario as platformNavigation } from "@/tests/artillery/platform-navigation.art";
export { scenario as workforce } from "@/tests/artillery/workforce-for-perf.art";

export { fetchAllUsers } from "@/framework/artillery/fetchUsers";
export { ensureXlOrgIsReady } from "@/framework/artillery/XlOrg";
export { $rewriteMetricName } from "@/framework/artillery/rewriteMetricName";

the trouble is that the way multiple scenarios works means that I often get one of the scenarios picked 3 times and another not picked at all, which means that ensure stats that are based off the not-picked one fail because they don't get triggered.

Is there are way of telling the artillery runner that I want exactly N versions (e.g. 1 or 3 or whatever) of each scenario to run, or failing that a way I can run them in sequence, I tried changing my import/exporter to run them, but obviously I lose the browser between scenarios, though it did run the first in the list.

[shanti-razr]

I am also doing something similar. Having the ability to just say run all tests X number in a round robin would be way better than the current system where it is all random.

[hassy]

There are two ways to do this:

1. Assign weights to scenarios - https://www.artillery.io/docs/reference/test-script#scenarios-section. For example:

scenarios:
  - engine: playwright
    weight: 1
    testFunction: "platformNavigation"
  - engine: playwright
    weight: 2
    testFunction: "appsec"
  - engine: playwright
    weight: 3
    testFunction: "bookmarking"

The weight config above would make Artillery:

• pick bookmarking for around 50% of all virtual users (3 / (1 + 2 + 3))
• pick appsec for around 33% of all virtual users (2 / (1 + 2 + 3))
• pick platformNavigation for around 16% of all virtual users

(weights are relative to each other, and are probabilistic so the numbers won't be exact)

2. The other option to run multiple scenarios in a sequence would be to call them both from the same testFunction. Something like this:

import { scenario as platformNavigation } from "@/tests/artillery/platform-navigation.art";
import { scenario as appsec } from "@/tests/artillery/app-sec.art";

async function runInSequence(page) {
  // We're passing the same page object here so there should be no issues with reusing the same browser instance
  await platformNavigation(page);
  await appsec(page);
}

export const runInSequence;

scenarios:
  - engine: playwright
    testFunction: runInSequence

[zoe-jobson-IL]

When I try that I get this https://app.artillery.io/share/sh_8653fec1341f189239897c804fa212be988f1c9800e75faf77fe634c7310964e

this is my new test

import { EventEmitter, ScenarioContext } from "artillery";
import { Page } from "playwright";

import { ITestLike } from "@/framework/interactions/domain-interactions";
import { scenario as appsec } from "@/tests/artillery/app-sec.art";
import { scenario as bookmarking } from "@/tests/artillery/bookmarking.art";
import { scenario as crisisSim } from "@/tests/artillery/crisis-sim-multi-feature.art";
import { scenario as leaderboard } from "@/tests/artillery/leaderboard.art";
import { scenario as mitre } from "@/tests/artillery/mitre-click-on-sub-technique.art";
import { scenario as platformNavigation } from "@/tests/artillery/platform-navigation.art";
import { scenario as workforce } from "@/tests/artillery/workforce-for-perf.art";
export { fetchAllUsers } from "@/framework/artillery/fetchUsers";
import { UserDetails } from "@/framework/artillery/fetchUsers";
export { ensureXlOrgIsReady } from "@/framework/artillery/XlOrg";
export { $rewriteMetricName } from "@/framework/artillery/rewriteMetricName";
export async function runInSequence(
  page: Page,
  context: ScenarioContext<{
    users: { employees: UserDetails[]; managers: UserDetails[] };
  }>,
  _: EventEmitter,
  test: ITestLike
): Promise<void> {
  // We're passing the same page object here so there should be no issues with reusing the same browser instance
  // run all tests in sequence so that they get run th esame number of times and always get run.
  await platformNavigation(page, context, _, test);
  await appsec(page, context, _, test);
  await bookmarking(page, context, _, test);
  await crisisSim(page, context, _, test);
  await leaderboard(page, context, _, test);
  await mitre(page, context, _, test);
  await workforce(page, context, _, test);
}