test: add rendering tests (qlik-oss#99)

.circleci/config.yml

@@ -63,6 +63,23 @@ jobs:
           name: Unit tests
           command: yarn run test
 
+  rendering-tests:
+    <<: *defaults
+    docker:
+      - image: mcr.microsoft.com/playwright:focal
+    steps:
+      - attach_workspace:
+          at: ~/project
+      - run:
+          name: Rendering tests
+          command: yarn run test:rendering
+      - store_artifacts:
+          path: test/rendering/render.mjs-snapshots
+      - store_artifacts:
+          path: test/rendering/artifacts
+      - store_artifacts:
+          path: test/rendering/test-report
+
   types:
     <<: *defaults
     steps:
@@ -92,6 +109,9 @@ workflows:
       - unit-tests:
           requires:
             - build
+      - rendering-tests:
+          requires:
+            - build
       - lint:
           requires:
             - build

.gitignore

@@ -11,3 +11,6 @@ sn-pivot-table-ext/
 coverage/
 example/
 core/esm/
+test/rendering/artifacts
+test/rendering/test-report
+test/rendering/state.json

package.json

@@ -37,6 +37,9 @@
     "sense": "nebula sense",
     "types:check": "tsc --noEmit",
     "test": "jest",
+    "test:rendering": "playwright test",
+    "test:local:rendering": "./test/rendering/scripts/run-rendering-test.sh",
+    "test:local:update:screenshots": "./test/rendering/scripts/update-screenshots.sh",
     "spec": "scriptappy-from-jsdoc -c ./scriptappy.config.js"
   },
   "devDependencies": {
@@ -52,6 +55,7 @@
     "@nebula.js/cli-build": "2.10.0",
     "@nebula.js/cli-sense": "2.10.0",
     "@nebula.js/cli-serve": "2.10.0",
+    "@playwright/test": "1.21.1",
     "@rollup/plugin-typescript": "8.3.2",
     "@testing-library/dom": "8.13.0",
     "@testing-library/jest-dom": "5.16.4",

playwright.config.ts

@@ -0,0 +1,50 @@
+import { PlaywrightTestConfig, devices } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  use: {
+    // Run the browser in headless mode
+    // headless: false,
+    // Record trace for each test, but remove it from successful test runs.
+    trace: 'retain-on-failure',
+  },
+  // Look for test files in the "test/rendering" directory, relative to this configuration file
+  testDir: 'test/rendering',
+  testMatch: /render\.mjs/,
+  outputDir: './test/rendering/artifacts/',
+  reporter: [
+    ['list'],
+    [
+      'html',
+      {
+        outputFolder: './test/rendering/test-report',
+        open: process.env.CI ? 'never' : 'on-failure',
+      },
+    ],
+  ],
+  // Forbid test.only on CI
+  forbidOnly: !!process.env.CI,
+  // The maximum number of retry attempts per test, two retries on CI
+  retries: process.env.CI ? 2 : 0,
+  // Limit the number of workers
+  workers: 1,
+  // Multiple "projects" can run your tests in multiple browsers and configurations
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+    // {
+    //   name: 'webkit',
+    //   use: { ...devices['Desktop Safari'] },
+    // },
+  ],
+  expect: {
+    toMatchSnapshot: { threshold: 0.1 },
+  },
+};
+
+export default config;

test/rendering/README.md

@@ -0,0 +1,59 @@
+# Rendering Test
+
+## Overview
+
+Nebula development serve uses [mocked Qlik core engine (Enigma mocker)](https://github.com/qlik-oss/nebula.js/blob/master/apis/enigma-mocker/README.md) to render the sn-pivot-table based on the [testing fixture files](https://github.com/qlik-oss/nebula.js/tree/master/commands/serve/docs).
+
+Testing cases use [Playwright](https://playwright.dev/) to capture screenshots of those rendered charts and matches against baseline to ensure our charts look as intended.
+
+Visual regression testing flow:
+
+1. Start Nebula development serve and launch Playwright
+2. Iterate testing fixture files
+3. Create test case per testing fixture file
+4. Render a chart based on testing fixture file in Nebula serve using Enigma mocker
+5. Capture screenshot by Playwright
+6. Compare screenshot with baseline image
+
+To capture same screenshot on the same operating system both locally and on a CI server, we use a docker instance to take the contents of `dist` and start a http-server. The rendering tests then run on your local machine or on the CI server.
+
+## Updating snapshots
+
+If you've updated the UI, you need to run the update-screenshots.sh script:
+
+    # Install dependencies
+    yarn --frozen-lockfile
+
+    # Build nebula.js visualization
+    yarn build
+
+    chmod 777 ./test/rendering/scripts/update-screenshots.sh
+    yarn test:local:update:screenshots
+
+It will spin up a docker container with playwright and enable us to emulate our CI server for updating the reference screenshots. The `--update-snapshots` will generate new screenshots for you.
+
+Make sure to commit these after you've **confirmed the screenshot changes**.
+
+Sometimes tests might break, if you are certain no UI changes have been made just re-run the failed workflow.
+
+## Test cases description
+
+### scenario_1.fix.js
+
+2 dimension as rows and measure group (pseudo dimension) also as row.
+
+### scenario_2.fix.js
+
+1 dimension as row, 1 dimension as column and measure group (pseudo dimension) as column. Also a custom null value text.
+
+### scenario_3.fix.js
+
+1 dimension as column and 1 measure as column.
+
+### scenario_4.fix.js
+
+2 dimension as rows, first dimension have an expanded value.
+
+### scenario_5.fix.js
+
+2 dimension as columns, first dimension have an expanded value.