[e2e] speed up unavailableApis.spec.ts, make api calls retries env …

…variable (#968) closes #916 closes #993 ## Changes - dev: Adds `SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS` as an environment variable - dev: Disables `React-Query` _retries_ based on `SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS` - e2e: Updates `unavailableApis.spec.ts` to not wait for retries - e2e: further utilized `uploadFile.ts` to deduplicate code usage - e2e: `yarn test:e2e` edited to include chmod and `./run-playwright.sh` ## How to test this PR 1. Add `SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="false"` to your `.env` 2. ~~`chmod +x run-playwright.sh`~~ 3. ~~`sh run-playwright.sh` or `sh run-playwright.sh --headless`~~ 4. Run `yarn test:e2e` 5. Run `unavailableApis.spec.ts` 6. Verify the test passes and that the blocked APIs show up only once (see screenshot) ## Screenshots <img width="1392" alt="Screenshot 2024-10-03 at 3 57 20 PM" src="https://github.com/user-attachments/assets/de80dbf5-458a-484b-89e8-8a8c68de9d92"> ## Notes - ~~The tester must now manually set `SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS` to `true` before running tests~~ - The script `run-playwright.sh` is used so that the tester doesn't need to manually flip `SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS` before tests ## Questions - Do we want an actual environment variable to manually control the amount of `React-Query` retries for all requests or is this approach (i.e. `SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS` -- a test flag) sufficient?
cfpb · Oct 9, 2024 · 1513c6a · 1513c6a
1 parent 1ab3869
commit 1513c6a
Show file tree

Hide file tree

Showing 22 changed files with 133 additions and 95 deletions.
diff --git a/.env.example b/.env.example
@@ -9,3 +9,4 @@ SBL_LOGOUT_REDIRECT_URL=""
 SBL_VALIDATION_TIMEOUT_SECONDS="1200"
 SBL_LONGPOLLING_DELAY_SECONDS="backoff"
 SBL_UPLOAD_FILE_SIZE_LIMIT_BYTES="50000000"
+SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="false"
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -18,7 +18,8 @@ env:
   SBL_VALIDATION_TIMEOUT_SECONDS: "1200"
   SBL_LONGPOLLING_DELAY_SECONDS: "backoff"
   SBL_UPLOAD_FILE_SIZE_LIMIT_BYTES: "50000000"
-
+  SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS: "false"
+
 jobs:
   React:
     runs-on: ubuntu-latest

diff --git a/e2e/example.spec.demo.ts b/e2e/example.spec.demo.ts
@@ -1,14 +1,12 @@
 // this test won't run, but may be helpful as an example when writing future tests
 
 import { expect } from '@playwright/test';
-import path from 'node:path';
 import { test } from './fixtures/testFixture';
 import pointOfContactJson from './test-data/point-of-contact/point-of-contact-data-1.json';
+import { ResultUploadMessage, uploadFile } from './utils/uploadFile';
 
 test('proof of concept', async ({ page }) => {
   test.slow();
-  const tenSecondTimeout = 10_000;
-  const sixtySecondTimeout = 60_000;
   const minorDelay = 500;
 
   await test.step('Unauthenticated homepage: navigate to Authenticated homepage', async () => {
@@ -45,28 +43,15 @@ test('proof of concept', async ({ page }) => {
   });
 
   await test.step('Upload file: navigate to Review warnings after only warnings upload', async () => {
-    await test.step('Upload file: upload small file with only warnings (sbl-validations-all-pass-small.csv)', async () => {
-      await expect(page.locator('h2')).toContainText('Select a file to upload');
-      const fileChooserPromise = page.waitForEvent('filechooser');
-      await page.getByLabel('Select a .csv file to upload').click();
-      const fileChooser = await fileChooserPromise;
-      await fileChooser.setFiles(
-        path.join(
-          // eslint-disable-next-line unicorn/prefer-module
-          __dirname,
-          './test-data/sample-sblar-files/sbl-validations-all-pass-small.csv',
-        ),
-      );
-      await expect(page.getByText('File upload in progress')).toBeVisible();
-      await expect(page.getByText('File upload successful')).toBeVisible({
-        timeout: tenSecondTimeout,
-      });
-      await expect(page.getByText('Validation checks in progress')).toBeVisible(
-        { timeout: tenSecondTimeout },
-      );
-      await expect(
-        page.getByText('Warnings were found in your file'),
-      ).toBeVisible({ timeout: sixtySecondTimeout });
+    await uploadFile({
+      testUsed: test,
+      pageUsed: page,
+      newUpload: true,
+      testTitle:
+        'Upload file: upload small file with only warnings (sbl-validations-all-pass-small.csv)',
+      filePath:
+        '../test-data/sample-sblar-files/sbl-validations-all-pass-small.csv',
+      resultMessage: ResultUploadMessage.warning,
     });
 
     await test.step('Upload file: navigate to Resolve errors (syntax) with no errors after upload', async () => {

diff --git a/e2e/fixtures/testFixture.ts b/e2e/fixtures/testFixture.ts
@@ -1,6 +1,5 @@
 import type { Page } from '@playwright/test';
 import { test as baseTest, expect } from '@playwright/test';
-import path from 'node:path';
 import pointOfContactJson from '../test-data/point-of-contact/point-of-contact-data-1.json';
 import createDomainAssociation from '../utils/createDomainAssociation';
 import createInstitution from '../utils/createInstitution';
@@ -12,6 +11,7 @@ import {
   expectedWithAssociationsUrl,
   getTestDataObject,
 } from '../utils/testFixture.utils';
+import { ResultUploadMessage, uploadFile } from '../utils/uploadFile';
 
 export const test = baseTest.extend<{
   isNonAssociatedUser: boolean; // Skips creating a domain association and creating a financial institution
@@ -195,29 +195,15 @@ export const test = baseTest.extend<{
   ) => {
     navigateToUploadFile;
     await test.step('Upload file: navigate to Review warnings after only warnings upload', async () => {
-      await test.step('Upload file: upload small file with only warnings (sbl-validations-all-pass-small.csv)', async () => {
-        await expect(page.locator('h2')).toContainText(
-          'Select a file to upload',
-        );
-        const fileChooserPromise = page.waitForEvent('filechooser');
-        await page.getByLabel('Select a .csv file to upload').click();
-        const fileChooser = await fileChooserPromise;
-        await fileChooser.setFiles(
-          path.join(
-            __dirname,
-            '../test-data/sample-sblar-files/sbl-validations-all-pass-small.csv',
-          ),
-        );
-        await expect(page.getByText('File upload in progress')).toBeVisible();
-        await expect(page.getByText('File upload successful')).toBeVisible({
-          timeout: 30_000,
-        });
-        await expect(
-          page.getByText('Validation checks in progress'),
-        ).toBeVisible({ timeout: 10_000 });
-        await expect(
-          page.getByText('Warnings were found in your file'),
-        ).toBeVisible({ timeout: 60_000 });
+      await uploadFile({
+        testUsed: test,
+        pageUsed: page,
+        newUpload: true,
+        testTitle:
+          'Upload file: upload small file with only warnings (sbl-validations-all-pass-small.csv)',
+        filePath:
+          '../test-data/sample-sblar-files/sbl-validations-all-pass-small.csv',
+        resultMessage: ResultUploadMessage.warning,
       });
 
       await test.step('Upload file: navigate to Resolve errors (syntax) with no errors after upload', async () => {

diff --git a/e2e/pages/filing-app/formAlerts.spec.ts b/e2e/pages/filing-app/formAlerts.spec.ts
@@ -1,6 +1,6 @@
 import { expect } from '@playwright/test';
 import { test } from '../../fixtures/testFixture';
-import uploadFile from '../../utils/uploadFile';
+import { ResultUploadMessage, uploadFile } from '../../utils/uploadFile';
 
 test('Form Alerts', async ({
   page,
@@ -48,7 +48,15 @@ test('Form Alerts', async ({
 
   // Upload file with syntax errors
   await test.step('Upload syntax errors file', async () => {
-    await uploadFile(page, true, 'syntax');
+    await uploadFile({
+      testUsed: test,
+      pageUsed: page,
+      newUpload: true,
+      testTitle:
+        'Upload file: upload small file with syntax errors (all_syntax_errors.csv)',
+      filePath: '../test-data/sample-sblar-files/all_syntax_errors.csv',
+      resultMessage: ResultUploadMessage.syntax,
+    });
 
     // Continue to next page
     await test.step('Click: Continue', async () => {
@@ -87,7 +95,16 @@ test('Form Alerts', async ({
 
   // Upload file with logic errors
   await test.step('Upload logic errors file', async () => {
-    await uploadFile(page, false, 'logic');
+    await uploadFile({
+      testUsed: test,
+      pageUsed: page,
+      newUpload: false,
+      testTitle:
+        'Upload file: upload small file with logic errors (logic-errors_single&multi_and_warnings.csv)',
+      filePath:
+        '../test-data/sample-sblar-files/logic-errors_single&multi_and_warnings.csv',
+      resultMessage: ResultUploadMessage.logic,
+    });
 
     // Continue to next page
     await test.step('Click: Continue', async () => {
@@ -141,7 +158,15 @@ test('Form Alerts', async ({
 
   // Upload file with warnings
   await test.step('Upload warnings file', async () => {
-    await uploadFile(page, false, 'warning');
+    // Upload file
+    await uploadFile({
+      testUsed: test,
+      pageUsed: page,
+      newUpload: false,
+      testTitle: 'Upload Warnings file',
+      filePath: '../test-data/sample-sblar-files/logic-warnings_small.csv',
+      resultMessage: ResultUploadMessage.warning,
+    });
 
     await test.step('Click: Continue', async () => {
       await page

diff --git a/e2e/pages/filing-app/unavailableApis.spec.ts b/e2e/pages/filing-app/unavailableApis.spec.ts
@@ -3,7 +3,7 @@ import { test } from '../../fixtures/testFixture';
 import { blockApi } from '../../utils/blockApi';
 import { ResultUploadMessage, uploadFile } from '../../utils/uploadFile';
 
-test('Form Alerts and API', async ({
+test('Blocking API Calls - Error Boundaries', async ({
   page,
   navigateToProvideTypeOfFinancialInstitution,
 }) => {
@@ -26,10 +26,6 @@ test('Form Alerts and API', async ({
       await test.step('Refresh page', async () => {
         await page.reload();
       });
-      // ToDo: Make retries less when testing (#916)
-      await test.step('Waiting for retries timeout', async () => {
-        await page.waitForSelector('h1', { state: 'visible' });
-      });
       await expect(page.locator('h1'), 'h1 is correct').toContainText(
         'An unknown error occurred',
       );
@@ -66,10 +62,6 @@ test('Form Alerts and API', async ({
       await test.step('Refresh page', async () => {
         await page.reload();
       });
-      // ToDo: Make retries less when testing (#916)
-      await test.step('Waiting for retries timeout', async () => {
-        await page.waitForSelector('h1', { state: 'visible' });
-      });
       await expect(page.locator('h1'), 'h1 is correct').toContainText(
         'An unknown error occurred',
       );
@@ -86,7 +78,7 @@ test('Form Alerts and API', async ({
       testUsed: test,
       pageUsed: page,
       newUpload: true,
-      testTitle: 'Upload file',
+      testTitle: 'Upload passing file with warnings',
       filePath:
         '../test-data/sample-sblar-files/sbl-validations-all-pass-small.csv',
       resultMessage: ResultUploadMessage.warning,
@@ -116,10 +108,6 @@ test('Form Alerts and API', async ({
       await test.step('Refresh page', async () => {
         await page.reload();
       });
-      // ToDo: Make retries less when testing (#916)
-      await test.step('Waiting for retries timeout', async () => {
-        await page.waitForSelector('h1', { state: 'visible' });
-      });
       await expect(page.locator('h1'), 'h1 is correct').toContainText(
         'An unknown error occurred',
       );
@@ -146,7 +134,9 @@ test('Form Alerts and API', async ({
 
   // Review warnings page
   await test.step('Review warnings page', async () => {
-    await expect(page.locator('h1')).toContainText('Review warnings');
+    await expect(page.locator('h1')).toContainText('Review warnings', {
+      timeout: 50_000,
+    });
 
     // Block API Call: **/v1/institutions/
     await test.step('Block API: /v1/institutions', async () => {
@@ -155,17 +145,16 @@ test('Form Alerts and API', async ({
 
     // Confirm Error Alert
     await test.step('Error Alert is visible', async () => {
-      // ToDo: Make retries less when testing (#916)
       test.setTimeout(150_000);
       await test.step('Refresh page', async () => {
         await page.reload();
       });
-      // ToDo: Make retries less when testing (#916)
-      await test.step('Waiting for retries timeout', async () => {
-        await page.waitForSelector('h1', { state: 'visible' });
-      });
+
       await expect(page.locator('h1'), 'h1 is visible').toContainText(
         'Review warnings',
+        {
+          timeout: 50_000,
+        },
       );
       await expect(
         page.locator('#main .m-notification__error'),
@@ -204,10 +193,7 @@ test('Form Alerts and API', async ({
       await test.step('Refresh page', async () => {
         await page.reload();
       });
-      // ToDo: Make retries less when testing (#916)
-      await test.step('Waiting for retries timeout', async () => {
-        await page.waitForSelector('h1', { state: 'visible' });
-      });
+
       await expect(page.locator('h1'), 'h1 is correct').toContainText(
         'An unknown error occurred',
       );

diff --git a/e2e/pages/filing-app/uploadFile/completeUploadLogicErrors.spec.ts b/e2e/pages/filing-app/uploadFile/completeUploadLogicErrors.spec.ts
@@ -14,7 +14,7 @@ test('Resolve Errors (Logic)', async ({ page, navigateToUploadFile }) => {
       pageUsed: page,
       newUpload: true,
       testTitle:
-        'Upload file: upload small file with only warnings (logic-errors_medium.csv)',
+        'Upload file: upload small file with logic errors (logic-errors_single&multi_and_warnings.csv)',
       filePath:
         '../test-data/sample-sblar-files/logic-errors_single&multi_and_warnings.csv',
       resultMessage: ResultUploadMessage.logic,

diff --git a/e2e/pages/filing-app/uploadFile/completeUploadSyntaxErrors.spec.ts b/e2e/pages/filing-app/uploadFile/completeUploadSyntaxErrors.spec.ts
@@ -14,7 +14,7 @@ test('Resolve Errors (Syntax)', async ({ page, navigateToUploadFile }) => {
       pageUsed: page,
       newUpload: true,
       testTitle:
-        'Upload file: upload small file with only warnings (all_syntax_errors.csv)',
+        'Upload file: upload small file with syntax errors (all_syntax_errors.csv)',
       filePath: '../test-data/sample-sblar-files/all_syntax_errors.csv',
       resultMessage: ResultUploadMessage.syntax,
     });

diff --git a/e2e/test-data/sample-sblar-files/logic-warnings_small.csv b/e2e/test-data/sample-sblar-files/logic-warnings_small.csv
@@ -0,0 +1,8 @@
+uid,app_date,app_method,app_recipient,ct_credit_product,ct_credit_product_ff,ct_guarantee,ct_guarantee_ff,ct_loan_term_flag,ct_loan_term,credit_purpose,credit_purpose_ff,amount_applied_for_flag,amount_applied_for,amount_approved,action_taken,action_taken_date,denial_reasons,denial_reasons_ff,pricing_interest_rate_type,pricing_init_rate_period,pricing_fixed_rate,pricing_adj_margin,pricing_adj_index_name,pricing_adj_index_name_ff,pricing_adj_index_value,pricing_origination_charges,pricing_broker_fees,pricing_initial_charges,pricing_mca_addcost_flag,pricing_mca_addcost,pricing_prepenalty_allowed,pricing_prepenalty_exists,census_tract_adr_type,census_tract_number,gross_annual_revenue_flag,gross_annual_revenue,naics_code_flag,naics_code,number_of_workers,time_in_business_type,time_in_business,business_ownership_status,num_principal_owners_flag,num_principal_owners,po_1_ethnicity,po_1_ethnicity_ff,po_1_race,po_1_race_anai_ff,po_1_race_asian_ff,po_1_race_baa_ff,po_1_race_pi_ff,po_1_gender_flag,po_1_gender_ff,po_2_ethnicity,po_2_ethnicity_ff,po_2_race,po_2_race_anai_ff,po_2_race_asian_ff,po_2_race_baa_ff,po_2_race_pi_ff,po_2_gender_flag,po_2_gender_ff,po_3_ethnicity,po_3_ethnicity_ff,po_3_race,po_3_race_anai_ff,po_3_race_asian_ff,po_3_race_baa_ff,po_3_race_pi_ff,po_3_gender_flag,po_3_gender_ff,po_4_ethnicity,po_4_ethnicity_ff,po_4_race,po_4_race_anai_ff,po_4_race_asian_ff,po_4_race_baa_ff,po_4_race_pi_ff,po_4_gender_flag,po_4_gender_ff
+000TESTFIUIDDONOTUSEXGXVID11XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+000TESTFIUIDDONOTUSEXGXVID12XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+000TESTFIUIDDONOTUSEXGXVID13XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+000TESTFIUIDDONOTUSEXGXVID14XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+000TESTFIUIDDONOTUSEXGXVID21XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+000TESTFIUIDDONOTUSEXGXVID31XTC1,20241201,1,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+000TESTFIUIDDONOTUSEXGXVID31XTC2,20241201,2,1,988,,999,,999,,999,,999,,,5,20241231,999,,999,,,,999,,,,,,999,,999,999,988,,988,,988,,988,988,,988,988,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
diff --git a/e2e/utils/uploadFile.ts b/e2e/utils/uploadFile.ts
@@ -47,7 +47,9 @@ export async function uploadFile({
     const fileChooserPromise = pageUsed.waitForEvent('filechooser');
     await (newUpload
       ? pageUsed.getByLabel('Select a .csv file to upload').click()
-      : pageUsed.getByLabel('Replace your previously').click());
+      : pageUsed
+          .getByLabel('Replace your previously uploaded .csv file')
+          .click());
     const fileChooser = await fileChooserPromise;
     await fileChooser.setFiles(path.join(__dirname, filePath));
     await expect(pageUsed.getByText('File upload in progress')).toBeVisible();

diff --git a/package.json b/package.json
@@ -22,8 +22,7 @@
     "seed": "cd ../sbl-project/dev_setup/mock_data/ && sh create_institutions.sh && sh insert_filing_period.sh",
     "test": "vitest",
     "test:ci": "vitest run",
-    "test:e2e": "yarn playwright test --ui --workers 4",
-    "test:e2e:headless": "yarn playwright test --workers 4",
+    "test:e2e": "chmod +x ./run-playwright.sh && sh ./run-playwright.sh",
     "test:e2e:ci": "vite build && yarn preview:test 'cypress run --record'",
     "format": "prettier -uw --cache .",
     "run-tsc": "tsc",

diff --git a/run-playwright.sh b/run-playwright.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Set SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS to "true" in the .env file and check if SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS exists in the .env file
+if grep -q 'SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS' .env; then
+  echo 'Updating SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="true" in .env'
+  sed -i '' 's/SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS=.*/SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="true"/' .env
+else
+  echo 'SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS not found, adding SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="true" to .env'
+  echo 'SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="true"' >> .env
+fi
+
+# Check for --headless argument
+if [ "$1" == "--headless" ]; then
+  echo "Running Playwright tests in headless mode..."
+  yarn playwright test --workers 4
+else
+  echo "Running Playwright tests with UI..."
+  yarn playwright test --ui --workers 4
+fi
+
+# Check if Playwright exited successfully
+if [ $? -eq 0 ]; then
+  echo "Playwright tests completed successfully."
+else
+  echo "Playwright tests failed."
+fi
+
+# Reset SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS to "false" in the .env file
+echo 'Resetting SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="false" in .env'
+sed -i '' 's/SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS=.*/SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS="false"/' .env
+
+# Confirm the changes
+echo 'SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS has been reset to "false" in .env'
diff --git a/src/api/common.ts b/src/api/common.ts
@@ -47,6 +47,13 @@ export const FILE_SIZE_LIMIT_BYTES = convertEnvVarNumber(
   FILE_SIZE_LIMIT_50MB,
 );
 
+export const IS_PLAYWRIGHT_TEST: boolean =
+  String(import.meta.env.SBL_ENABLE_PLAYWRIGHT_TEST_SETTINGS) === 'true';
+
+// React-Query set retries based on if Playwright test settings are enabled
+export const getRetries = (retries = 0): boolean | number =>
+  IS_PLAYWRIGHT_TEST ? false : retries;
+
 export interface ValidationError {
   message: string;
   errors: Record<string, string[]>;

diff --git a/src/main.tsx b/src/main.tsx
@@ -1,5 +1,6 @@
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
 import App from 'App';
+import { getRetries } from 'api/common';
 import { createRoot } from 'react-dom/client';
 import { AuthProvider } from 'react-oidc-context';
 import { MAX_RETRIES } from 'utils/constants';
@@ -21,10 +22,7 @@ const queryClient = new QueryClient({
   defaultOptions: {
     queries: {
       // staleTime: Number.POSITIVE_INFINITY,
-      // retry: MAX_RETRIES,
-      retry: (failureCount, error): boolean => {
-        return failureCount <= MAX_RETRIES;
-      },
+      retry: getRetries(MAX_RETRIES),
       refetchOnWindowFocus: false, // default: true
     },
   },