From 929ea2a477345444e6162b7274cc5d5dc8305248 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Zakrzewski?= <41862803+stanislaw-zakrzewski@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:05:11 +0200 Subject: [PATCH 1/3] [131] update mongodb validation for the case schema to allow search (#136) * Update libraries batch-2 first part * Move some testing to TODO * Update Yup when * update set field values * Update breaking changes * Update comment * Remove unused * update datepicker * Update libraries batch-2 second part * Update return values of the functions for countries * update typescript * Update mismatched versions * WIP update router and eslint + vite instead of cra * Update tests for new vitest library * Update command for test and coverage * Skip tests that were blocking flow * Update Location.test.tsx * Update varaible name * Fix merging issues * Update index.tsx * Update index.tsx * WIP tests for location and landing page * Fixed all landing page tests * Unskip and fix verification status indicator tests * Fix automated source form tests * WIP unmocking EditCase tests * Update EditCase.test.tsx * Fix viewcase tests * Resolve warnings regarding act * Update Dockerfile * Update Location.test.tsx * Fix tests failing at the first day of the month * Cleanup sourcetable * Update Users Uploads and Sources tables * Add backfill back It was removed after splitting one component into multiple * Rename collection from cases to day0cases * Update day0cases.schema.json * Update day0cases.schema.json * Initial schema work * initial schema work * Fix location lat/long update issue * Add missing schema element - isGovernmentSource for Additional Sources * Update tests for searchbar and data guide * Update README.md --- .../data-service/src/controllers/case.ts | 7 + .../migrations/20210902121948-initial.js | 99 +++++------ .../20210922082905-additional-case-indexes.js | 44 ++--- data-serving/scripts/setup-db/package.json | 1 + ...es.indexes.json => day0cases.indexes.json} | 28 +++- ...ases.schema.json => day0cases.schema.json} | 154 +++++++++++++++++- dev/README.md | 5 + .../e2e/components/LinelistTableTest.spec.ts | 59 +++++++ .../ui/src/components/DataGuideDialog.tsx | 12 +- .../ui/src/components/SearchBar.tsx | 3 +- .../new-case-form-fields/Location.tsx | 27 +-- 11 files changed, 348 insertions(+), 91 deletions(-) rename data-serving/scripts/setup-db/schemas/{cases.indexes.json => day0cases.indexes.json} (83%) rename data-serving/scripts/setup-db/schemas/{cases.schema.json => day0cases.schema.json} (60%) diff --git a/data-serving/data-service/src/controllers/case.ts b/data-serving/data-service/src/controllers/case.ts index f90663ee2..02ce0eaef 100644 --- a/data-serving/data-service/src/controllers/case.ts +++ b/data-serving/data-service/src/controllers/case.ts @@ -795,6 +795,13 @@ export class CasesController { res.status(201).json(result); } catch (e) { const err = e as Error; + if (err.name === 'MongoServerError') { + logger.error((e as any).errInfo); + res.status(422).json({ + message: (err as any).errInfo, + }); + return; + } if (err instanceof GeocodeNotFoundError) { res.status(404).json({ message: err.message, diff --git a/data-serving/scripts/setup-db/migrations/20210902121948-initial.js b/data-serving/scripts/setup-db/migrations/20210902121948-initial.js index 27063fa83..3b3bc08e5 100644 --- a/data-serving/scripts/setup-db/migrations/20210902121948-initial.js +++ b/data-serving/scripts/setup-db/migrations/20210902121948-initial.js @@ -1,57 +1,60 @@ const fs = require('fs'); module.exports = { - async up(db, client) { - await createCollectionValidationAndIndexes( - db, - 'cases', - 'schemas/cases.schema.json', - 'schemas/cases.indexes.json' - ); - - await createCollectionValidationAndIndexes( - db, - 'sources', - 'schemas/sources.schema.json', - 'schemas/sources.indexes.json' - ); - }, - - async down(db, client) { - // initial migration has no rollback - } + async up(db, client) { + await createCollectionValidationAndIndexes( + db, + 'day0cases', + 'schemas/day0cases.schema.json', + 'schemas/day0cases.indexes.json', + ); + + await createCollectionValidationAndIndexes( + db, + 'sources', + 'schemas/sources.schema.json', + 'schemas/sources.indexes.json', + ); + }, + + async down(db, client) { + // initial migration has no rollback + }, }; -async function createCollectionValidationAndIndexes(db, collectionName, schemaPath, indexPath) { - const schemaFile = await fs.promises.readFile(schemaPath); - const schema = JSON.parse(schemaFile); +async function createCollectionValidationAndIndexes( + db, + collectionName, + schemaPath, + indexPath, +) { + const schemaFile = await fs.promises.readFile(schemaPath); + const schema = JSON.parse(schemaFile); + + const indexFile = await fs.promises.readFile(indexPath); + const indexes = JSON.parse(indexFile); + /* + * because this migration might run against a DB from before we had the migrations infra, + * check whether the collection already exists. If it does, then modify its validation schema. + * If it doesn't, then create it. + */ + try { + await db.collection(collectionName); + await db.command({ + collMod: collectionName, + validator: schema, + }); + } catch { + await db.createCollection(collectionName, { + validator: schema, + }); + } + + const collection = db.collection(collectionName); + await collection.dropIndexes(); - const indexFile = await fs.promises.readFile(indexPath); - const indexes = JSON.parse(indexFile); - /* - * because this migration might run against a DB from before we had the migrations infra, - * check whether the collection already exists. If it does, then modify its validation schema. - * If it doesn't, then create it. - */ - try { - await db.collection(collectionName); await db.command({ - collMod: collectionName, - validator: schema, - }); - } - catch { - await db.createCollection(collectionName, { - validator: schema, + createIndexes: collectionName, + indexes: indexes, }); - } - - const collection = db.collection(collectionName); - await collection.dropIndexes(); - - await db.command({ - createIndexes: collectionName, - indexes: indexes, - }); } - diff --git a/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js b/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js index 4fe9bbbee..b47fbf266 100644 --- a/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js +++ b/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js @@ -1,29 +1,29 @@ const indexes = [ - { - name: 'byGenderAndCountry', - key: { - 'demographics.gender': -1, - 'location.countryISO3': -1 + { + name: 'byGenderAndCountry', + key: { + 'demographics.gender': -1, + 'location.countryISO3': -1, + }, + collation: { + locale: 'en_US', + strength: 2, + }, }, - collation: { - locale: 'en_US', - strength: 2, - }, - } ]; module.exports = { - async up(db, client) { - await db.command({ - createIndexes: 'cases', - indexes: indexes, - }); - }, + async up(db, client) { + await db.command({ + createIndexes: 'day0cases', + indexes: indexes, + }); + }, - async down(db, client) { - await db.command({ - dropIndexes: 'cases', - index: ['byGenderAndCountry'] - }); - } + async down(db, client) { + await db.command({ + dropIndexes: 'day0cases', + index: ['byGenderAndCountry'], + }); + }, }; diff --git a/data-serving/scripts/setup-db/package.json b/data-serving/scripts/setup-db/package.json index 1eec4d3a5..fcdd3b211 100644 --- a/data-serving/scripts/setup-db/package.json +++ b/data-serving/scripts/setup-db/package.json @@ -8,6 +8,7 @@ "lint": "tsc --noEmit && eslint '*/**/*.{js,ts,tsx}' --quiet --fix", "import-sample-data": "python3 ./import-sample-data.py", "migrate": "npm ci && migrate-mongo up", + "migrate-down": "npm ci && migrate-mongo down", "delete-all-cases": "mongosh $CONN --eval 'db.cases.deleteMany({})'" }, "repository": { diff --git a/data-serving/scripts/setup-db/schemas/cases.indexes.json b/data-serving/scripts/setup-db/schemas/day0cases.indexes.json similarity index 83% rename from data-serving/scripts/setup-db/schemas/cases.indexes.json rename to data-serving/scripts/setup-db/schemas/day0cases.indexes.json index 210417dd7..a7007c6b6 100644 --- a/data-serving/scripts/setup-db/schemas/cases.indexes.json +++ b/data-serving/scripts/setup-db/schemas/day0cases.indexes.json @@ -4,7 +4,9 @@ "key": { "demographics.occupation": "text", "location.country": "text", - "location.city": "text", + "location.admin1": "text", + "location.admin2": "text", + "location.admin3": "text", "caseReference.sourceUrl": "text", "caseStatus": "text" } @@ -74,9 +76,29 @@ } }, { - "name": "locationCityIdx", + "name": "locationAdmin1Idx", "key": { - "location.city": -1 + "location.admin1": -1 + }, + "collation": { + "locale": "en_US", + "strength": 2 + } + }, + { + "name": "locationAdmin2Idx", + "key": { + "location.admin2": -1 + }, + "collation": { + "locale": "en_US", + "strength": 2 + } + }, + { + "name": "locationAdmin3Idx", + "key": { + "location.admin3": -1 }, "collation": { "locale": "en_US", diff --git a/data-serving/scripts/setup-db/schemas/cases.schema.json b/data-serving/scripts/setup-db/schemas/day0cases.schema.json similarity index 60% rename from data-serving/scripts/setup-db/schemas/cases.schema.json rename to data-serving/scripts/setup-db/schemas/day0cases.schema.json index addf0e2c8..4175de3de 100644 --- a/data-serving/scripts/setup-db/schemas/cases.schema.json +++ b/data-serving/scripts/setup-db/schemas/day0cases.schema.json @@ -2,6 +2,7 @@ "$jsonSchema": { "bsonType": "object", "additionalProperties": false, + "required": ["caseStatus"], "properties": { "_id": { "bsonType": "number", @@ -13,6 +14,102 @@ "list": { "bsonType": "bool" }, + "caseStatus": { + "bsonType": "string" + }, + "comment": { + "bsonType": "string" + }, + "curators": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "verifiedBy": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "_id": { + "bsonType": "objectId" + }, + "name": { + "bsonType": "string" + }, + "email": { + "bsonType": "string" + }, + "roles": { + "bsonType": "array", + "uniqueItems": true, + "items": { + "bsonType": "string" + } + } + } + }, + "createdBy": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "_id": { + "bsonType": "objectId" + }, + "name": { + "bsonType": "string" + }, + "email": { + "bsonType": "string" + }, + "roles": { + "bsonType": "array", + "uniqueItems": true, + "items": { + "bsonType": "string" + } + } + } + } + } + }, + "revisionMetadata": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "revisionNumber": { + "bsonType": "number", + "minimum": 0 + }, + "creationMetadata": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "curator": { + "bsonType": "string" + }, + "date": { + "bsonType": "date" + }, + "notes": { + "bsonType": "string" + } + } + }, + "updateMetadata": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "curator": { + "bsonType": "string" + }, + "date": { + "bsonType": "date" + }, + "notes": { + "bsonType": "string" + } + } + } + } + }, "caseReference": { "bsonType": "object", "additionalProperties": false, @@ -26,6 +123,9 @@ "sourceUrl": { "bsonType": "string" }, + "isGovernmentSource": { + "bsonType": "bool" + }, "uploadIds": { "bsonType": "array", "uniqueItems": true, @@ -42,6 +142,9 @@ "properties": { "sourceUrl": { "bsonType": "string" + }, + "isGovernmentSource": { + "bsonType": "bool" } } } @@ -68,6 +171,13 @@ } } }, + "ageBuckets": { + "bsonType": "array", + "uniqueItems": true, + "items": { + "bsonType": "objectId" + } + }, "gender": { "bsonType": "string" }, @@ -111,16 +221,56 @@ "bsonType": "object", "additionalProperties": false, "properties": { + "admin1": { + "bsonType": "string" + }, + "admin1WikiId": { + "bsonType": "string" + }, + "admin2": { + "bsonType": "string" + }, + "admin2WikiId": { + "bsonType": "string" + }, + "admin3": { + "bsonType": "string" + }, + "admin3WikiId": { + "bsonType": "string" + }, + "comment": { + "bsonType": "string" + }, "country": { "bsonType": "string" }, "countryISO3": { "bsonType": "string" }, + "geometry": { + "bsonType": "object", + "additionalProperties": false, + "properties": { + "latitude": { + "bsonType": "number", + "minimum": -90, + "maximum": 90 + }, + "longitude": { + "bsonType": "number", + "minimum": -180, + "maximum": 180 + } + } + }, + "geoResolution": { + "bsonType": "string" + }, "location": { "bsonType": "string" }, - "place": { + "name": { "bsonType": "string" }, "query": { @@ -135,7 +285,7 @@ "bsonType": ["date"] }, "dateLastModified": { - "bsonType": ["date"] + "bsonType": ["date", "null"] }, "dateOnset": { "bsonType": ["date"] diff --git a/dev/README.md b/dev/README.md index b05064b04..619e09eb6 100644 --- a/dev/README.md +++ b/dev/README.md @@ -52,6 +52,11 @@ For local development, it's fine to use your own values for these secrets if you have them. For instance, if you have a developer Mapbox API token, or if you'd like to use a different GMail account for mailing notifications, or different OAuth client values. +#### Database Schema + +Before inputting any case data to the local portal instance a database schema migration needs to be run. +Navigate to the `data-serving/scripts/setup-db/` and follow instructions in the [README](../data-serving/scripts/setup-db/README.md) to set up the database schema. + #### Permissions Give your user all the permissions to access the portal and make CRUD updates. diff --git a/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts b/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts index e11307ab7..7cc942796 100644 --- a/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts +++ b/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts @@ -491,4 +491,63 @@ describe('Linelist table', function () { cy.get('tr').eq(1).contains('France'); }); + + it('Can search the data', () => { + cy.addCase({ + country: 'France', + countryISO3: 'FRA', + dateEntry: '2020-05-01', + dateReported: '2020-05-01', + sourceUrl: 'www.example.com', + caseStatus: CaseStatus.Confirmed, + occupation: 'Actor', + }); + cy.addCase({ + country: 'Germany', + countryISO3: 'DEU', + dateEntry: '2020-05-01', + dateReported: '2020-05-01', + sourceUrl: 'www.example.com', + caseStatus: CaseStatus.Confirmed, + occupation: 'Doctor', + }); + cy.addCase({ + country: 'Argentina', + countryISO3: 'ARG', + dateEntry: '2020-05-01', + dateReported: '2020-05-01', + sourceUrl: 'www.example.com', + caseStatus: CaseStatus.Confirmed, + occupation: 'Engineer', + }); + + cy.intercept('GET', getDefaultQuery({ limit: 50 })).as('getCases'); + cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'Argentina' })).as('getCasesWithSearch1'); + cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'Doctor' })).as('getCasesWithSearch2'); + + cy.visit('/cases'); + cy.wait('@getCases'); + + cy.contains('Argentina').should('exist'); + cy.contains('France').should('exist'); + cy.contains('Germany').should('exist'); + + cy.get('#search-field').type('Argentina'); + cy.wait('@getCasesWithSearch1'); + cy.contains('Argentina').should('exist'); + cy.contains('France').should('not.exist'); + cy.contains('Germany').should('not.exist'); + + cy.get('#clear-search').click(); + cy.wait('@getCases'); + cy.contains('Argentina').should('exist'); + cy.contains('France').should('exist'); + cy.contains('Germany').should('exist'); + + cy.get('#search-field').type('Doctor'); + cy.wait('@getCasesWithSearch2'); + cy.contains('Argentina').should('not.exist'); + cy.contains('France').should('not.exist'); + cy.contains('Germany').should('exist'); + }); }); diff --git a/verification/curator-service/ui/src/components/DataGuideDialog.tsx b/verification/curator-service/ui/src/components/DataGuideDialog.tsx index 95c406d5b..d24c7a03c 100644 --- a/verification/curator-service/ui/src/components/DataGuideDialog.tsx +++ b/verification/curator-service/ui/src/components/DataGuideDialog.tsx @@ -96,8 +96,16 @@ const SearchGuideDialog = ({ the left and choose ascending or descending. - For full-text search, enter any - combination of search terms. + For full-text search, enter any combination of search terms. +
+ Full-text search covers: occupation, admin0, admin1, admin2, admin3, sourceUrl and caseStatus. +
+ Search terms must be exact (example: "German" will not match "Germany"). +
+ Full-text search matches cases that contain any ot the search terms, not a combination. +
+ No special characters apart from "." are allowed and if the "." is used in a search term + given search term must be contained within quotation marks. You can use the icons on the right to navigate diff --git a/verification/curator-service/ui/src/components/SearchBar.tsx b/verification/curator-service/ui/src/components/SearchBar.tsx index 9b9d79d53..a4ebf82e2 100644 --- a/verification/curator-service/ui/src/components/SearchBar.tsx +++ b/verification/curator-service/ui/src/components/SearchBar.tsx @@ -203,7 +203,7 @@ export default function SearchBar({ } }} placeholder="Fulltext search" - value={searchInput} + value={decodeURI(searchInput)} variant="outlined" fullWidth InputProps={{ @@ -254,6 +254,7 @@ export default function SearchBar({ { setSearchInput(''); navigate({ diff --git a/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx b/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx index 607260287..906b812af 100644 --- a/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx +++ b/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx @@ -120,7 +120,6 @@ export default function Location(): JSX.Element { setSelectedAdmin1(foundAdmin1Entry); } } - // eslint-disable-next-line react-hooks/exhaustive-deps }, [values.location.countryISO3, admin1Entries]); useEffect(() => { @@ -177,8 +176,6 @@ export default function Location(): JSX.Element { // Update mapbox indicator for admin1 setAdmin1AvailableOnMap(!!values.location.admin1WikiId); - - // eslint-disable-next-line react-hooks/exhaustive-deps }, [values.location.admin1WikiId, values.location.admin1, admin2Entries]); useEffect(() => { @@ -235,8 +232,6 @@ export default function Location(): JSX.Element { // Update mapbox indicator for admin2 setAdmin2AvailableOnMap(!!values.location.admin2WikiId); - - // eslint-disable-next-line react-hooks/exhaustive-deps }, [values.location.admin2WikiId, values.location.admin2, admin3Entries]); useEffect(() => { @@ -258,8 +253,6 @@ export default function Location(): JSX.Element { // Update mapbox indicator for admin3 setAdmin3AvailableOnMap(!!values.location.admin3WikiId); - - // eslint-disable-next-line react-hooks/exhaustive-deps }, [values.location.admin3WikiId, values.location.admin3]); useEffect(() => { @@ -326,15 +319,14 @@ export default function Location(): JSX.Element { 'location.geometry.latitude', values.location.geocodeLocation.geometry?.latitude || values.location.geometry?.latitude || - '', + undefined, ); setFieldValue( 'location.geometry.longitude', values.location.geocodeLocation.geometry?.longitude || values.location.geometry?.longitude || - '', + undefined, ); - // eslint-disable-next-line react-hooks/exhaustive-deps }, [values.location.geocodeLocation]); return ( @@ -486,7 +478,10 @@ export default function Location(): JSX.Element { _: unknown, newValue: adminEntry | null, ): void => { - setFieldValue('location.admin1', newValue?.name || ''); + setFieldValue( + 'location.admin1', + newValue?.name || '', + ); setFieldValue( 'location.admin1WikiId', newValue?.wiki || '', @@ -588,7 +583,10 @@ export default function Location(): JSX.Element { _: unknown, newValue: adminEntry | null, ): void => { - setFieldValue('location.admin2', newValue?.name || ''); + setFieldValue( + 'location.admin2', + newValue?.name || '', + ); setFieldValue( 'location.admin2WikiId', newValue?.wiki || '', @@ -690,7 +688,10 @@ export default function Location(): JSX.Element { _: unknown, newValue: adminEntry | null, ): void => { - setFieldValue('location.admin3', newValue?.name || ''); + setFieldValue( + 'location.admin3', + newValue?.name || '', + ); setFieldValue( 'location.admin3WikiId', newValue?.wiki || '', From 881e39ef33e2dd1be1738c8ede4d683f226a3fac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Zakrzewski?= <41862803+stanislaw-zakrzewski@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:56:30 +0100 Subject: [PATCH 2/3] [98] add query for curator comments (#154) * Allow search for curators comment * Fix highlighting of the text in view case form * Update DataGuideDialog.tsx * Cleanup * Inform user if there is any not closed quote * Cleanup code + remove console.log * Add cypress tests for comments and uneven quote warning * WIP fixed issues with filters and special characters in free text search * Update index.tsx * Update SearchBar.tsx * Update index.tsx * Fix setting filters * Fix issues with text-search not visible * Cleanup * Cleanup SearchBar * Remove console.log --- .../setup-db/schemas/day0cases.indexes.json | 13 +++- .../e2e/components/LinelistTableTest.spec.ts | 27 +++++++ .../ui/cypress/support/commands.ts | 2 + .../ui/src/components/App/index.tsx | 6 +- .../ui/src/components/DataGuideDialog.tsx | 55 +++++++++++--- .../ui/src/components/FiltersDialog/index.tsx | 14 +++- .../ui/src/components/SearchBar.tsx | 57 ++++++++------- .../ui/src/components/ViewCase.tsx | 73 ++++++++++++------- 8 files changed, 178 insertions(+), 69 deletions(-) diff --git a/data-serving/scripts/setup-db/schemas/day0cases.indexes.json b/data-serving/scripts/setup-db/schemas/day0cases.indexes.json index a7007c6b6..b55dd708a 100644 --- a/data-serving/scripts/setup-db/schemas/day0cases.indexes.json +++ b/data-serving/scripts/setup-db/schemas/day0cases.indexes.json @@ -8,7 +8,8 @@ "location.admin2": "text", "location.admin3": "text", "caseReference.sourceUrl": "text", - "caseStatus": "text" + "caseStatus": "text", + "comment": "text" } }, { @@ -145,6 +146,16 @@ "strength": 2 } }, + { + "name": "commentIdx", + "key": { + "comment": -1 + }, + "collation": { + "locale": "en_US", + "strength": 2 + } + }, { "name": "countryAndDate", "key": { diff --git a/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts b/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts index 7cc942796..be520b375 100644 --- a/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts +++ b/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts @@ -501,6 +501,7 @@ describe('Linelist table', function () { sourceUrl: 'www.example.com', caseStatus: CaseStatus.Confirmed, occupation: 'Actor', + comment: 'note', }); cy.addCase({ country: 'Germany', @@ -524,6 +525,7 @@ describe('Linelist table', function () { cy.intercept('GET', getDefaultQuery({ limit: 50 })).as('getCases'); cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'Argentina' })).as('getCasesWithSearch1'); cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'Doctor' })).as('getCasesWithSearch2'); + cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'note' })).as('getCasesWithSearch3'); cy.visit('/cases'); cy.wait('@getCases'); @@ -549,5 +551,30 @@ describe('Linelist table', function () { cy.contains('Argentina').should('not.exist'); cy.contains('France').should('not.exist'); cy.contains('Germany').should('exist'); + + cy.get('#clear-search').click(); + cy.wait('@getCases'); + cy.contains('Argentina').should('exist'); + cy.contains('France').should('exist'); + cy.contains('Germany').should('exist'); + + cy.get('#search-field').type('note'); + cy.wait('@getCasesWithSearch3'); + cy.contains('Argentina').should('not.exist'); + cy.contains('France').should('exist'); + cy.contains('Germany').should('not.exist'); + }); + + it('Informs user when uneven number of quotes is present in free-text search', () => { + cy.intercept('GET', getDefaultQuery({ limit: 50, query: '"Bus driver"' })).as('getCasesWithSearch'); + cy.visit('/cases'); + cy.contains('Please make sure you have an even number of quotes.').should('not.exist'); + + cy.get('#search-field').type('"Bus driver'); + cy.contains('Please make sure you have an even number of quotes.').should('exist'); + + cy.get('#search-field').type('"'); + cy.wait('@getCasesWithSearch'); + cy.contains('Please make sure you have an even number of quotes.').should('not.exist'); }); }); diff --git a/verification/curator-service/ui/cypress/support/commands.ts b/verification/curator-service/ui/cypress/support/commands.ts index 4ddc86446..c951c0be8 100644 --- a/verification/curator-service/ui/cypress/support/commands.ts +++ b/verification/curator-service/ui/cypress/support/commands.ts @@ -43,6 +43,7 @@ interface AddCaseProps { gender?: Gender; outcome?: Outcome; uploadIds?: string[]; + comment?: string; } declare global { @@ -111,6 +112,7 @@ export function addCase(opts: AddCaseProps): void { travelHistory: {}, genomeSequences: {}, vaccination: {}, + comment: opts.comment || '', }, }); } diff --git a/verification/curator-service/ui/src/components/App/index.tsx b/verification/curator-service/ui/src/components/App/index.tsx index 0cc3fe81d..44c0e58c4 100644 --- a/verification/curator-service/ui/src/components/App/index.tsx +++ b/verification/curator-service/ui/src/components/App/index.tsx @@ -327,13 +327,14 @@ export default function App(): JSX.Element { }; const onModalClose = (): void => { + const searchQueryObject = new URLSearchParams(searchQuery); navigate( { pathname: location.state && location.state.lastLocation ? location.state.lastLocation : '/cases', - search: searchQuery, + search: searchQueryObject.toString(), }, { state: { lastLocation: '/case/view' } }, ); @@ -371,7 +372,8 @@ export default function App(): JSX.Element { ) return; - dispatch(setSearchQuery(location.search)); + const searchParams = new URLSearchParams(location.search); + dispatch(setSearchQuery(searchParams.toString())); // Save searchQuery to local storage not to lost it when user goes through auth process localStorage.setItem('searchQuery', location.search); diff --git a/verification/curator-service/ui/src/components/DataGuideDialog.tsx b/verification/curator-service/ui/src/components/DataGuideDialog.tsx index d24c7a03c..8a3c92801 100644 --- a/verification/curator-service/ui/src/components/DataGuideDialog.tsx +++ b/verification/curator-service/ui/src/components/DataGuideDialog.tsx @@ -1,7 +1,7 @@ import React, { useEffect } from 'react'; import { Box, Portal, Theme, Typography } from '@mui/material'; import { withStyles } from 'tss-react/mui'; -import CloseIcon from '@mui/icons-material/Close'; +import { Close as CloseIcon } from '@mui/icons-material'; import Draggable, { ControlPosition } from 'react-draggable'; // As per this issue from react-draggable library: https://github.com/react-grid-layout/react-draggable/pull/648 @@ -96,16 +96,49 @@ const SearchGuideDialog = ({ the left and choose ascending or descending. - For full-text search, enter any combination of search terms. -
- Full-text search covers: occupation, admin0, admin1, admin2, admin3, sourceUrl and caseStatus. -
- Search terms must be exact (example: "German" will not match "Germany"). -
- Full-text search matches cases that contain any ot the search terms, not a combination. -
- No special characters apart from "." are allowed and if the "." is used in a search term - given search term must be contained within quotation marks. + For full-text search, enter any + combination of search terms. Rules for full-text + search: +
+ You can use the icons on the right to navigate diff --git a/verification/curator-service/ui/src/components/FiltersDialog/index.tsx b/verification/curator-service/ui/src/components/FiltersDialog/index.tsx index 810a28c8d..a18d3c69e 100644 --- a/verification/curator-service/ui/src/components/FiltersDialog/index.tsx +++ b/verification/curator-service/ui/src/components/FiltersDialog/index.tsx @@ -17,7 +17,7 @@ import { useMediaQuery, } from '@mui/material'; import { Close as CloseIcon } from '@mui/icons-material'; -import { filtersToURL, URLToFilters } from '../util/searchQuery'; +import { URLToFilters } from '../util/searchQuery'; import { hasAnyRole } from '../util/helperFunctions'; import { useAppSelector, useAppDispatch } from '../../hooks/redux'; import { fetchCountries } from '../../redux/filters/thunk'; @@ -153,13 +153,19 @@ export default function FiltersDialog({ handleSetModalAlert(); dispatch(setModalOpen(false)); - const searchQuery = filtersToURL(values); + const searchParams = new URLSearchParams(); + for (const [key, value] of Object.entries(values)) { + if (value) searchParams.set(key, value); + } + const q = (new URLSearchParams(location.search)).get('q') + if (q) searchParams.set('q', q); + const searchParamsString = searchParams.toString(); - sendCustomGtmEvent('filters_applied', { query: searchQuery }); + sendCustomGtmEvent('filters_applied', { query: searchParamsString }); navigate({ pathname: '/cases', - search: searchQuery, + search: searchParamsString, }); }, }); diff --git a/verification/curator-service/ui/src/components/SearchBar.tsx b/verification/curator-service/ui/src/components/SearchBar.tsx index a4ebf82e2..f45287740 100644 --- a/verification/curator-service/ui/src/components/SearchBar.tsx +++ b/verification/curator-service/ui/src/components/SearchBar.tsx @@ -15,7 +15,6 @@ import clsx from 'clsx'; import DataGuideDialog from './DataGuideDialog'; import { useDebounce } from '../hooks/useDebounce'; import FiltersDialog from './FiltersDialog'; -import { searchQueryToURL, URLToSearchQuery } from './util/searchQuery'; import { useLocation, useNavigate } from 'react-router-dom'; import { KeyboardEvent, ChangeEvent } from 'react'; import { useAppSelector, useAppDispatch } from '../hooks/redux'; @@ -85,9 +84,7 @@ export default function SearchBar({ const [isUserTyping, setIsUserTyping] = useState(false); const [isDataGuideOpen, setIsDataGuideOpen] = useState(false); const [searchInput, setSearchInput] = useState( - location.search.includes('?q=') - ? URLToSearchQuery(location.search) - : '', + new URLSearchParams(location.search).get('q') || '', ); const [modalAlert, setModalAlert] = useState(false); const guideButtonRef = React.useRef(null); @@ -103,28 +100,30 @@ export default function SearchBar({ } }, [filtersBreadcrumb]); + useEffect(() => { + const q = new URLSearchParams(location.search).get('q') || ''; + if (q !== searchInput) setSearchInput(q); + }, [location.search]); + // Set search query debounce to 1000ms const debouncedSearch = useDebounce(searchInput, 2000); - // Update search input based on search query - useEffect(() => { - if (!location.search.includes('?q=')) { - setSearchInput(''); - return; - } + const handleNavigating = (q: string) => { + const searchParams = new URLSearchParams(location.search); + q !== '' ? searchParams.set('q', q) : searchParams.delete('q'); - setSearchInput(URLToSearchQuery(location.search)); - }, [location.search]); + navigate({ + pathname: '/cases', + search: searchParams.toString(), + }); + }; // Apply filter parameters after delay useEffect(() => { if (!isUserTyping) return; - setIsUserTyping(false); - navigate({ - pathname: '/cases', - search: searchQueryToURL(debouncedSearch), - }); + + handleNavigating(debouncedSearch); //eslint-disable-next-line }, [debouncedSearch]); @@ -136,10 +135,8 @@ export default function SearchBar({ if (ev.key === 'Enter') { ev.preventDefault(); setIsUserTyping(false); - navigate({ - pathname: '/cases', - search: searchQueryToURL(searchInput), - }); + + handleNavigating(searchInput); } }; @@ -174,16 +171,24 @@ export default function SearchBar({ return searchStringStrippedOutColon; } + const renderSearchErrorMessage = () => { + if (searchError) { + return 'Incorrect entry. ":" characters have been removed. Please use filters instead.'; + } else { + const quoteCount = searchInput.split('"').length - 1; + if (quoteCount % 2 !== 0) { + return 'Incorrect entry. Please make sure you have an even number of quotes.'; + } + } + }; + return ( <>
({ errorMessage: { @@ -98,14 +101,14 @@ export default function ViewCase(props: Props): JSX.Element { {loading && } {errorMessage && ( - {errorMessage} - + )} {c && ( { + subs != '' && i % 2 ? quoted.push(subs) : notQuoted.push(subs); + }); + } else notQuoted.push(q); + + const regex = /"([^"]+)"|(\S{1,})/g; + // Make sure that terms in quotes will be highlighted as one search term + for (const quotedEntry of quoted) { + let match; + let accumulator: string[] = []; + while ((match = regex.exec(quotedEntry))) { + accumulator.push(match[match[1] ? 1 : 2]); + } + searchQueryArray.push(accumulator.join(' ')); + } + for (const notQuotedEntry of notQuoted) { + let match; + while ((match = regex.exec(notQuotedEntry))) { + searchQueryArray.push(match[match[1] ? 1 : 2]); + } } - return searchQueryArray; } words(searchQuery); From 8db28d3d61d82e516d37453fc849cf4e0d03c81e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stanis=C5=82aw=20Zakrzewski?= <41862803+stanislaw-zakrzewski@users.noreply.github.com> Date: Wed, 20 Nov 2024 08:11:42 +0100 Subject: [PATCH 3/3] [104] change date format (#155) * Format date properly for csv, tsv and json download * Update tests * Use dedicated function for date formatting * Remove only from tests --- .../data-service/src/controllers/case.ts | 8 +++ data-serving/data-service/src/util/case.ts | 65 +++++++++++-------- .../data-service/test/util/case.test.ts | 27 ++++++-- 3 files changed, 67 insertions(+), 33 deletions(-) diff --git a/data-serving/data-service/src/controllers/case.ts b/data-serving/data-service/src/controllers/case.ts index 02ce0eaef..11e59454b 100644 --- a/data-serving/data-service/src/controllers/case.ts +++ b/data-serving/data-service/src/controllers/case.ts @@ -352,6 +352,14 @@ export class CasesController { header: false, columns: this.csvHeaders, delimiter: delimiter, + cast: { + date: (value: Date) => { + if (value) { + return new Date(value).toISOString().split('T')[0]; + } + return value; + }, + } }); res.write(stringifiedCase); doc = await cursor.next(); diff --git a/data-serving/data-service/src/util/case.ts b/data-serving/data-service/src/util/case.ts index db8faba83..3989908f0 100644 --- a/data-serving/data-service/src/util/case.ts +++ b/data-serving/data-service/src/util/case.ts @@ -103,6 +103,11 @@ export const removeBlankHeader = (headers: string[]): string[] => { return headers; }; +export const formatDateWithoutTime = (date: Date | undefined): string => { + if (!date) return ''; + return date.toISOString().split('T')[0]; +}; + export const denormalizeFields = async ( doc: CaseDocument, ): Promise> => { @@ -178,9 +183,8 @@ function denormalizeCaseReferenceFields( additionalSources.push(source.sourceUrl); } } - denormalizedData[ - 'caseReference.additionalSources' - ] = additionalSources.join(','); + denormalizedData['caseReference.additionalSources'] = + additionalSources.join(','); denormalizedData['caseReference.sourceEntryId'] = doc.sourceEntryId || ''; denormalizedData['caseReference.sourceId'] = doc.sourceId || ''; denormalizedData['caseReference.sourceUrl'] = doc.sourceUrl || ''; @@ -211,38 +215,46 @@ export const denormalizeEventsFields = ( const denormalizedData: Record = {}; denormalizedData['events.dateEntry'] = doc.dateEntry - ? doc.dateEntry.toDateString() + ? formatDateWithoutTime(doc.dateEntry) : undefined || ''; denormalizedData['events.dateReported'] = doc.dateReported - ? doc.dateReported.toDateString() + ? formatDateWithoutTime(doc.dateReported) : undefined || ''; - denormalizedData['events.dateOnset'] = doc.dateOnset?.toDateString() || ''; - denormalizedData['events.dateConfirmation'] = - doc.dateConfirmation?.toDateString() || ''; + denormalizedData['events.dateOnset'] = formatDateWithoutTime(doc.dateOnset); + denormalizedData['events.dateConfirmation'] = formatDateWithoutTime( + doc.dateConfirmation, + ); denormalizedData['events.confirmationMethod'] = doc.confirmationMethod || ''; - denormalizedData['events.dateOfFirstConsult'] = - doc.dateOfFirstConsult?.toDateString() || ''; + denormalizedData['events.dateOfFirstConsult'] = formatDateWithoutTime( + doc.dateOfFirstConsult, + ); denormalizedData['events.hospitalized'] = doc.hospitalized || ''; denormalizedData['events.reasonForHospitalization'] = doc.reasonForHospitalization || ''; - denormalizedData['events.dateHospitalization'] = - doc.dateHospitalization?.toDateString() || ''; - denormalizedData['events.dateDischargeHospital'] = - doc.dateDischargeHospital?.toDateString() || ''; + denormalizedData['events.dateHospitalization'] = formatDateWithoutTime( + doc.dateHospitalization, + ); + denormalizedData['events.dateDischargeHospital'] = formatDateWithoutTime( + doc.dateDischargeHospital, + ); denormalizedData['events.intensiveCare'] = doc.intensiveCare || ''; - denormalizedData['events.dateAdmissionICU'] = - doc.dateAdmissionICU?.toDateString() || ''; - denormalizedData['events.dateDischargeICU'] = - doc.dateDischargeICU?.toDateString() || ''; + denormalizedData['events.dateAdmissionICU'] = formatDateWithoutTime( + doc.dateAdmissionICU, + ); + denormalizedData['events.dateDischargeICU'] = formatDateWithoutTime( + doc.dateDischargeICU, + ); denormalizedData['events.homeMonitoring'] = doc.homeMonitoring || ''; denormalizedData['events.isolated'] = doc.isolated || ''; - denormalizedData['events.dateIsolation'] = - doc.dateIsolation?.toDateString() || ''; + denormalizedData['events.dateIsolation'] = formatDateWithoutTime( + doc.dateIsolation, + ); denormalizedData['events.outcome'] = doc.outcome || ''; - denormalizedData['events.dateDeath'] = doc.dateDeath?.toDateString() || ''; - denormalizedData['events.dateRecovered'] = - doc.dateRecovered?.toDateString() || ''; + denormalizedData['events.dateDeath'] = formatDateWithoutTime(doc.dateDeath); + denormalizedData['events.dateRecovered'] = formatDateWithoutTime( + doc.dateRecovered, + ); return denormalizedData; }; @@ -334,7 +346,7 @@ function denormalizeTravelHistoryFields( denormalizedData['travelHistory.travelHistory'] = doc?.travelHistory || ''; denormalizedData['travelHistory.travelHistoryEntry'] = - doc?.travelHistoryEntry?.toDateString() || ''; + formatDateWithoutTime(doc?.travelHistoryEntry); denormalizedData['travelHistory.travelHistoryStart'] = doc?.travelHistoryStart || ''; denormalizedData['travelHistory.travelHistoryLocation'] = @@ -352,8 +364,9 @@ function denormalizeVaccineFields( denormalizedData['vaccination.vaccination'] = doc?.vaccination || ''; denormalizedData['vaccination.vaccineName'] = doc?.vaccineName || ''; - denormalizedData['vaccination.vaccineDate'] = - doc?.vaccineDate?.toDateString() || ''; + denormalizedData['vaccination.vaccineDate'] = formatDateWithoutTime( + doc?.vaccineDate, + ); denormalizedData['vaccination.vaccineSideEffects'] = doc?.vaccineSideEffects || ''; diff --git a/data-serving/data-service/test/util/case.test.ts b/data-serving/data-service/test/util/case.test.ts index 205bcb7d7..b43715dc5 100644 --- a/data-serving/data-service/test/util/case.test.ts +++ b/data-serving/data-service/test/util/case.test.ts @@ -13,7 +13,11 @@ import { RevisionMetadataDocument } from '../model/revision-metadata'; import { TransmissionDocument } from '../model/transmission'; import { TravelHistoryDocument } from '../model/travel-history'; import { VaccineDocument } from '../model/vaccine'; -import { removeBlankHeader, denormalizeFields } from '../../src/util/case'; +import { + removeBlankHeader, + denormalizeFields, + formatDateWithoutTime, +} from '../../src/util/case'; import mongoose from 'mongoose'; import { MongoMemoryServer } from 'mongodb-memory-server'; import { GenomeSequenceDocument } from '../../src/model/genome-sequence'; @@ -421,16 +425,16 @@ describe('Case', () => { const denormalizedCase = await denormalizeFields(caseDoc); expect(denormalizedCase['events.dateEntry']).toEqual( - eventsDoc.dateEntry.toDateString(), + formatDateWithoutTime(eventsDoc.dateEntry), ); expect(denormalizedCase['events.dateReported']).toEqual( - eventsDoc.dateReported.toDateString(), + formatDateWithoutTime(eventsDoc.dateReported), ); expect(denormalizedCase['events.dateOnset']).toEqual( - eventsDoc.dateOnset?.toDateString(), + formatDateWithoutTime(eventsDoc.dateOnset), ); expect(denormalizedCase['events.dateConfirmation']).toEqual( - eventsDoc.dateConfirmation?.toDateString(), + formatDateWithoutTime(eventsDoc.dateConfirmation), ); expect(denormalizedCase['events.confirmationMethod']).toEqual(''); expect(denormalizedCase['events.dateOfFirstConsult']).toEqual(''); @@ -612,7 +616,7 @@ describe('Case', () => { expect(denormalizedCase['travelHistory.travelHistory']).toEqual('Y'); expect(denormalizedCase['travelHistory.travelHistoryEntry']).toEqual( - travelHistoryDoc.travelHistoryEntry.toDateString(), + formatDateWithoutTime(travelHistoryDoc.travelHistoryEntry), ); expect(denormalizedCase['travelHistory.travelHistoryStart']).toEqual( 'start', @@ -652,7 +656,7 @@ describe('Case', () => { expect(denormalizedCase['vaccination.vaccination']).toEqual('Y'); expect(denormalizedCase['vaccination.vaccineName']).toEqual('Pfizer'); expect(denormalizedCase['vaccination.vaccineDate']).toEqual( - vaccinationDoc.vaccineDate.toDateString(), + formatDateWithoutTime(vaccinationDoc.vaccineDate), ); expect(denormalizedCase['vaccination.vaccineSideEffects']).toEqual( 'cough', @@ -689,4 +693,13 @@ describe('Case', () => { '1234', ); }); + + it('formatsDateWithoutTimeCorrectly', async () => { + const correctDateString = '2024-01-01'; + const correctDate = new Date(`${correctDateString}T03:24:00`); + const missingDate = undefined; + + expect(formatDateWithoutTime(correctDate)).toEqual(correctDateString); + expect(formatDateWithoutTime(missingDate)).toEqual(''); + }); });