From 929ea2a477345444e6162b7274cc5d5dc8305248 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Stanis=C5=82aw=20Zakrzewski?=
<41862803+stanislaw-zakrzewski@users.noreply.github.com>
Date: Thu, 17 Oct 2024 14:05:11 +0200
Subject: [PATCH] [131] update mongodb validation for the case schema to allow
search (#136)
* Update libraries batch-2 first part
* Move some testing to TODO
* Update Yup when
* update set field values
* Update breaking changes
* Update comment
* Remove unused
* update datepicker
* Update libraries batch-2 second part
* Update return values of the functions for countries
* update typescript
* Update mismatched versions
* WIP update router and eslint + vite instead of cra
* Update tests for new vitest library
* Update command for test and coverage
* Skip tests that were blocking flow
* Update Location.test.tsx
* Update varaible name
* Fix merging issues
* Update index.tsx
* Update index.tsx
* WIP tests for location and landing page
* Fixed all landing page tests
* Unskip and fix verification status indicator tests
* Fix automated source form tests
* WIP unmocking EditCase tests
* Update EditCase.test.tsx
* Fix viewcase tests
* Resolve warnings regarding act
* Update Dockerfile
* Update Location.test.tsx
* Fix tests failing at the first day of the month
* Cleanup sourcetable
* Update Users Uploads and Sources tables
* Add backfill back
It was removed after splitting one component into multiple
* Rename collection from cases to day0cases
* Update day0cases.schema.json
* Update day0cases.schema.json
* Initial schema work
* initial schema work
* Fix location lat/long update issue
* Add missing schema element - isGovernmentSource for Additional Sources
* Update tests for searchbar and data guide
* Update README.md
---
.../data-service/src/controllers/case.ts | 7 +
.../migrations/20210902121948-initial.js | 99 +++++------
.../20210922082905-additional-case-indexes.js | 44 ++---
data-serving/scripts/setup-db/package.json | 1 +
...es.indexes.json => day0cases.indexes.json} | 28 +++-
...ases.schema.json => day0cases.schema.json} | 154 +++++++++++++++++-
dev/README.md | 5 +
.../e2e/components/LinelistTableTest.spec.ts | 59 +++++++
.../ui/src/components/DataGuideDialog.tsx | 12 +-
.../ui/src/components/SearchBar.tsx | 3 +-
.../new-case-form-fields/Location.tsx | 27 +--
11 files changed, 348 insertions(+), 91 deletions(-)
rename data-serving/scripts/setup-db/schemas/{cases.indexes.json => day0cases.indexes.json} (83%)
rename data-serving/scripts/setup-db/schemas/{cases.schema.json => day0cases.schema.json} (60%)
diff --git a/data-serving/data-service/src/controllers/case.ts b/data-serving/data-service/src/controllers/case.ts
index f90663ee2..02ce0eaef 100644
--- a/data-serving/data-service/src/controllers/case.ts
+++ b/data-serving/data-service/src/controllers/case.ts
@@ -795,6 +795,13 @@ export class CasesController {
res.status(201).json(result);
} catch (e) {
const err = e as Error;
+ if (err.name === 'MongoServerError') {
+ logger.error((e as any).errInfo);
+ res.status(422).json({
+ message: (err as any).errInfo,
+ });
+ return;
+ }
if (err instanceof GeocodeNotFoundError) {
res.status(404).json({
message: err.message,
diff --git a/data-serving/scripts/setup-db/migrations/20210902121948-initial.js b/data-serving/scripts/setup-db/migrations/20210902121948-initial.js
index 27063fa83..3b3bc08e5 100644
--- a/data-serving/scripts/setup-db/migrations/20210902121948-initial.js
+++ b/data-serving/scripts/setup-db/migrations/20210902121948-initial.js
@@ -1,57 +1,60 @@
const fs = require('fs');
module.exports = {
- async up(db, client) {
- await createCollectionValidationAndIndexes(
- db,
- 'cases',
- 'schemas/cases.schema.json',
- 'schemas/cases.indexes.json'
- );
-
- await createCollectionValidationAndIndexes(
- db,
- 'sources',
- 'schemas/sources.schema.json',
- 'schemas/sources.indexes.json'
- );
- },
-
- async down(db, client) {
- // initial migration has no rollback
- }
+ async up(db, client) {
+ await createCollectionValidationAndIndexes(
+ db,
+ 'day0cases',
+ 'schemas/day0cases.schema.json',
+ 'schemas/day0cases.indexes.json',
+ );
+
+ await createCollectionValidationAndIndexes(
+ db,
+ 'sources',
+ 'schemas/sources.schema.json',
+ 'schemas/sources.indexes.json',
+ );
+ },
+
+ async down(db, client) {
+ // initial migration has no rollback
+ },
};
-async function createCollectionValidationAndIndexes(db, collectionName, schemaPath, indexPath) {
- const schemaFile = await fs.promises.readFile(schemaPath);
- const schema = JSON.parse(schemaFile);
+async function createCollectionValidationAndIndexes(
+ db,
+ collectionName,
+ schemaPath,
+ indexPath,
+) {
+ const schemaFile = await fs.promises.readFile(schemaPath);
+ const schema = JSON.parse(schemaFile);
+
+ const indexFile = await fs.promises.readFile(indexPath);
+ const indexes = JSON.parse(indexFile);
+ /*
+ * because this migration might run against a DB from before we had the migrations infra,
+ * check whether the collection already exists. If it does, then modify its validation schema.
+ * If it doesn't, then create it.
+ */
+ try {
+ await db.collection(collectionName);
+ await db.command({
+ collMod: collectionName,
+ validator: schema,
+ });
+ } catch {
+ await db.createCollection(collectionName, {
+ validator: schema,
+ });
+ }
+
+ const collection = db.collection(collectionName);
+ await collection.dropIndexes();
- const indexFile = await fs.promises.readFile(indexPath);
- const indexes = JSON.parse(indexFile);
- /*
- * because this migration might run against a DB from before we had the migrations infra,
- * check whether the collection already exists. If it does, then modify its validation schema.
- * If it doesn't, then create it.
- */
- try {
- await db.collection(collectionName);
await db.command({
- collMod: collectionName,
- validator: schema,
- });
- }
- catch {
- await db.createCollection(collectionName, {
- validator: schema,
+ createIndexes: collectionName,
+ indexes: indexes,
});
- }
-
- const collection = db.collection(collectionName);
- await collection.dropIndexes();
-
- await db.command({
- createIndexes: collectionName,
- indexes: indexes,
- });
}
-
diff --git a/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js b/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js
index 4fe9bbbee..b47fbf266 100644
--- a/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js
+++ b/data-serving/scripts/setup-db/migrations/20210922082905-additional-case-indexes.js
@@ -1,29 +1,29 @@
const indexes = [
- {
- name: 'byGenderAndCountry',
- key: {
- 'demographics.gender': -1,
- 'location.countryISO3': -1
+ {
+ name: 'byGenderAndCountry',
+ key: {
+ 'demographics.gender': -1,
+ 'location.countryISO3': -1,
+ },
+ collation: {
+ locale: 'en_US',
+ strength: 2,
+ },
},
- collation: {
- locale: 'en_US',
- strength: 2,
- },
- }
];
module.exports = {
- async up(db, client) {
- await db.command({
- createIndexes: 'cases',
- indexes: indexes,
- });
- },
+ async up(db, client) {
+ await db.command({
+ createIndexes: 'day0cases',
+ indexes: indexes,
+ });
+ },
- async down(db, client) {
- await db.command({
- dropIndexes: 'cases',
- index: ['byGenderAndCountry']
- });
- }
+ async down(db, client) {
+ await db.command({
+ dropIndexes: 'day0cases',
+ index: ['byGenderAndCountry'],
+ });
+ },
};
diff --git a/data-serving/scripts/setup-db/package.json b/data-serving/scripts/setup-db/package.json
index 1eec4d3a5..fcdd3b211 100644
--- a/data-serving/scripts/setup-db/package.json
+++ b/data-serving/scripts/setup-db/package.json
@@ -8,6 +8,7 @@
"lint": "tsc --noEmit && eslint '*/**/*.{js,ts,tsx}' --quiet --fix",
"import-sample-data": "python3 ./import-sample-data.py",
"migrate": "npm ci && migrate-mongo up",
+ "migrate-down": "npm ci && migrate-mongo down",
"delete-all-cases": "mongosh $CONN --eval 'db.cases.deleteMany({})'"
},
"repository": {
diff --git a/data-serving/scripts/setup-db/schemas/cases.indexes.json b/data-serving/scripts/setup-db/schemas/day0cases.indexes.json
similarity index 83%
rename from data-serving/scripts/setup-db/schemas/cases.indexes.json
rename to data-serving/scripts/setup-db/schemas/day0cases.indexes.json
index 210417dd7..a7007c6b6 100644
--- a/data-serving/scripts/setup-db/schemas/cases.indexes.json
+++ b/data-serving/scripts/setup-db/schemas/day0cases.indexes.json
@@ -4,7 +4,9 @@
"key": {
"demographics.occupation": "text",
"location.country": "text",
- "location.city": "text",
+ "location.admin1": "text",
+ "location.admin2": "text",
+ "location.admin3": "text",
"caseReference.sourceUrl": "text",
"caseStatus": "text"
}
@@ -74,9 +76,29 @@
}
},
{
- "name": "locationCityIdx",
+ "name": "locationAdmin1Idx",
"key": {
- "location.city": -1
+ "location.admin1": -1
+ },
+ "collation": {
+ "locale": "en_US",
+ "strength": 2
+ }
+ },
+ {
+ "name": "locationAdmin2Idx",
+ "key": {
+ "location.admin2": -1
+ },
+ "collation": {
+ "locale": "en_US",
+ "strength": 2
+ }
+ },
+ {
+ "name": "locationAdmin3Idx",
+ "key": {
+ "location.admin3": -1
},
"collation": {
"locale": "en_US",
diff --git a/data-serving/scripts/setup-db/schemas/cases.schema.json b/data-serving/scripts/setup-db/schemas/day0cases.schema.json
similarity index 60%
rename from data-serving/scripts/setup-db/schemas/cases.schema.json
rename to data-serving/scripts/setup-db/schemas/day0cases.schema.json
index addf0e2c8..4175de3de 100644
--- a/data-serving/scripts/setup-db/schemas/cases.schema.json
+++ b/data-serving/scripts/setup-db/schemas/day0cases.schema.json
@@ -2,6 +2,7 @@
"$jsonSchema": {
"bsonType": "object",
"additionalProperties": false,
+ "required": ["caseStatus"],
"properties": {
"_id": {
"bsonType": "number",
@@ -13,6 +14,102 @@
"list": {
"bsonType": "bool"
},
+ "caseStatus": {
+ "bsonType": "string"
+ },
+ "comment": {
+ "bsonType": "string"
+ },
+ "curators": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "verifiedBy": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "_id": {
+ "bsonType": "objectId"
+ },
+ "name": {
+ "bsonType": "string"
+ },
+ "email": {
+ "bsonType": "string"
+ },
+ "roles": {
+ "bsonType": "array",
+ "uniqueItems": true,
+ "items": {
+ "bsonType": "string"
+ }
+ }
+ }
+ },
+ "createdBy": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "_id": {
+ "bsonType": "objectId"
+ },
+ "name": {
+ "bsonType": "string"
+ },
+ "email": {
+ "bsonType": "string"
+ },
+ "roles": {
+ "bsonType": "array",
+ "uniqueItems": true,
+ "items": {
+ "bsonType": "string"
+ }
+ }
+ }
+ }
+ }
+ },
+ "revisionMetadata": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "revisionNumber": {
+ "bsonType": "number",
+ "minimum": 0
+ },
+ "creationMetadata": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "curator": {
+ "bsonType": "string"
+ },
+ "date": {
+ "bsonType": "date"
+ },
+ "notes": {
+ "bsonType": "string"
+ }
+ }
+ },
+ "updateMetadata": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "curator": {
+ "bsonType": "string"
+ },
+ "date": {
+ "bsonType": "date"
+ },
+ "notes": {
+ "bsonType": "string"
+ }
+ }
+ }
+ }
+ },
"caseReference": {
"bsonType": "object",
"additionalProperties": false,
@@ -26,6 +123,9 @@
"sourceUrl": {
"bsonType": "string"
},
+ "isGovernmentSource": {
+ "bsonType": "bool"
+ },
"uploadIds": {
"bsonType": "array",
"uniqueItems": true,
@@ -42,6 +142,9 @@
"properties": {
"sourceUrl": {
"bsonType": "string"
+ },
+ "isGovernmentSource": {
+ "bsonType": "bool"
}
}
}
@@ -68,6 +171,13 @@
}
}
},
+ "ageBuckets": {
+ "bsonType": "array",
+ "uniqueItems": true,
+ "items": {
+ "bsonType": "objectId"
+ }
+ },
"gender": {
"bsonType": "string"
},
@@ -111,16 +221,56 @@
"bsonType": "object",
"additionalProperties": false,
"properties": {
+ "admin1": {
+ "bsonType": "string"
+ },
+ "admin1WikiId": {
+ "bsonType": "string"
+ },
+ "admin2": {
+ "bsonType": "string"
+ },
+ "admin2WikiId": {
+ "bsonType": "string"
+ },
+ "admin3": {
+ "bsonType": "string"
+ },
+ "admin3WikiId": {
+ "bsonType": "string"
+ },
+ "comment": {
+ "bsonType": "string"
+ },
"country": {
"bsonType": "string"
},
"countryISO3": {
"bsonType": "string"
},
+ "geometry": {
+ "bsonType": "object",
+ "additionalProperties": false,
+ "properties": {
+ "latitude": {
+ "bsonType": "number",
+ "minimum": -90,
+ "maximum": 90
+ },
+ "longitude": {
+ "bsonType": "number",
+ "minimum": -180,
+ "maximum": 180
+ }
+ }
+ },
+ "geoResolution": {
+ "bsonType": "string"
+ },
"location": {
"bsonType": "string"
},
- "place": {
+ "name": {
"bsonType": "string"
},
"query": {
@@ -135,7 +285,7 @@
"bsonType": ["date"]
},
"dateLastModified": {
- "bsonType": ["date"]
+ "bsonType": ["date", "null"]
},
"dateOnset": {
"bsonType": ["date"]
diff --git a/dev/README.md b/dev/README.md
index b05064b04..619e09eb6 100644
--- a/dev/README.md
+++ b/dev/README.md
@@ -52,6 +52,11 @@ For local development, it's fine to use your own values for these
secrets if you have them. For instance, if you have a developer Mapbox API
token, or if you'd like to use a different GMail account for mailing notifications, or different OAuth client values.
+#### Database Schema
+
+Before inputting any case data to the local portal instance a database schema migration needs to be run.
+Navigate to the `data-serving/scripts/setup-db/` and follow instructions in the [README](../data-serving/scripts/setup-db/README.md) to set up the database schema.
+
#### Permissions
Give your user all the permissions to access the portal and make CRUD updates.
diff --git a/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts b/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts
index e11307ab7..7cc942796 100644
--- a/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts
+++ b/verification/curator-service/ui/cypress/e2e/components/LinelistTableTest.spec.ts
@@ -491,4 +491,63 @@ describe('Linelist table', function () {
cy.get('tr').eq(1).contains('France');
});
+
+ it('Can search the data', () => {
+ cy.addCase({
+ country: 'France',
+ countryISO3: 'FRA',
+ dateEntry: '2020-05-01',
+ dateReported: '2020-05-01',
+ sourceUrl: 'www.example.com',
+ caseStatus: CaseStatus.Confirmed,
+ occupation: 'Actor',
+ });
+ cy.addCase({
+ country: 'Germany',
+ countryISO3: 'DEU',
+ dateEntry: '2020-05-01',
+ dateReported: '2020-05-01',
+ sourceUrl: 'www.example.com',
+ caseStatus: CaseStatus.Confirmed,
+ occupation: 'Doctor',
+ });
+ cy.addCase({
+ country: 'Argentina',
+ countryISO3: 'ARG',
+ dateEntry: '2020-05-01',
+ dateReported: '2020-05-01',
+ sourceUrl: 'www.example.com',
+ caseStatus: CaseStatus.Confirmed,
+ occupation: 'Engineer',
+ });
+
+ cy.intercept('GET', getDefaultQuery({ limit: 50 })).as('getCases');
+ cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'Argentina' })).as('getCasesWithSearch1');
+ cy.intercept('GET', getDefaultQuery({ limit: 50, query: 'Doctor' })).as('getCasesWithSearch2');
+
+ cy.visit('/cases');
+ cy.wait('@getCases');
+
+ cy.contains('Argentina').should('exist');
+ cy.contains('France').should('exist');
+ cy.contains('Germany').should('exist');
+
+ cy.get('#search-field').type('Argentina');
+ cy.wait('@getCasesWithSearch1');
+ cy.contains('Argentina').should('exist');
+ cy.contains('France').should('not.exist');
+ cy.contains('Germany').should('not.exist');
+
+ cy.get('#clear-search').click();
+ cy.wait('@getCases');
+ cy.contains('Argentina').should('exist');
+ cy.contains('France').should('exist');
+ cy.contains('Germany').should('exist');
+
+ cy.get('#search-field').type('Doctor');
+ cy.wait('@getCasesWithSearch2');
+ cy.contains('Argentina').should('not.exist');
+ cy.contains('France').should('not.exist');
+ cy.contains('Germany').should('exist');
+ });
});
diff --git a/verification/curator-service/ui/src/components/DataGuideDialog.tsx b/verification/curator-service/ui/src/components/DataGuideDialog.tsx
index 95c406d5b..d24c7a03c 100644
--- a/verification/curator-service/ui/src/components/DataGuideDialog.tsx
+++ b/verification/curator-service/ui/src/components/DataGuideDialog.tsx
@@ -96,8 +96,16 @@ const SearchGuideDialog = ({
the left and choose ascending or descending.
- For full-text search, enter any
- combination of search terms.
+ For full-text search, enter any combination of search terms.
+
+ Full-text search covers: occupation, admin0, admin1, admin2, admin3, sourceUrl and caseStatus.
+
+ Search terms must be exact (example: "German" will not match "Germany").
+
+ Full-text search matches cases that contain any ot the search terms, not a combination.
+
+ No special characters apart from "." are allowed and if the "." is used in a search term
+ given search term must be contained within quotation marks.
You can use the icons on the right to navigate
diff --git a/verification/curator-service/ui/src/components/SearchBar.tsx b/verification/curator-service/ui/src/components/SearchBar.tsx
index 9b9d79d53..a4ebf82e2 100644
--- a/verification/curator-service/ui/src/components/SearchBar.tsx
+++ b/verification/curator-service/ui/src/components/SearchBar.tsx
@@ -203,7 +203,7 @@ export default function SearchBar({
}
}}
placeholder="Fulltext search"
- value={searchInput}
+ value={decodeURI(searchInput)}
variant="outlined"
fullWidth
InputProps={{
@@ -254,6 +254,7 @@ export default function SearchBar({
{
setSearchInput('');
navigate({
diff --git a/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx b/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx
index 607260287..906b812af 100644
--- a/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx
+++ b/verification/curator-service/ui/src/components/new-case-form-fields/Location.tsx
@@ -120,7 +120,6 @@ export default function Location(): JSX.Element {
setSelectedAdmin1(foundAdmin1Entry);
}
}
- // eslint-disable-next-line react-hooks/exhaustive-deps
}, [values.location.countryISO3, admin1Entries]);
useEffect(() => {
@@ -177,8 +176,6 @@ export default function Location(): JSX.Element {
// Update mapbox indicator for admin1
setAdmin1AvailableOnMap(!!values.location.admin1WikiId);
-
- // eslint-disable-next-line react-hooks/exhaustive-deps
}, [values.location.admin1WikiId, values.location.admin1, admin2Entries]);
useEffect(() => {
@@ -235,8 +232,6 @@ export default function Location(): JSX.Element {
// Update mapbox indicator for admin2
setAdmin2AvailableOnMap(!!values.location.admin2WikiId);
-
- // eslint-disable-next-line react-hooks/exhaustive-deps
}, [values.location.admin2WikiId, values.location.admin2, admin3Entries]);
useEffect(() => {
@@ -258,8 +253,6 @@ export default function Location(): JSX.Element {
// Update mapbox indicator for admin3
setAdmin3AvailableOnMap(!!values.location.admin3WikiId);
-
- // eslint-disable-next-line react-hooks/exhaustive-deps
}, [values.location.admin3WikiId, values.location.admin3]);
useEffect(() => {
@@ -326,15 +319,14 @@ export default function Location(): JSX.Element {
'location.geometry.latitude',
values.location.geocodeLocation.geometry?.latitude ||
values.location.geometry?.latitude ||
- '',
+ undefined,
);
setFieldValue(
'location.geometry.longitude',
values.location.geocodeLocation.geometry?.longitude ||
values.location.geometry?.longitude ||
- '',
+ undefined,
);
- // eslint-disable-next-line react-hooks/exhaustive-deps
}, [values.location.geocodeLocation]);
return (
@@ -486,7 +478,10 @@ export default function Location(): JSX.Element {
_: unknown,
newValue: adminEntry | null,
): void => {
- setFieldValue('location.admin1', newValue?.name || '');
+ setFieldValue(
+ 'location.admin1',
+ newValue?.name || '',
+ );
setFieldValue(
'location.admin1WikiId',
newValue?.wiki || '',
@@ -588,7 +583,10 @@ export default function Location(): JSX.Element {
_: unknown,
newValue: adminEntry | null,
): void => {
- setFieldValue('location.admin2', newValue?.name || '');
+ setFieldValue(
+ 'location.admin2',
+ newValue?.name || '',
+ );
setFieldValue(
'location.admin2WikiId',
newValue?.wiki || '',
@@ -690,7 +688,10 @@ export default function Location(): JSX.Element {
_: unknown,
newValue: adminEntry | null,
): void => {
- setFieldValue('location.admin3', newValue?.name || '');
+ setFieldValue(
+ 'location.admin3',
+ newValue?.name || '',
+ );
setFieldValue(
'location.admin3WikiId',
newValue?.wiki || '',