Skip to content

Commit bae350c

Browse files
marcolarosamarcolarosa
committed
Revise data uploading capability
* implement fastify tus -> s3 direct plugin * rework item upload tab * implement capability to upload item permission forms commit 23ff14dd507e83d9859a2fa3bf50aaece0b43daa Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 13:58:41 2023 +1100 fix tests commit ac110df Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 12:11:01 2023 +1100 fix label commit 53f26b0 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 12:08:26 2023 +1100 wire up permission form upload components into publish form commit 4036768 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 12:08:04 2023 +1100 wire up permission files display on administration tab commit ecbc297 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 11:21:05 2023 +1100 ignore the tus cache commit 3b1227b Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 11:18:45 2023 +1100 revise uploader ; make it image specific commit 46225bc Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 11:18:08 2023 +1100 refactor and wire up task triggering on upload commit 64207b5 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 11:17:41 2023 +1100 add not about disabling OCR commit 664c785 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 11:17:09 2023 +1100 add task to process image without ocr; genericise the mapping commit f63a6df Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 11:16:49 2023 +1100 enable sending the task from the UI commit 5567e98 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 10:14:53 2023 +1100 rework processing and embed error reporter commit 2b8d6f7 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 09:59:06 2023 +1100 refactor into individual components commit 5a63c3f Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 09:58:41 2023 +1100 implement warning box commit 40154fa Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 09:58:16 2023 +1100 revise - look for specific task ids commit 0cdeb0b Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 09:57:58 2023 +1100 revise match code; return taskId commit 6a897d3 Author: Marco La Rosa <m@lr.id.au> Date: Fri Mar 10 09:57:30 2023 +1100 reimplement method; look for specific taskIds commit d372659 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:50:26 2023 +1100 implement upload transcription help commit 752bd61 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:46:27 2023 +1100 implement component to upload single file commit 94e5dc3 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:46:07 2023 +1100 implement new and improved item publish component commit edfb23c Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:45:54 2023 +1100 implement info box commit 7729f7c Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:45:21 2023 +1100 refactor; wire up component to upload transcription commit 9eabad5 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:43:47 2023 +1100 revise help commit 855b04c Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:43:31 2023 +1100 remove legacy components commit 6a320a8 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:43:07 2023 +1100 wire up item specific publish component commit bd64eba Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:42:45 2023 +1100 cleanup legacy stuff commit d19cf6d Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:41:57 2023 +1100 revise config to support tus capability commit 1a6ad37 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:41:29 2023 +1100 disable tus container commit 860e317 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:41:12 2023 +1100 rename method; return bucket and item path commit 7f20396 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:40:47 2023 +1100 wire up route to list item permission forms commit d738519 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:40:32 2023 +1100 implement method to list permission forms commit 154545a Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:40:11 2023 +1100 add permission files to specialFiles array commit cceaab9 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:39:56 2023 +1100 update fastify tus s3 plugin commit e3b7430 Author: Marco La Rosa <m@lr.id.au> Date: Thu Mar 9 15:39:35 2023 +1100 setup cors and define bodyLimit prop on fastify commit daa155d Author: Marco La Rosa <m@lr.id.au> Date: Mon Mar 6 15:45:07 2023 +1100 wire up the fastify tus s3 uploader commit 5565623 Author: Marco La Rosa <m@lr.id.au> Date: Mon Mar 6 15:44:49 2023 +1100 add fastify tus s3 uploader
1 parent 0e5e0c5 commit bae350c

40 files changed

+12142
-1567
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,4 @@ eng.traineddata
1414
*.*~
1515

1616
docker-metadata
17+
.cache

‎api/index.js

+46-4
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ import fastifyCompress from "@fastify/compress";
2121
import cors from "@fastify/cors";
2222
import fastifySensible from "@fastify/sensible";
2323
import fastifyIO from "fastify-socket.io";
24+
import fastifyTusS3Plugin from "@paradisec-platform/fastify-tus-s3-plugin";
25+
2426
const envToLogger = {
2527
development: {
2628
transport: {
@@ -33,7 +35,10 @@ const envToLogger = {
3335
production: true,
3436
test: false,
3537
};
36-
const fastify = Fastify({ logger: envToLogger[process.env.NODE_ENV] });
38+
const fastify = Fastify({
39+
logger: envToLogger[process.env.NODE_ENV],
40+
bodyLimit: 256 * 1024 * 1024,
41+
});
3742

3843
main();
3944
async function main() {
@@ -45,12 +50,49 @@ async function main() {
4550
process.exit();
4651
}
4752

48-
if (process.env.NODE_ENV === "development") {
49-
fastify.register(cors, { origin: "*" });
50-
}
53+
await fastify.register(cors, {
54+
origin: "*",
55+
methods: ["OPTIONS", "GET", "HEAD", "PATCH", "POST", "DELETE"],
56+
allowedHeaders: [
57+
"content-type",
58+
"upload-length",
59+
"content-length",
60+
"upload-offset",
61+
"upload-expires",
62+
"location",
63+
"upload-metadata",
64+
"tus-resumable",
65+
"tus-version",
66+
"tus-max-size",
67+
"tus-extension",
68+
],
69+
exposedHeaders: [
70+
"content-type",
71+
"upload-length",
72+
"content-length",
73+
"upload-offset",
74+
"upload-expires",
75+
"location",
76+
"upload-metadata",
77+
"tus-resumable",
78+
"tus-version",
79+
"tus-max-size",
80+
"tus-extension",
81+
],
82+
});
5183
fastify.register(fastifySensible);
5284
fastify.register(fastifyIO);
5385
fastify.register(fastifyCompress);
86+
fastify.register(fastifyTusS3Plugin, {
87+
awsAccessKeyId: configuration.api.services.s3.awsAccessKeyId,
88+
awsSecretAccessKey: configuration.api.services.s3.awsSecretAccessKey,
89+
region: configuration.api.services.s3.region,
90+
endpoint: configuration.api.services.s3.endpointUrl,
91+
forcePathStyle: configuration.api.services.s3.forcePathStyle,
92+
cachePath: "./.cache",
93+
uploadRoutePath: "/files",
94+
defaultUploadExpiration: { hours: 6 },
95+
});
5496
fastify.addHook("onRequest", async (req, res) => {
5597
configuration = await loadConfiguration();
5698
req.io = fastify.io;

‎api/package-lock.json

+1,220-1,157
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/package.json

+1
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
"@fastify/cors": "^8.2.0",
4141
"@fastify/one-line-logger": "^1.1.1",
4242
"@fastify/sensible": "^5.2.0",
43+
"@paradisec-platform/fastify-tus-s3-plugin": "^0.2.2",
4344
"cross-fetch": "^3.1.5",
4445
"date-fns": "^2.29.3",
4546
"fastify": "^4.13.0",

‎api/src/common/index.js

+2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ export const specialFiles = [
2626
"ro-crate-metadata.json",
2727
"-digivol.csv",
2828
"-tei.xml",
29+
"-rights-holder-permission.pdf",
30+
"-language-authority-permission.pdf",
2931
"nocfl.identifier.json",
3032
"nocfl.inventory.json",
3133
completedResources,

‎api/src/lib/admin.spec.js

+10-4
Original file line numberDiff line numberDiff line change
@@ -149,13 +149,13 @@ describe("Admin management tests", () => {
149149
id: identifier,
150150
type: "item",
151151
});
152-
await storeItem.createItem();
152+
await storeItem.createObject();
153153

154154
let storeCollection = await getStoreHandle({
155155
id: identifier,
156156
type: "collection",
157157
});
158-
await storeCollection.createItem();
158+
await storeCollection.createObject();
159159

160160
let user = users.filter((u) => !u.administrator)[0];
161161
let adminUser = users.filter((u) => u.administrator)[0];
@@ -177,14 +177,20 @@ describe("Admin management tests", () => {
177177

178178
// retrieve all items
179179
let { items } = await getAdminItems({ user: adminUser });
180-
expect(items[0]).toMatchObject({ identifier, connected: false });
180+
expect(items.filter((i) => i.identifier === identifier)[0]).toMatchObject({
181+
identifier,
182+
connected: false,
183+
});
181184

182185
// import collections
183186
await importCollectionsFromStorageIntoTheDb({ user: adminUser, configuration });
184187

185188
// retrieve all collections
186189
let { collections } = await getAdminCollections({ user: adminUser });
187-
expect(collections[0]).toMatchObject({ identifier, connected: false });
190+
expect(collections.filter((c) => c.identifier === identifier)[0]).toMatchObject({
191+
identifier,
192+
connected: false,
193+
});
188194

189195
await models.item.destroy({ where: { identifier } });
190196
await models.collection.destroy({ where: { identifier } });

‎api/src/lib/collection.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -61,9 +61,9 @@ export async function linkCollectionToUser({ collectionId, userId }) {
6161

6262
export async function createCollectionLocationInObjectStore({ identifier }) {
6363
let store = await getStoreHandle({ id: identifier, type: "collection" });
64-
let exists = await store.itemExists();
64+
let exists = await store.exists();
6565
if (!exists) {
66-
await store.createItem();
66+
await store.createObject();
6767
}
6868
}
6969

‎api/src/lib/crate-tools.spec.js

-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ describe("Crate tools tests", () => {
3636

3737
await registerAllFiles({ crate, resources });
3838
expect(crate.rootDataset.hasPart.length).toBe(4);
39-
console.log(crate.toJSON());
4039

4140
await models.item.destroy({ where: { identifier } });
4241
await store.removeObject();

‎api/src/lib/item.js

+21-13
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,19 @@ export async function listItemResources({ identifier, offset = 0, limit }) {
171171
return { resources, total };
172172
}
173173

174+
export async function listItemPermissionForms({ identifier }) {
175+
let store = await getStoreHandle({ id: identifier, type: "item" });
176+
if (!(await store.exists())) {
177+
throw new Error(`Item with identifier '${identifier}' does not exist in the store`);
178+
}
179+
180+
let files = await store.listResources();
181+
files = files
182+
.filter((file) => file.Key.match(/permission\.pdf/))
183+
.map((file) => ({ name: file.Key }));
184+
return { files };
185+
}
186+
174187
export async function listItemResourceFiles({ identifier, resource }) {
175188
let store = await getStoreHandle({ id: identifier, type: "item" });
176189
if (!(await store.exists())) {
@@ -258,20 +271,15 @@ export function groupFilesByResource({ identifier, files, naming }) {
258271
return { files: groupBy(resources, "resourceId") };
259272
}
260273

261-
export async function getResourceProcessingStatus({ itemId, resources, dateFrom }) {
262-
let dateTo = new Date();
263-
return await models.task.findAll({
264-
where: {
265-
[Op.and]: {
266-
itemId,
267-
updatedAt: {
268-
[Op.between]: [dateFrom, dateTo],
269-
},
270-
[Op.or]: { resource: resources },
271-
},
274+
// TODO this method does not have tests
275+
export async function getResourceProcessingStatus({ itemId, taskIds }) {
276+
let where = {
277+
[Op.and]: {
278+
itemId,
279+
[Op.or]: taskIds.map((id) => ({ id })),
272280
},
273-
order: [["updatedAt", "DESC"]],
274-
});
281+
};
282+
return models.task.findAll({ where, order: [["updatedAt", "DESC"]] });
275283
}
276284

277285
export async function statItemFile({ identifier, file }) {

‎api/src/lib/item.spec.js

+15
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import {
1010
getItemResourceLink,
1111
deleteItemResource,
1212
deleteItemResourceFile,
13+
listItemPermissionForms,
1314
listItemResourceFiles,
1415
itemResourceExists,
1516
markResourceComplete,
@@ -224,6 +225,20 @@ describe("Item management tests", () => {
224225
await item.destroy();
225226
await bucket.removeObjects({ prefix: identifier });
226227
});
228+
it("should be able to list item permission forms", async () => {
229+
let user = users.filter((u) => !u.administrator)[0];
230+
let { item } = await setupTestItem({ identifier, store, user });
231+
await store.put({ json: {}, target: `${identifier}-rights-holder-permission.pdf` });
232+
await store.put({
233+
json: {},
234+
target: `${identifier}-language-authority-permission.pdf`,
235+
});
236+
237+
let { files } = await listItemPermissionForms({ identifier });
238+
expect(files.length).toEqual(2);
239+
await item.destroy();
240+
await bucket.removeObjects({ prefix: identifier });
241+
});
227242
it("should be able to mark all resources as complete", async () => {
228243
let user = users.filter((u) => !u.administrator)[0];
229244
let { item } = await setupTestItem({ identifier, store, user });

‎api/src/routes/admin.spec.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -165,13 +165,13 @@ describe("Admin route tests", () => {
165165
id: identifier,
166166
type: "item",
167167
});
168-
await storeItem.createItem();
168+
await storeItem.createObject();
169169

170170
let storeCollection = await getStoreHandle({
171171
id: identifier,
172172
type: "collection",
173173
});
174-
await storeCollection.createItem();
174+
await storeCollection.createObject();
175175

176176
// connect as an admin
177177
let adminUser = users.filter((u) => u.administrator)[0];

‎api/src/routes/data/index.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import {
77
} from "../../common/index.js";
88
const log = getLogger();
99

10-
import { authenticateTusRequest, triggerProcessing, getItemPath } from "./upload.js";
10+
import { authenticateTusRequest, triggerProcessing, getUploadDetails } from "./upload.js";
1111

1212
export function setupRoutes(fastify, options, done) {
1313
fastify.addHook("preHandler", demandAuthenticatedUser);
@@ -16,7 +16,7 @@ export function setupRoutes(fastify, options, done) {
1616
fastify.get(
1717
"/upload/pre-create/:itemType/:identifier",
1818
{ preHandler: requireItemAccess },
19-
getItemPath
19+
getUploadDetails
2020
);
2121
fastify.get(
2222
"/upload/post-finish/:identifier/:resource",

‎api/src/routes/data/upload.js

+13-8
Original file line numberDiff line numberDiff line change
@@ -7,40 +7,45 @@ export async function authenticateTusRequest(req, res) {
77
return {};
88
}
99

10-
export async function getItemPath(req, res) {
10+
export async function getUploadDetails(req, res) {
1111
if (!req.params.identifier) return;
1212

1313
if (!req.session.user.upload) {
1414
return res.unauthorized();
1515
}
1616
let store = await getStoreHandle({ id: req.params.identifier, type: req.params.itemType });
17-
return { path: store.getObjectPath() };
17+
return {
18+
path: store.getObjectPath(),
19+
bucket: req.session.configuration.api.services.s3.bucket,
20+
};
1821
}
1922

2023
export async function triggerProcessing(req) {
2124
// await new Promise((resolve) => setTimeout(resolve, 15000));
2225
const { identifier, resource } = req.params;
26+
const { action } = req.body;
2327

2428
log.info(`Process: ${identifier}/${resource}`);
2529
let name;
26-
if (resource.match(/digivol\.csv/)) {
30+
if (resource === `${identifier}-digivol.csv`) {
2731
// process digivol file
2832
name = "process-digivol";
29-
} else if (resource.match(/tei\.xml/)) {
33+
} else if (resource === `${identifier}-tei.xml`) {
3034
// process tei xml file
3135
name = "process-tei";
3236
} else {
3337
// process uploaded image
34-
name = "process-image";
38+
name = action ? action : "process-image";
3539
}
36-
const task = {
40+
41+
let task = {
3742
rabbit: this.rabbit,
3843
configuration: req.session.configuration,
3944
item: req.session.item.get(),
4045
name,
4146
body: { resource },
4247
};
43-
await submitTask(task);
48+
task = await submitTask(task);
4449

45-
return {};
50+
return { taskId: task.id };
4651
}

‎api/src/routes/item.js

+13-4
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import {
1313
lookupItemByIdentifier,
1414
getItems,
1515
listItemResources,
16+
listItemPermissionForms,
1617
getItemResource,
1718
getItemResourceLink,
1819
putItemResource,
@@ -47,6 +48,7 @@ export function setupRoutes(fastify, options, done) {
4748
fastify.delete("/items/:identifier", deleteItemHandler);
4849
fastify.get("/items/:identifier/status", getItemStatisticsHandler);
4950
fastify.get("/items/:identifier/resources", getItemResourcesHandler);
51+
fastify.get("/items/:identifier/permission-forms", getItemPermissionFormsHandler);
5052
fastify.put("/items/:identifier/reprocess-imports", putReprocessImports);
5153
fastify.get("/items/:identifier/resources/:resource/files", getResourceFilesListHandler);
5254
fastify.get(
@@ -259,6 +261,14 @@ async function getItemResourcesHandler(req) {
259261
return { resources, total };
260262
}
261263

264+
async function getItemPermissionFormsHandler(req) {
265+
let query = {
266+
identifier: req.params.identifier,
267+
};
268+
let { files } = await listItemPermissionForms(query);
269+
return { files };
270+
}
271+
262272
async function putReprocessImports(req) {
263273
const { identifier } = req.params;
264274
const files = [
@@ -376,15 +386,14 @@ async function saveItemTranscriptionHandler(req, res) {
376386
}
377387
}
378388

389+
// TODO this method does not have tests
379390
async function postResourceProcessingStatus(req) {
391+
const { taskIds } = req.body;
380392
let tasks = await getResourceProcessingStatus({
381393
itemId: req.session.item.id,
382-
resources: req.body.resources.map((r) => r.resource),
383-
dateFrom: req.body.dateFrom,
394+
taskIds,
384395
});
385396
tasks = tasks.map((t) => t.get());
386-
tasks = groupBy(tasks, "resource");
387-
tasks = Object.keys(tasks).map((r) => tasks[r].shift());
388397
return { tasks };
389398
}
390399
// TODO: this method does not have tests

0 commit comments

Comments
 (0)