Skip to content

Commit 08fb12b

Browse files
marcolarosamarcolarosa
committed
Rework to use nocfl store as backend
This is a big change where the backend structure in the S3 bucket is now managed by the nocfl store implementation. commit 2a344c9e58c936e77b44aa4261af4a27285b8aeb Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 29 12:04:09 2022 +1000 implement script to migrate from old backend to store structure commit 354835ea44331397e8fd1abdfda9a576101bd4bd Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 29 11:00:48 2022 +1000 update dependencies commit 0bae5094faf26711f24646acde2cd93880cf7605 Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 29 11:00:38 2022 +1000 revise delete item commit 78b7e77f1b5ce1cf71b1371e396e2fd5ac59d8d7 Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 29 10:20:49 2022 +1000 revise to use batch upload commit af5d22ec96a0358e02437633ffe3983fa42dbd9a Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 29 09:38:48 2022 +1000 update dependencies commit 830305e Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 28 15:12:11 2022 +1000 add test for ocr processing commit b3dbfd4 Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 28 15:03:45 2022 +1000 add tests commit 2eb2167 Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 28 15:03:30 2022 +1000 support passing class name - needed for tests commit 767e974 Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 28 10:26:01 2022 +1000 refactor commit 97e4133 Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 15:15:09 2022 +1000 add tests and test data for image processing methods commit b73284d Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:50:46 2022 +1000 refactor and cleanup these methods commit 823b5d0 Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:50:37 2022 +1000 update method reference commit 4e3d5aa Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:50:22 2022 +1000 cleanup; rename method commit 4b9b60e Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:49:56 2022 +1000 revise methods to use the store rather than direct s3 access commit cb318a1 Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:49:36 2022 +1000 cleanup commit a33c2d5 Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:49:00 2022 +1000 export method to get store handle commit 4f2efd7 Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:48:34 2022 +1000 add method to get a store handle commit 150d485 Author: Marco La Rosa <m@lr.id.au> Date: Wed Jul 27 13:44:12 2022 +1000 update app dependencies commit c875300 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 15:20:59 2022 +1000 refactor as composition; get item path in storage and pass through to tus commit 4293396 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 15:20:46 2022 +1000 implement api endpoint to get item path in storage commit c30f9c9 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 15:20:28 2022 +1000 expect to get a path to upload file to commit 5fd16e9 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 14:15:05 2022 +1000 bugfix - revise return format commit 1fe2201 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 13:33:25 2022 +1000 simplify method commit f53e30e Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 11:57:52 2022 +1000 updating methods and tests commit c917893 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 09:24:10 2022 +1000 revise tests to use the store commit d8d1cc8 Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 09:23:52 2022 +1000 revise get admin entries to work with the store commit dc27dad Author: Marco La Rosa <m@lr.id.au> Date: Tue Jul 26 09:23:15 2022 +1000 add listObjects method operating on a prefix; add prefix to listResources commit e9bdcec Author: Marco La Rosa <m@lr.id.au> Date: Mon Jul 25 15:22:55 2022 +1000 update app dependencies commit 383a2d1 Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 15 10:46:58 2022 +1000 revise to use nocfl store commit 8cf1b6c Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 15 10:46:49 2022 +1000 revise to use nocfl store commit 77a18d8 Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 15 10:46:22 2022 +1000 bugfix commit ca21075 Author: Marco La Rosa <m@lr.id.au> Date: Fri Jul 15 10:14:55 2022 +1000 update dependencies commit ef3110b Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 14 11:15:44 2022 +1000 cleanup commit 40ab501 Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 14 11:15:33 2022 +1000 cleanup commit 4664c3f Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 14 11:06:51 2022 +1000 add configuration option domain commit 1d53597 Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 14 11:06:43 2022 +1000 rework to use nocfl store commit 3ac241b Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 14 11:06:13 2022 +1000 implement method to get a handle to the store This replaces using the s3 methods directly commit eda2017 Author: Marco La Rosa <m@lr.id.au> Date: Thu Jul 14 11:05:44 2022 +1000 update dependencies
1 parent 436f3f5 commit 08fb12b

35 files changed

+2960
-2672
lines changed

‎api/package-lock.json

+1,362-1,155
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎api/package.json

+13-13
Original file line numberDiff line numberDiff line change
@@ -13,31 +13,31 @@
1313
"author": "Marco La Rosa",
1414
"license": "GPL-3.0",
1515
"devDependencies": {
16-
"@babel/cli": "^7.18.6",
17-
"@babel/core": "^7.18.6",
18-
"@babel/node": "^7.18.6",
19-
"@babel/plugin-proposal-optional-chaining": "^7.18.6",
20-
"@babel/preset-env": "^7.18.6",
21-
"babel-jest": "^28.1.2",
16+
"@babel/cli": "^7.18.9",
17+
"@babel/core": "^7.18.9",
18+
"@babel/node": "^7.18.9",
19+
"@babel/plugin-proposal-optional-chaining": "^7.18.9",
20+
"@babel/preset-env": "^7.18.9",
21+
"babel-jest": "^28.1.3",
2222
"babel-loader": "^8.2.5",
2323
"chance": "^1.1.8",
2424
"clean-webpack-plugin": "^4.0.0",
2525
"copy-webpack-plugin": "^11.0.0",
2626
"debug": "^4.3.4",
27-
"jest": "^28.1.2",
27+
"jest": "^28.1.3",
2828
"jest-fetch-mock": "^3.0.3",
2929
"nodemon": "^2.0.19",
30-
"webpack": "^5.73.0",
30+
"webpack": "^5.74.0",
3131
"webpack-cli": "^4.10.0",
3232
"webpack-node-externals": "^3.0.0"
3333
},
3434
"dependencies": {
35-
"@aws-sdk/client-s3": "^3.127.0",
36-
"@aws-sdk/lib-storage": "^3.127.0",
37-
"@aws-sdk/s3-request-presigner": "^3.127.0",
38-
"@coedl/nocfl-js": "^1.0.0",
35+
"@aws-sdk/client-s3": "^3.137.0",
36+
"@aws-sdk/lib-storage": "^3.137.0",
37+
"@aws-sdk/s3-request-presigner": "^3.137.0",
38+
"@coedl/nocfl-js": "^1.7.1",
3939
"cross-fetch": "^3.1.5",
40-
"date-fns": "^2.28.0",
40+
"date-fns": "^2.29.1",
4141
"foo-foo-mq": "^7.1.0",
4242
"fs-extra": "^10.1.0",
4343
"jose": "^4.8.3",

‎api/src/common/getS3Handle.js

+22
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import { loadConfiguration } from ".";
22
import { S3, Bucket } from "./s3";
3+
import { Store } from "@coedl/nocfl-js";
34

45
export async function getS3Handle() {
56
const configuration = await loadConfiguration();
@@ -19,3 +20,24 @@ export async function getS3Handle() {
1920
s3 = new S3(params);
2021
return { s3, bucket };
2122
}
23+
24+
export async function getStoreHandle({ id, className }) {
25+
const configuration = await loadConfiguration();
26+
let s3 = configuration.api.services.s3;
27+
let credentials = {
28+
bucket: s3.bucket,
29+
accessKeyId: s3.awsAccessKeyId,
30+
secretAccessKey: s3.awsSecretAccessKey,
31+
region: s3.region,
32+
};
33+
if (s3.endpointUrl && s3.forcePathStyle) {
34+
credentials.endpoint = s3.endpointUrl;
35+
credentials.forcePathStyle = s3.forcePathStyle;
36+
}
37+
return new Store({
38+
id,
39+
className,
40+
domain: configuration.api.domain,
41+
credentials,
42+
});
43+
}

‎api/src/common/index.js

+21-4
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { ensureDir } from "fs-extra";
33
export { loadConfiguration, filterPrivateInformation } from "./configuration";
44
export { getLogger, logEvent } from "./logger";
55
export { submitTask, registerTask } from "./task";
6-
export { getS3Handle } from "./getS3Handle";
6+
export { getS3Handle, getStoreHandle } from "./getS3Handle";
77
import { getS3Handle } from "./getS3Handle";
88
export {
99
route,
@@ -32,15 +32,32 @@ export async function getUserTempLocation({ userId }) {
3232
return tempdir;
3333
}
3434

35-
export async function loadFiles({ continuationToken }) {
35+
export async function loadFiles({ prefix, continuationToken }) {
3636
let { bucket } = await getS3Handle();
37-
let resources = await bucket.listObjects({ continuationToken });
37+
let resources = await bucket.listObjects({ prefix, continuationToken });
3838
if (resources.NextContinuationToken) {
3939
return [
4040
...resources.Contents,
41-
...(await loadFiles({ continuationToken: resources.NextContinuationToken })),
41+
...(await loadFiles({ prefix, continuationToken: resources.NextContinuationToken })),
4242
];
4343
} else {
4444
return resources.Contents;
4545
}
4646
}
47+
48+
export async function listObjects({ prefix, continuationToken }) {
49+
let { bucket } = await getS3Handle();
50+
let resources = await bucket.listObjects({ prefix, continuationToken });
51+
if (resources.NextContinuationToken) {
52+
return [
53+
...resources.Contents?.filter((r) => r.Key.match("nocfl.identifier.json")).map((r) =>
54+
path.basename(path.dirname(r.Key))
55+
),
56+
...(await loadFiles({ prefix, continuationToken: resources.NextContinuationToken })),
57+
];
58+
} else {
59+
return resources.Contents?.filter((r) => r.Key.match("nocfl.identifier.json")).map((r) =>
60+
path.basename(path.dirname(r.Key))
61+
);
62+
}
63+
}

‎api/src/common/jwt.spec.js

+2-15
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { generateToken, verifyToken } from "./jwt";
44
import { loadConfiguration } from "../common";
55
const chance = require("chance").Chance();
66
import MockDate from "mockdate";
7-
import { copy, move, readJSON, writeJSON } from "fs-extra";
7+
import { copy, move, readJSON, writeJSON, readdir } from "fs-extra";
88
import { setupBeforeAll, setupBeforeEach, teardownAfterAll, teardownAfterEach } from "./";
99

1010
describe("JWT tests", () => {
@@ -63,25 +63,12 @@ describe("JWT tests", () => {
6363
let configuration = await loadConfiguration();
6464
let { token, expires } = await generateToken({ configuration, user });
6565

66-
await copy(
67-
"/srv/configuration/development-configuration.json",
68-
"/srv/configuration/development-configuration-copy.json"
69-
);
70-
let config = await readJSON("/srv/configuration/development-configuration.json");
71-
config.api.session.secret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
72-
await writeJSON("/srv/configuration/development-configuration.json", config);
73-
configuration = await loadConfiguration();
66+
configuration.api.session.secret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
7467
try {
7568
let data = await verifyToken({ token, configuration });
7669
} catch (error) {
7770
expect(error.message).toBe("signature verification failed");
7871
}
79-
80-
move(
81-
"/srv/configuration/development-configuration-copy.json",
82-
"/srv/configuration/development-configuration.json",
83-
{ overwrite: true }
84-
);
8572
await user.destroy();
8673
});
8774
});

‎api/src/common/s3.js

+24-6
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77
ListBucketsCommand,
88
GetObjectCommand,
99
PutObjectCommand,
10+
CopyObjectCommand,
1011
ListObjectsV2Command,
1112
DeleteObjectsCommand,
1213
} from "@aws-sdk/client-s3";
@@ -114,6 +115,21 @@ export class Bucket {
114115
return (await this.stat({ path }))?.$metadata?.httpStatusCode === 200 ? true : false;
115116
}
116117

118+
async copy({ source, target }) {
119+
source = path.join(this.bucket, source);
120+
target = path.join(target);
121+
const command = new CopyObjectCommand({
122+
Bucket: this.bucket,
123+
CopySource: source,
124+
Key: target,
125+
});
126+
try {
127+
const response = await this.client.send(command);
128+
} catch (error) {
129+
console.log(error);
130+
}
131+
}
132+
117133
async upload({
118134
localPath = undefined,
119135
content = undefined,
@@ -268,14 +284,16 @@ export class Bucket {
268284
async removeObjects({ keys = [], prefix = undefined }) {
269285
if (prefix) {
270286
let objects = (await this.listObjects({ prefix })).Contents;
271-
keys = objects.map((entry) => entry.Key);
287+
if (objects) keys = objects.map((entry) => entry.Key);
272288
}
273289
let objs = keys.map((k) => ({ Key: k }));
274-
const command = new DeleteObjectsCommand({
275-
Bucket: this.bucket,
276-
Delete: { Objects: objs },
277-
});
278-
return (await this.client.send(command)).$metadata;
290+
if (objs?.length) {
291+
const command = new DeleteObjectsCommand({
292+
Bucket: this.bucket,
293+
Delete: { Objects: objs },
294+
});
295+
return (await this.client.send(command)).$metadata;
296+
}
279297
}
280298

281299
async syncLocalPathToBucket({ localPath }) {

‎api/src/common/test-utils.js

+10-12
Original file line numberDiff line numberDiff line change
@@ -69,27 +69,25 @@ export async function generateLogs(info, warn, error) {
6969
}
7070
}
7171

72-
export async function setupTestItem({ user, bucket }) {
73-
const identifier = chance.word();
72+
export async function setupTestItem({ identifier, store, user }) {
7473
let item = await createItem({ identifier, userId: user.id });
7574
expect(item.identifier).toEqual(identifier);
7675

77-
await bucket.upload({
76+
await store.put({
7877
json: { some: "thing" },
79-
target: `${identifier}/${identifier}-01.json`,
78+
target: `${identifier}-01.json`,
8079
});
81-
await bucket.upload({
80+
await store.put({
8281
content: "text",
83-
target: `${identifier}/${identifier}-01.txt`,
82+
target: `${identifier}-01.txt`,
8483
});
85-
await bucket.upload({
84+
await store.put({
8685
json: { some: "thing" },
87-
target: `${identifier}/${identifier}-02.json`,
86+
target: `${identifier}-02.json`,
8887
});
89-
await bucket.upload({
88+
await store.put({
9089
content: "text",
91-
target: `${identifier}/${identifier}-02.txt`,
90+
target: `${identifier}-02.txt`,
9291
});
93-
94-
return { item, identifier };
92+
return { item };
9593
}

‎api/src/lib/collection.js

+7-45
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import models from "../models";
2-
import { loadConfiguration, getS3Handle, getUserTempLocation } from "../common";
3-
import path from "path";
4-
import { writeJson, remove } from "fs-extra";
2+
import { getStoreHandle } from "../common";
53

64
export async function lookupCollectionByIdentifier({ identifier, userId }) {
75
let clause = {
@@ -31,17 +29,12 @@ export async function getCollections({ userId, offset = 0, limit = 10 }) {
3129

3230
export async function createCollection({ identifier, userId }) {
3331
let collection = await models.collection.findOne({ where: { identifier } });
34-
let item = await models.item.findOne({ where: { identifier } });
3532
if (collection) {
3633
throw new Error(`A collection with that identifier already exists.`);
3734
}
38-
if (item) {
39-
throw new Error(`An item with that identifier already exists.`);
40-
}
41-
4235
collection = await models.collection.create({ identifier, data: { private: true } });
4336
await linkCollectionToUser({ collectionId: collection.id, userId });
44-
await createCollectionLocationInObjectStore({ identifier, userId });
37+
await createCollectionLocationInObjectStore({ identifier });
4538
return collection;
4639
}
4740

@@ -51,42 +44,11 @@ export async function linkCollectionToUser({ collectionId, userId }) {
5144
await user.addCollections([collection]);
5245
}
5346

54-
export async function createCollectionLocationInObjectStore({ identifier, userId }) {
55-
let { bucket } = await getS3Handle();
56-
57-
let pathExists = await bucket.pathExists({ path: identifier });
58-
if (!pathExists) {
59-
// create stub ro-crate file
60-
let context = ["https://w3id.org/ro/crate/1.1/context"];
61-
let graph = [
62-
{
63-
"@id": "ro-crate-metadata.json",
64-
"@type": "CreativeWork",
65-
conformsTo: {
66-
"@id": "https://w3id.org/ro/crate/1.1/context",
67-
},
68-
about: {
69-
"@id": "./",
70-
},
71-
},
72-
{
73-
"@id": "./",
74-
"@type": "Dataset",
75-
name: identifier,
76-
},
77-
];
78-
let tempdir = await getUserTempLocation({ userId });
79-
let crateFile = path.join(tempdir, "ro-crate-metadata.json");
80-
await writeJson(crateFile, {
81-
"@context": context,
82-
"@graph": graph,
83-
});
84-
await bucket.upload({
85-
localPath: crateFile,
86-
target: path.join(identifier, "ro-crate-metadata.json"),
87-
});
88-
await bucket.upload({ json: {}, target: path.join(identifier, ".collection") });
89-
await remove(tempdir);
47+
export async function createCollectionLocationInObjectStore({ identifier }) {
48+
let store = await getStoreHandle({ id: identifier, className: "collection" });
49+
let exists = await store.itemExists();
50+
if (!exists) {
51+
await store.createItem();
9052
}
9153
}
9254

0 commit comments

Comments
 (0)