From 9fd2a1bdcb9028645f2786f6c4980474ff8558f9 Mon Sep 17 00:00:00 2001 From: kadami Date: Fri, 13 Oct 2023 03:44:33 +0700 Subject: [PATCH 1/8] upload restructure for multer-s3, add s3 utils, modify to handle streams in update controller --- desci-server/package.json | 2 + desci-server/src/controllers/data/update.ts | 17 +- desci-server/src/routes/v1/data.ts | 6 +- desci-server/src/services/ipfs.ts | 2 +- desci-server/src/services/s3.ts | 26 + desci-server/yarn.lock | 1106 ++++++++++++++++++- 6 files changed, 1147 insertions(+), 12 deletions(-) create mode 100644 desci-server/src/services/s3.ts diff --git a/desci-server/package.json b/desci-server/package.json index 28f81fb37..0ad72f85e 100755 --- a/desci-server/package.json +++ b/desci-server/package.json @@ -82,6 +82,7 @@ "mkdirp": "^1.0.4", "morgan": "^1.10.0", "multer": "^1.4.5-lts.1", + "multer-s3": "^3.0.1", "multiformats": "^9.6.4", "multihashes": "^4.0.3", "nebulus": "^0.0.5", @@ -124,6 +125,7 @@ "@types/jsonwebtoken": "^8.5.4", "@types/mocha": "^10.0.1", "@types/morgan": "^1.9.3", + "@types/multer-s3": "^3.0.1", "@types/node": "^16.4.13", "@types/supertest": "^2.0.12", "@types/validator": "^13.6.3", diff --git a/desci-server/src/controllers/data/update.ts b/desci-server/src/controllers/data/update.ts index 6fa8d0225..155173c32 100644 --- a/desci-server/src/controllers/data/update.ts +++ b/desci-server/src/controllers/data/update.ts @@ -28,6 +28,7 @@ import { pinDirectory, RecursiveLsResult, } from 'services/ipfs'; +import { fetchFileStreamFromS3 } from 'services/s3'; import { arrayXor, calculateTotalZipUncompressedSize, @@ -126,7 +127,8 @@ export const update = async (req: Request, res: Response { - return { path: f.originalname, content: f.buffer }; - }); + const structuredFilesForPinning: IpfsDirStructuredInput[] = await Promise.all( + files.map(async (f: any) => { + const fileStream = await fetchFileStreamFromS3(f.location); + return { path: f.originalname, content: fileStream }; + }), + ); if (structuredFilesForPinning.length || externalUrlFiles?.length) { const filesToPin = structuredFilesForPinning.length ? structuredFilesForPinning : externalUrlFiles; if (filesToPin.length) uploaded = await pinDirectory(filesToPin); - if (!uploaded.length) res.status(400).json({ error: 'Failed uploading to ipfs' }); + if (!uploaded.length) return res.status(400).json({ error: 'Failed uploading to ipfs' }); logger.info('[UPDATE DATASET] Pinned files: ', uploaded.length); } @@ -289,7 +294,7 @@ export const update = async (req: Request, res: Response { + const key = url.replace(`https://${process.env.AWS_S3_BUCKET_NAME}.s3.amazonaws.com/`, ''); + + const params = { + Bucket: process.env.AWS_S3_BUCKET_NAME, + Key: key, + }; + + try { + const data = await s3Client.send(new GetObjectCommand(params)); + return data.Body as ReadableStream; + } catch (error) { + logger.error('Error fetching from S3:', error); + return null; + } +} diff --git a/desci-server/yarn.lock b/desci-server/yarn.lock index e812eed48..72e39c751 100755 --- a/desci-server/yarn.lock +++ b/desci-server/yarn.lock @@ -48,6 +48,584 @@ resolved "https://registry.yarnpkg.com/@assemblyscript/loader/-/loader-0.9.4.tgz#a483c54c1253656bb33babd464e3154a173e1577" integrity sha512-HazVq9zwTVwGmqdwYzu7WyQ6FQVZ7SwET0KKQuKm55jD0IfUpZgN0OPIiZG3zV1iSrVYcN0bdwLRXI/VNCYsUA== +"@aws-crypto/crc32@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/crc32/-/crc32-3.0.0.tgz#07300eca214409c33e3ff769cd5697b57fdd38fa" + integrity sha512-IzSgsrxUcsrejQbPVilIKy16kAT52EwB6zSaI+M3xxIhKh5+aldEyvI+z6erM7TCLB2BJsFrtHjp6/4/sr+3dA== + dependencies: + "@aws-crypto/util" "^3.0.0" + "@aws-sdk/types" "^3.222.0" + tslib "^1.11.1" + +"@aws-crypto/crc32c@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/crc32c/-/crc32c-3.0.0.tgz#016c92da559ef638a84a245eecb75c3e97cb664f" + integrity sha512-ENNPPManmnVJ4BTXlOjAgD7URidbAznURqD0KvfREyc4o20DPYdEldU1f5cQ7Jbj0CJJSPaMIk/9ZshdB3210w== + dependencies: + "@aws-crypto/util" "^3.0.0" + "@aws-sdk/types" "^3.222.0" + tslib "^1.11.1" + +"@aws-crypto/ie11-detection@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/ie11-detection/-/ie11-detection-3.0.0.tgz#640ae66b4ec3395cee6a8e94ebcd9f80c24cd688" + integrity sha512-341lBBkiY1DfDNKai/wXM3aujNBkXR7tq1URPQDL9wi3AUbI80NR74uF1TXHMm7po1AcnFk8iu2S2IeU/+/A+Q== + dependencies: + tslib "^1.11.1" + +"@aws-crypto/sha1-browser@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/sha1-browser/-/sha1-browser-3.0.0.tgz#f9083c00782b24714f528b1a1fef2174002266a3" + integrity sha512-NJth5c997GLHs6nOYTzFKTbYdMNA6/1XlKVgnZoaZcQ7z7UJlOgj2JdbHE8tiYLS3fzXNCguct77SPGat2raSw== + dependencies: + "@aws-crypto/ie11-detection" "^3.0.0" + "@aws-crypto/supports-web-crypto" "^3.0.0" + "@aws-crypto/util" "^3.0.0" + "@aws-sdk/types" "^3.222.0" + "@aws-sdk/util-locate-window" "^3.0.0" + "@aws-sdk/util-utf8-browser" "^3.0.0" + tslib "^1.11.1" + +"@aws-crypto/sha256-browser@3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-browser/-/sha256-browser-3.0.0.tgz#05f160138ab893f1c6ba5be57cfd108f05827766" + integrity sha512-8VLmW2B+gjFbU5uMeqtQM6Nj0/F1bro80xQXCW6CQBWgosFWXTx77aeOF5CAIAmbOK64SdMBJdNr6J41yP5mvQ== + dependencies: + "@aws-crypto/ie11-detection" "^3.0.0" + "@aws-crypto/sha256-js" "^3.0.0" + "@aws-crypto/supports-web-crypto" "^3.0.0" + "@aws-crypto/util" "^3.0.0" + "@aws-sdk/types" "^3.222.0" + "@aws-sdk/util-locate-window" "^3.0.0" + "@aws-sdk/util-utf8-browser" "^3.0.0" + tslib "^1.11.1" + +"@aws-crypto/sha256-js@3.0.0", "@aws-crypto/sha256-js@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/sha256-js/-/sha256-js-3.0.0.tgz#f06b84d550d25521e60d2a0e2a90139341e007c2" + integrity sha512-PnNN7os0+yd1XvXAy23CFOmTbMaDxgxXtTKHybrJ39Y8kGzBATgBFibWJKH6BhytLI/Zyszs87xCOBNyBig6vQ== + dependencies: + "@aws-crypto/util" "^3.0.0" + "@aws-sdk/types" "^3.222.0" + tslib "^1.11.1" + +"@aws-crypto/supports-web-crypto@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/supports-web-crypto/-/supports-web-crypto-3.0.0.tgz#5d1bf825afa8072af2717c3e455f35cda0103ec2" + integrity sha512-06hBdMwUAb2WFTuGG73LSC0wfPu93xWwo5vL2et9eymgmu3Id5vFAHBbajVWiGhPO37qcsdCap/FqXvJGJWPIg== + dependencies: + tslib "^1.11.1" + +"@aws-crypto/util@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@aws-crypto/util/-/util-3.0.0.tgz#1c7ca90c29293f0883468ad48117937f0fe5bfb0" + integrity sha512-2OJlpeJpCR48CC8r+uKVChzs9Iungj9wkZrl8Z041DWEWvyIHILYKCPNzJghKsivj+S3mLo6BVc7mBNzdxA46w== + dependencies: + "@aws-sdk/types" "^3.222.0" + "@aws-sdk/util-utf8-browser" "^3.0.0" + tslib "^1.11.1" + +"@aws-sdk/client-s3@^3.0.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-s3/-/client-s3-3.427.0.tgz#ae99d9e780d7d30e17247e78b18afcd32af3cdca" + integrity sha512-YKjJ9zgn0oE393HURKgvjNoX6lxUjb+dkTBE1GymFnGCPl6VxQbKXajXWNqUyN+oPPlZ2osEiljPaN0RserUjA== + dependencies: + "@aws-crypto/sha1-browser" "3.0.0" + "@aws-crypto/sha256-browser" "3.0.0" + "@aws-crypto/sha256-js" "3.0.0" + "@aws-sdk/client-sts" "3.427.0" + "@aws-sdk/credential-provider-node" "3.427.0" + "@aws-sdk/middleware-bucket-endpoint" "3.425.0" + "@aws-sdk/middleware-expect-continue" "3.425.0" + "@aws-sdk/middleware-flexible-checksums" "3.425.0" + "@aws-sdk/middleware-host-header" "3.425.0" + "@aws-sdk/middleware-location-constraint" "3.425.0" + "@aws-sdk/middleware-logger" "3.425.0" + "@aws-sdk/middleware-recursion-detection" "3.425.0" + "@aws-sdk/middleware-sdk-s3" "3.427.0" + "@aws-sdk/middleware-signing" "3.425.0" + "@aws-sdk/middleware-ssec" "3.425.0" + "@aws-sdk/middleware-user-agent" "3.427.0" + "@aws-sdk/region-config-resolver" "3.425.0" + "@aws-sdk/signature-v4-multi-region" "3.425.0" + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-endpoints" "3.427.0" + "@aws-sdk/util-user-agent-browser" "3.425.0" + "@aws-sdk/util-user-agent-node" "3.425.0" + "@aws-sdk/xml-builder" "3.310.0" + "@smithy/config-resolver" "^2.0.11" + "@smithy/eventstream-serde-browser" "^2.0.10" + "@smithy/eventstream-serde-config-resolver" "^2.0.10" + "@smithy/eventstream-serde-node" "^2.0.10" + "@smithy/fetch-http-handler" "^2.2.1" + "@smithy/hash-blob-browser" "^2.0.10" + "@smithy/hash-node" "^2.0.10" + "@smithy/hash-stream-node" "^2.0.10" + "@smithy/invalid-dependency" "^2.0.10" + "@smithy/md5-js" "^2.0.10" + "@smithy/middleware-content-length" "^2.0.12" + "@smithy/middleware-endpoint" "^2.0.10" + "@smithy/middleware-retry" "^2.0.13" + "@smithy/middleware-serde" "^2.0.10" + "@smithy/middleware-stack" "^2.0.4" + "@smithy/node-config-provider" "^2.0.13" + "@smithy/node-http-handler" "^2.1.6" + "@smithy/protocol-http" "^3.0.6" + "@smithy/smithy-client" "^2.1.9" + "@smithy/types" "^2.3.4" + "@smithy/url-parser" "^2.0.10" + "@smithy/util-base64" "^2.0.0" + "@smithy/util-body-length-browser" "^2.0.0" + "@smithy/util-body-length-node" "^2.1.0" + "@smithy/util-defaults-mode-browser" "^2.0.13" + "@smithy/util-defaults-mode-node" "^2.0.15" + "@smithy/util-retry" "^2.0.3" + "@smithy/util-stream" "^2.0.14" + "@smithy/util-utf8" "^2.0.0" + "@smithy/util-waiter" "^2.0.10" + fast-xml-parser "4.2.5" + tslib "^2.5.0" + +"@aws-sdk/client-sso@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-sso/-/client-sso-3.427.0.tgz#852f0bb00c7bc5e3d3c8751a6ff4e86a1484726f" + integrity sha512-sFVFEmsQ1rmgYO1SgrOTxE/MTKpeE4hpOkm1WqhLQK7Ij136vXpjCxjH1JYZiHiUzO1wr9t4ex4dlB5J3VS/Xg== + dependencies: + "@aws-crypto/sha256-browser" "3.0.0" + "@aws-crypto/sha256-js" "3.0.0" + "@aws-sdk/middleware-host-header" "3.425.0" + "@aws-sdk/middleware-logger" "3.425.0" + "@aws-sdk/middleware-recursion-detection" "3.425.0" + "@aws-sdk/middleware-user-agent" "3.427.0" + "@aws-sdk/region-config-resolver" "3.425.0" + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-endpoints" "3.427.0" + "@aws-sdk/util-user-agent-browser" "3.425.0" + "@aws-sdk/util-user-agent-node" "3.425.0" + "@smithy/config-resolver" "^2.0.11" + "@smithy/fetch-http-handler" "^2.2.1" + "@smithy/hash-node" "^2.0.10" + "@smithy/invalid-dependency" "^2.0.10" + "@smithy/middleware-content-length" "^2.0.12" + "@smithy/middleware-endpoint" "^2.0.10" + "@smithy/middleware-retry" "^2.0.13" + "@smithy/middleware-serde" "^2.0.10" + "@smithy/middleware-stack" "^2.0.4" + "@smithy/node-config-provider" "^2.0.13" + "@smithy/node-http-handler" "^2.1.6" + "@smithy/protocol-http" "^3.0.6" + "@smithy/smithy-client" "^2.1.9" + "@smithy/types" "^2.3.4" + "@smithy/url-parser" "^2.0.10" + "@smithy/util-base64" "^2.0.0" + "@smithy/util-body-length-browser" "^2.0.0" + "@smithy/util-body-length-node" "^2.1.0" + "@smithy/util-defaults-mode-browser" "^2.0.13" + "@smithy/util-defaults-mode-node" "^2.0.15" + "@smithy/util-retry" "^2.0.3" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@aws-sdk/client-sts@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/client-sts/-/client-sts-3.427.0.tgz#839df8e1aa8795ffbffc7f5d79ccbc6a1220ab33" + integrity sha512-le2wLJKILyWuRfPz2HbyaNtu5kEki+ojUkTqCU6FPDRrqUvEkaaCBH9Awo/2AtrCfRkiobop8RuTTj6cAnpiJg== + dependencies: + "@aws-crypto/sha256-browser" "3.0.0" + "@aws-crypto/sha256-js" "3.0.0" + "@aws-sdk/credential-provider-node" "3.427.0" + "@aws-sdk/middleware-host-header" "3.425.0" + "@aws-sdk/middleware-logger" "3.425.0" + "@aws-sdk/middleware-recursion-detection" "3.425.0" + "@aws-sdk/middleware-sdk-sts" "3.425.0" + "@aws-sdk/middleware-signing" "3.425.0" + "@aws-sdk/middleware-user-agent" "3.427.0" + "@aws-sdk/region-config-resolver" "3.425.0" + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-endpoints" "3.427.0" + "@aws-sdk/util-user-agent-browser" "3.425.0" + "@aws-sdk/util-user-agent-node" "3.425.0" + "@smithy/config-resolver" "^2.0.11" + "@smithy/fetch-http-handler" "^2.2.1" + "@smithy/hash-node" "^2.0.10" + "@smithy/invalid-dependency" "^2.0.10" + "@smithy/middleware-content-length" "^2.0.12" + "@smithy/middleware-endpoint" "^2.0.10" + "@smithy/middleware-retry" "^2.0.13" + "@smithy/middleware-serde" "^2.0.10" + "@smithy/middleware-stack" "^2.0.4" + "@smithy/node-config-provider" "^2.0.13" + "@smithy/node-http-handler" "^2.1.6" + "@smithy/protocol-http" "^3.0.6" + "@smithy/smithy-client" "^2.1.9" + "@smithy/types" "^2.3.4" + "@smithy/url-parser" "^2.0.10" + "@smithy/util-base64" "^2.0.0" + "@smithy/util-body-length-browser" "^2.0.0" + "@smithy/util-body-length-node" "^2.1.0" + "@smithy/util-defaults-mode-browser" "^2.0.13" + "@smithy/util-defaults-mode-node" "^2.0.15" + "@smithy/util-retry" "^2.0.3" + "@smithy/util-utf8" "^2.0.0" + fast-xml-parser "4.2.5" + tslib "^2.5.0" + +"@aws-sdk/credential-provider-env@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-env/-/credential-provider-env-3.425.0.tgz#1f5be812aeed558efaebce641e4c030b86875544" + integrity sha512-J20etnLvMKXRVi5FK4F8yOCNm2RTaQn5psQTGdDEPWJNGxohcSpzzls8U2KcMyUJ+vItlrThr4qwgpHG3i/N0w== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/credential-provider-ini@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.427.0.tgz#bf52067ed5ef6971c7785d09bdf3c6aa16afc2b1" + integrity sha512-NmH1cO/w98CKMltYec3IrJIIco19wRjATFNiw83c+FGXZ+InJwReqBnruxIOmKTx2KDzd6fwU1HOewS7UjaaaQ== + dependencies: + "@aws-sdk/credential-provider-env" "3.425.0" + "@aws-sdk/credential-provider-process" "3.425.0" + "@aws-sdk/credential-provider-sso" "3.427.0" + "@aws-sdk/credential-provider-web-identity" "3.425.0" + "@aws-sdk/types" "3.425.0" + "@smithy/credential-provider-imds" "^2.0.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/shared-ini-file-loader" "^2.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/credential-provider-node@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-node/-/credential-provider-node-3.427.0.tgz#f3bd63bc5ab5b897ce67d5960731f48c89ba7520" + integrity sha512-wYYbQ57nKL8OfgRbl8k6uXcdnYml+p3LSSfDUAuUEp1HKlQ8lOXFJ3BdLr5qrk7LhpyppSRnWBmh2c3kWa7ANQ== + dependencies: + "@aws-sdk/credential-provider-env" "3.425.0" + "@aws-sdk/credential-provider-ini" "3.427.0" + "@aws-sdk/credential-provider-process" "3.425.0" + "@aws-sdk/credential-provider-sso" "3.427.0" + "@aws-sdk/credential-provider-web-identity" "3.425.0" + "@aws-sdk/types" "3.425.0" + "@smithy/credential-provider-imds" "^2.0.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/shared-ini-file-loader" "^2.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/credential-provider-process@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-process/-/credential-provider-process-3.425.0.tgz#d5cd231e1732375fc918912f8083c8c45d9dc2ab" + integrity sha512-YY6tkLdvtb1Fgofp3b1UWO+5vwS14LJ/smGmuGpSba0V7gFJRdcrJ9bcb9vVgAGuMdjzRJ+bUKlLLtqXkaykEw== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/shared-ini-file-loader" "^2.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/credential-provider-sso@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.427.0.tgz#da54388247c0cf812e024c301a6f188550275850" + integrity sha512-c+tXyS/i49erHs4bAp6vKNYeYlyQ0VNMBgoco0LCn1rL0REtHbfhWMnqDLF6c2n3yIWDOTrQu0D73Idnpy16eA== + dependencies: + "@aws-sdk/client-sso" "3.427.0" + "@aws-sdk/token-providers" "3.427.0" + "@aws-sdk/types" "3.425.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/shared-ini-file-loader" "^2.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/credential-provider-web-identity@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.425.0.tgz#c1587cc39be70db2c828aeab7b68a8245bc86f91" + integrity sha512-/0R65TgRzL01JU3SzloivWNwdkbIhr06uY/F5pBHf/DynQqaspKNfdHn6AiozgSVDfwRHFjKBTUy6wvf3QFkuA== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/lib-storage@^3.46.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/lib-storage/-/lib-storage-3.427.0.tgz#ad7f5d0e7639b98d959363b2a99baf4a66d9b8c0" + integrity sha512-JE26Zo4SMMY2SGlD/FilF5MpLuP8D2IrW+ye/J77dwh91gyGnNa/lubJ7WbVjIAxgeaDHsAkcpNO4VR5k6JCKg== + dependencies: + "@smithy/abort-controller" "^2.0.1" + "@smithy/middleware-endpoint" "^2.0.10" + "@smithy/smithy-client" "^2.1.9" + buffer "5.6.0" + events "3.3.0" + stream-browserify "3.0.0" + tslib "^2.5.0" + +"@aws-sdk/middleware-bucket-endpoint@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-bucket-endpoint/-/middleware-bucket-endpoint-3.425.0.tgz#21a2658902482a88ea2201046ba324be6bb106b6" + integrity sha512-7UTfA10fmDw9cgHLApxRUNPywZTG4S/1TNZgTxndO/1OM9ZHtIatw1iLbqJD35gHrpEYI8Vo14YvcnD2ITuiMw== + dependencies: + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-arn-parser" "3.310.0" + "@smithy/node-config-provider" "^2.0.13" + "@smithy/protocol-http" "^3.0.6" + "@smithy/types" "^2.3.4" + "@smithy/util-config-provider" "^2.0.0" + tslib "^2.5.0" + +"@aws-sdk/middleware-expect-continue@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.425.0.tgz#65b1c0368582a5658729aa4c918aa1a135f05db2" + integrity sha512-CqAmnDST2o7+sKKw2/ffHKiYKE+jZb/Ce9U0P//ZYzqp9R1Wb016ID+W6DoxufyPJAS9dpRMcUDnAssmMIC/EA== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-flexible-checksums@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-flexible-checksums/-/middleware-flexible-checksums-3.425.0.tgz#57b12950b4174f2acdc3c4856b2b764c83c84b70" + integrity sha512-BDwn2vVVsC/AzmHXQlaZhEpKXL7GfKFpH7ZFccZuwEQBcyn8lVCcwtfaRe5P1mEe2wklVzOXd1dw8bt0+BOUPA== + dependencies: + "@aws-crypto/crc32" "3.0.0" + "@aws-crypto/crc32c" "3.0.0" + "@aws-sdk/types" "3.425.0" + "@smithy/is-array-buffer" "^2.0.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/types" "^2.3.4" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@aws-sdk/middleware-host-header@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-host-header/-/middleware-host-header-3.425.0.tgz#7bca371e1a5611ec20c06bd7017efa1900c367d0" + integrity sha512-E5Gt41LObQ+cr8QnLthwsH3MtVSNXy1AKJMowDr85h0vzqA/FHUkgHyOGntgozzjXT5M0MaSRYxS0xwTR5D4Ew== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-location-constraint@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-location-constraint/-/middleware-location-constraint-3.425.0.tgz#562cd09d85b8500bfcf94ff59b9ec8489e78f53a" + integrity sha512-3rt0LpGmL1LCRFuEObS1yERd9OEV+AEIAvhY7b53M7u7SyrjWQtpntWkI365L/QljhgMXQBfps2qO4JtrhQnsA== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-logger@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-logger/-/middleware-logger-3.425.0.tgz#e45f160b84798365e4acf8a283e9664ee9ee131b" + integrity sha512-INE9XWRXx2f4a/r2vOU0tAmgctVp7nEaEasemNtVBYhqbKLZvr9ndLBSgKGgJ8LIcXAoISipaMuFiqIGkFsm7A== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-recursion-detection@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-recursion-detection/-/middleware-recursion-detection-3.425.0.tgz#c348ec16ebb7c357bcb403904c24e8da1914961d" + integrity sha512-77gnzJ5b91bgD75L/ugpOyerx6lR3oyS4080X1YI58EzdyBMkDrHM4FbMcY2RynETi3lwXCFzLRyZjWXY1mRlw== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-sdk-s3@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-s3/-/middleware-sdk-s3-3.427.0.tgz#086b60ebdab052ef745d63a02406db87ff555b93" + integrity sha512-virGCf9vsqYCLpmngLOZOVSYgVr2cCOCvTuRoT9vf5tD/63JwaC173jnbdoJO6CWI7ID5Iz0eNdgITXVQ2mpew== + dependencies: + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-arn-parser" "3.310.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/smithy-client" "^2.1.9" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-sdk-sts@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-sdk-sts/-/middleware-sdk-sts-3.425.0.tgz#a020a04ddb5c6741d43d72afe79c24e6f1bb94b7" + integrity sha512-JFojrg76oKAoBknnr9EL5N2aJ1mRCtBqXoZYST58GSx8uYdFQ89qS65VNQ8JviBXzsrCNAn4vDhZ5Ch5E6TxGQ== + dependencies: + "@aws-sdk/middleware-signing" "3.425.0" + "@aws-sdk/types" "3.425.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-signing@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-signing/-/middleware-signing-3.425.0.tgz#fa133b8a76216d0b55558634b09cbe769f16b037" + integrity sha512-ZpOfgJHk7ovQ0sSwg3tU4NxFOnz53lJlkJRf7S+wxQALHM0P2MJ6LYBrZaFMVsKiJxNIdZBXD6jclgHg72ZW6Q== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/property-provider" "^2.0.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/signature-v4" "^2.0.0" + "@smithy/types" "^2.3.4" + "@smithy/util-middleware" "^2.0.3" + tslib "^2.5.0" + +"@aws-sdk/middleware-ssec@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-ssec/-/middleware-ssec-3.425.0.tgz#5faf5a44af23faf829f60d0ce08b36332f32cb3f" + integrity sha512-9HTuXnHYAZWkwPC8x9tElsQjFPxDT//orbIFauS7VF5HkLCKn9J6O6lW1wKMxrEnDwfN/Vi3nw479MoPj5Ss0Q== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/middleware-user-agent@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-user-agent/-/middleware-user-agent-3.427.0.tgz#a1b7cf9a848dcb4af454922abf5e9714bc4c20aa" + integrity sha512-y9HxYsNvnA3KqDl8w1jHeCwz4P9CuBEtu/G+KYffLeAMBsMZmh4SIkFFCO9wE/dyYg6+yo07rYcnnIfy7WA0bw== + dependencies: + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-endpoints" "3.427.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/region-config-resolver@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/region-config-resolver/-/region-config-resolver-3.425.0.tgz#b69cc305a4211c9f96f04ac3a10ff9a736ec13cb" + integrity sha512-u7uv/iUOapIJdRgRkO3wnpYsUgV6ponsZJQgVg/8L+n+Vo5PQL5gAcIuAOwcYSKQPFaeK+KbmByI4SyOK203Vw== + dependencies: + "@smithy/node-config-provider" "^2.0.13" + "@smithy/types" "^2.3.4" + "@smithy/util-config-provider" "^2.0.0" + "@smithy/util-middleware" "^2.0.3" + tslib "^2.5.0" + +"@aws-sdk/signature-v4-multi-region@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.425.0.tgz#9521058eec054b835070e85a1fd10877de49b154" + integrity sha512-7n2FRPE9rLaVa26xXQJ8TExrt53dWN824axQd1a0r5va0SmMQYG/iV5LBmwUlAntUSq46Lse4Q5YnbOVedGOmw== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/signature-v4" "^2.0.0" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/token-providers@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/token-providers/-/token-providers-3.427.0.tgz#d4b9aacda0a8fdd408bb95bf4b8de919df1227b8" + integrity sha512-4E5E+4p8lJ69PBY400dJXF06LUHYx5lkKzBEsYqWWhoZcoftrvi24ltIhUDoGVLkrLcTHZIWSdFAWSos4hXqeg== + dependencies: + "@aws-crypto/sha256-browser" "3.0.0" + "@aws-crypto/sha256-js" "3.0.0" + "@aws-sdk/middleware-host-header" "3.425.0" + "@aws-sdk/middleware-logger" "3.425.0" + "@aws-sdk/middleware-recursion-detection" "3.425.0" + "@aws-sdk/middleware-user-agent" "3.427.0" + "@aws-sdk/types" "3.425.0" + "@aws-sdk/util-endpoints" "3.427.0" + "@aws-sdk/util-user-agent-browser" "3.425.0" + "@aws-sdk/util-user-agent-node" "3.425.0" + "@smithy/config-resolver" "^2.0.11" + "@smithy/fetch-http-handler" "^2.2.1" + "@smithy/hash-node" "^2.0.10" + "@smithy/invalid-dependency" "^2.0.10" + "@smithy/middleware-content-length" "^2.0.12" + "@smithy/middleware-endpoint" "^2.0.10" + "@smithy/middleware-retry" "^2.0.13" + "@smithy/middleware-serde" "^2.0.10" + "@smithy/middleware-stack" "^2.0.4" + "@smithy/node-config-provider" "^2.0.13" + "@smithy/node-http-handler" "^2.1.6" + "@smithy/property-provider" "^2.0.0" + "@smithy/protocol-http" "^3.0.6" + "@smithy/shared-ini-file-loader" "^2.0.6" + "@smithy/smithy-client" "^2.1.9" + "@smithy/types" "^2.3.4" + "@smithy/url-parser" "^2.0.10" + "@smithy/util-base64" "^2.0.0" + "@smithy/util-body-length-browser" "^2.0.0" + "@smithy/util-body-length-node" "^2.1.0" + "@smithy/util-defaults-mode-browser" "^2.0.13" + "@smithy/util-defaults-mode-node" "^2.0.15" + "@smithy/util-retry" "^2.0.3" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@aws-sdk/types@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.425.0.tgz#8d4e94743a69c865a83785a9f3bcfd49945836f7" + integrity sha512-6lqbmorwerN4v+J5dqbHPAsjynI0mkEF+blf+69QTaKKGaxBBVaXgqoqul9RXYcK5MMrrYRbQIMd0zYOoy90kA== + dependencies: + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/types@^3.222.0": + version "3.428.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.428.0.tgz#fcb62a5fc38c4e579dc2b251194483aaad393df0" + integrity sha512-4T0Ps2spjg3qbWE6ZK13Vd3FnzpfliaiotqjxUK5YhjDrKXeT36HJp46JhDupElQuHtTkpdiJOSYk2lvY2H4IA== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@aws-sdk/util-arn-parser@3.310.0": + version "3.310.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.310.0.tgz#861ff8810851be52a320ec9e4786f15b5fc74fba" + integrity sha512-jL8509owp/xB9+Or0pvn3Fe+b94qfklc2yPowZZIFAkFcCSIdkIglz18cPDWnYAcy9JGewpMS1COXKIUhZkJsA== + dependencies: + tslib "^2.5.0" + +"@aws-sdk/util-endpoints@3.427.0": + version "3.427.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-endpoints/-/util-endpoints-3.427.0.tgz#09f7f36201ba80c1c669a0f4c506fb93de1e66d4" + integrity sha512-rSyiAIFF/EVvity/+LWUqoTMJ0a25RAc9iqx0WZ4tf1UjuEXRRXxZEb+jEZg1bk+pY84gdLdx9z5E+MSJCZxNQ== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/node-config-provider" "^2.0.13" + tslib "^2.5.0" + +"@aws-sdk/util-locate-window@^3.0.0": + version "3.310.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-locate-window/-/util-locate-window-3.310.0.tgz#b071baf050301adee89051032bd4139bba32cc40" + integrity sha512-qo2t/vBTnoXpjKxlsC2e1gBrRm80M3bId27r0BRB2VniSSe7bL1mmzM+/HFtujm0iAxtPM+aLEflLJlJeDPg0w== + dependencies: + tslib "^2.5.0" + +"@aws-sdk/util-user-agent-browser@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-browser/-/util-user-agent-browser-3.425.0.tgz#74d200d461ea2d75a8d4916c230ffe3a20fcb009" + integrity sha512-22Y9iMtjGcFjGILR6/xdp1qRezlHVLyXtnpEsbuPTiernRCPk6zfAnK/ATH77r02MUjU057tdxVkd5umUBTn9Q== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/types" "^2.3.4" + bowser "^2.11.0" + tslib "^2.5.0" + +"@aws-sdk/util-user-agent-node@3.425.0": + version "3.425.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-user-agent-node/-/util-user-agent-node-3.425.0.tgz#847c0d6526a34e174419dcecf0e12cd000158a84" + integrity sha512-SIR4F5uQeeVAi8lv4OgRirtdtNi5zeyogTuQgGi9su8F/WP1N6JqxofcwpUY5f8/oJ2UlXr/tx1f09UHfJJzvA== + dependencies: + "@aws-sdk/types" "3.425.0" + "@smithy/node-config-provider" "^2.0.13" + "@smithy/types" "^2.3.4" + tslib "^2.5.0" + +"@aws-sdk/util-utf8-browser@^3.0.0": + version "3.259.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/util-utf8-browser/-/util-utf8-browser-3.259.0.tgz#3275a6f5eb334f96ca76635b961d3c50259fd9ff" + integrity sha512-UvFa/vR+e19XookZF8RzFZBrw2EUkQWxiBW0yYQAhvk3C+QVGl0H3ouca8LDBlBfQKXwmW3huo/59H8rwb1wJw== + dependencies: + tslib "^2.3.1" + +"@aws-sdk/xml-builder@3.310.0": + version "3.310.0" + resolved "https://registry.yarnpkg.com/@aws-sdk/xml-builder/-/xml-builder-3.310.0.tgz#f0236f2103b438d16117e0939a6305ad69b7ff76" + integrity sha512-TqELu4mOuSIKQCqj63fGVs86Yh+vBx5nHRpWKNUNhB2nPTpfbziTs5c1X358be3peVWA4wPxW7Nt53KIg1tnNw== + dependencies: + tslib "^2.5.0" + "@babel/code-frame@7.12.11": version "7.12.11" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f" @@ -3134,6 +3712,444 @@ resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5" integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ== +"@smithy/abort-controller@^2.0.1", "@smithy/abort-controller@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/abort-controller/-/abort-controller-2.0.11.tgz#e1d96a2ecbf103d0b075a7456ce3afeeb9f76a87" + integrity sha512-MSzE1qR2JNyb7ot3blIOT3O3H0Jn06iNDEgHRaqZUwBgx5EG+VIx24Y21tlKofzYryIOcWpIohLrIIyocD6LMA== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/chunked-blob-reader-native@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/chunked-blob-reader-native/-/chunked-blob-reader-native-2.0.0.tgz#f6d0eeeb5481026b68b054f45540d924c194d558" + integrity sha512-HM8V2Rp1y8+1343tkZUKZllFhEQPNmpNdgFAncbTsxkZ18/gqjk23XXv3qGyXWp412f3o43ZZ1UZHVcHrpRnCQ== + dependencies: + "@smithy/util-base64" "^2.0.0" + tslib "^2.5.0" + +"@smithy/chunked-blob-reader@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/chunked-blob-reader/-/chunked-blob-reader-2.0.0.tgz#c44fe2c780eaf77f9e5381d982ac99a880cce51b" + integrity sha512-k+J4GHJsMSAIQPChGBrjEmGS+WbPonCXesoqP9fynIqjn7rdOThdH8FAeCmokP9mxTYKQAKoHCLPzNlm6gh7Wg== + dependencies: + tslib "^2.5.0" + +"@smithy/config-resolver@^2.0.11", "@smithy/config-resolver@^2.0.14": + version "2.0.14" + resolved "https://registry.yarnpkg.com/@smithy/config-resolver/-/config-resolver-2.0.14.tgz#16163e14053949f5a717be6f5802a7039e5ff4d1" + integrity sha512-K1K+FuWQoy8j/G7lAmK85o03O89s2Vvh6kMFmzEmiHUoQCRH1rzbDtMnGNiaMHeSeYJ6y79IyTusdRG+LuWwtg== + dependencies: + "@smithy/node-config-provider" "^2.1.1" + "@smithy/types" "^2.3.5" + "@smithy/util-config-provider" "^2.0.0" + "@smithy/util-middleware" "^2.0.4" + tslib "^2.5.0" + +"@smithy/credential-provider-imds@^2.0.0", "@smithy/credential-provider-imds@^2.0.16": + version "2.0.16" + resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-2.0.16.tgz#07da7ecd43eff92156ddc54f3b5330bbc128d5cd" + integrity sha512-tKa2xF+69TvGxJT+lnJpGrKxUuAZDLYXFhqnPEgnHz+psTpkpcB4QRjHj63+uj83KaeFJdTfW201eLZeRn6FfA== + dependencies: + "@smithy/node-config-provider" "^2.1.1" + "@smithy/property-provider" "^2.0.12" + "@smithy/types" "^2.3.5" + "@smithy/url-parser" "^2.0.11" + tslib "^2.5.0" + +"@smithy/eventstream-codec@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-codec/-/eventstream-codec-2.0.11.tgz#1ba090ea5dbf956e32d3d0d0986ffb0d0af8c57d" + integrity sha512-BQCTjxhCYRZIfXapa2LmZSaH8QUBGwMZw7XRN83hrdixbLjIcj+o549zjkedFS07Ve2TlvWUI6BTzP+nv7snBA== + dependencies: + "@aws-crypto/crc32" "3.0.0" + "@smithy/types" "^2.3.5" + "@smithy/util-hex-encoding" "^2.0.0" + tslib "^2.5.0" + +"@smithy/eventstream-serde-browser@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-browser/-/eventstream-serde-browser-2.0.11.tgz#eb9e9105d04c5dd0c96d5544857fe4e4d9d113a8" + integrity sha512-p9IK4uvwT6B3pT1VGlODvcVBfPVikjBFHAcKpvvNF+7lAEI+YiC6d0SROPkpjnvCgVBYyGXa3ciqrWnFze6mwQ== + dependencies: + "@smithy/eventstream-serde-universal" "^2.0.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/eventstream-serde-config-resolver@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-config-resolver/-/eventstream-serde-config-resolver-2.0.11.tgz#f5fda274bc823a5d84a1ab1634ea7f9f4e82d9cb" + integrity sha512-vN32E8yExo0Z8L7kXhlU9KRURrhqOpPdLxQMp3MwfMThrjiqbr1Sk5srUXc1ed2Ygl/l0TEN9vwNG0bQHg6AjQ== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/eventstream-serde-node@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-node/-/eventstream-serde-node-2.0.11.tgz#920e1b6ba6a216a58f519865b8585df61b675f87" + integrity sha512-Gjqbpg7UmD+YzkpgNShNcDNZcUpBWIkvX2XCGptz5PoxJU/UQbuF9eSc93ZlIb7j4aGjtFfqk23HUMW8Hopg2Q== + dependencies: + "@smithy/eventstream-serde-universal" "^2.0.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/eventstream-serde-universal@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/eventstream-serde-universal/-/eventstream-serde-universal-2.0.11.tgz#c86c0d29eae479590826ad61cb28301ec1105ebe" + integrity sha512-F8FsxLTbFN4+Esgpo+nNKcEajrgRZJ+pG9c8+MhLM4Odp5ejLHw2GMCXd81cGsgmfcbnzdDEXazPPVzOwj89MQ== + dependencies: + "@smithy/eventstream-codec" "^2.0.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/fetch-http-handler@^2.2.1", "@smithy/fetch-http-handler@^2.2.3": + version "2.2.3" + resolved "https://registry.yarnpkg.com/@smithy/fetch-http-handler/-/fetch-http-handler-2.2.3.tgz#86445f63dbf09ec331b6199fc2f0f44fec1b1417" + integrity sha512-0G9sePU+0R+8d7cie+OXzNbbkjnD4RfBlVCs46ZEuQAMcxK8OniemYXSSkOc80CCk8Il4DnlYZcUSvsIs2OB2w== + dependencies: + "@smithy/protocol-http" "^3.0.7" + "@smithy/querystring-builder" "^2.0.11" + "@smithy/types" "^2.3.5" + "@smithy/util-base64" "^2.0.0" + tslib "^2.5.0" + +"@smithy/hash-blob-browser@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/hash-blob-browser/-/hash-blob-browser-2.0.11.tgz#6bcd0ffc1f68427dff1d8051c893df92a36f3e7e" + integrity sha512-/6vq/NiH2EN3mWdwcLdjVohP+VCng+ZA1GnlUdx959egsfgIlLWQvCyjnB2ze9Hr6VHV5XEFLLpLQH2dHA6Sgw== + dependencies: + "@smithy/chunked-blob-reader" "^2.0.0" + "@smithy/chunked-blob-reader-native" "^2.0.0" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/hash-node@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/hash-node/-/hash-node-2.0.11.tgz#07d73eefa9ab28e4f03461c6ec0532b85792329d" + integrity sha512-PbleVugN2tbhl1ZoNWVrZ1oTFFas/Hq+s6zGO8B9bv4w/StTriTKA9W+xZJACOj9X7zwfoTLbscM+avCB1KqOQ== + dependencies: + "@smithy/types" "^2.3.5" + "@smithy/util-buffer-from" "^2.0.0" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@smithy/hash-stream-node@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/hash-stream-node/-/hash-stream-node-2.0.11.tgz#95c1ef3681d988770acdab863707daf068a851f8" + integrity sha512-Jn2yl+Dn0kvwKvSavvR1/BFVYa2wIkaJKWeTH48kno89gqHAJxMh1hrtBN6SJ7F8VhodNZTiNOlQVqCSfLheNQ== + dependencies: + "@smithy/types" "^2.3.5" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@smithy/invalid-dependency@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/invalid-dependency/-/invalid-dependency-2.0.11.tgz#41811da5da9950f52a0491ea532add2b1895349b" + integrity sha512-zazq99ujxYv/NOf9zh7xXbNgzoVLsqE0wle8P/1zU/XdhPi/0zohTPKWUzIxjGdqb5hkkwfBkNkl5H+LE0mvgw== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/is-array-buffer@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/is-array-buffer/-/is-array-buffer-2.0.0.tgz#8fa9b8040651e7ba0b2f6106e636a91354ff7d34" + integrity sha512-z3PjFjMyZNI98JFRJi/U0nGoLWMSJlDjAW4QUX2WNZLas5C0CmVV6LJ01JI0k90l7FvpmixjWxPFmENSClQ7ug== + dependencies: + tslib "^2.5.0" + +"@smithy/md5-js@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/md5-js/-/md5-js-2.0.11.tgz#0235c22eca4b5af72728f20348af5280bef2f275" + integrity sha512-YBIv+e95qeGvQA05ucwstmTeQ/bUzWgU+nO2Ffmif5awu6IzSR0Jfk3XLYh4mdy7f8DCgsn8qA63u7N9Lu0+5A== + dependencies: + "@smithy/types" "^2.3.5" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@smithy/middleware-content-length@^2.0.12": + version "2.0.13" + resolved "https://registry.yarnpkg.com/@smithy/middleware-content-length/-/middleware-content-length-2.0.13.tgz#eb8195510fac8e2d925e43f270f347d8e2ce038b" + integrity sha512-Md2kxWpaec3bXp1oERFPQPBhOXCkGSAF7uc1E+4rkwjgw3/tqAXRtbjbggu67HJdwaif76As8AV6XxbD1HzqTQ== + dependencies: + "@smithy/protocol-http" "^3.0.7" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/middleware-endpoint@^2.0.10": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@smithy/middleware-endpoint/-/middleware-endpoint-2.1.1.tgz#6eec29c380a8f0f9cadc9b28bf8b453c5b671985" + integrity sha512-YAqGagBvHqDEew4EGz9BrQ7M+f+u7ck9EL4zzYirOhIcXeBS/+q4A5+ObHDDwEp38lD6t88YUtFy3OptqEaDQg== + dependencies: + "@smithy/middleware-serde" "^2.0.11" + "@smithy/node-config-provider" "^2.1.1" + "@smithy/shared-ini-file-loader" "^2.2.0" + "@smithy/types" "^2.3.5" + "@smithy/url-parser" "^2.0.11" + "@smithy/util-middleware" "^2.0.4" + tslib "^2.5.0" + +"@smithy/middleware-retry@^2.0.13": + version "2.0.16" + resolved "https://registry.yarnpkg.com/@smithy/middleware-retry/-/middleware-retry-2.0.16.tgz#f87401a01317de351df5228e4591961d04663607" + integrity sha512-Br5+0yoiMS0ugiOAfJxregzMMGIRCbX4PYo1kDHtLgvkA/d++aHbnHB819m5zOIAMPvPE7AThZgcsoK+WOsUTA== + dependencies: + "@smithy/node-config-provider" "^2.1.1" + "@smithy/protocol-http" "^3.0.7" + "@smithy/service-error-classification" "^2.0.4" + "@smithy/types" "^2.3.5" + "@smithy/util-middleware" "^2.0.4" + "@smithy/util-retry" "^2.0.4" + tslib "^2.5.0" + uuid "^8.3.2" + +"@smithy/middleware-serde@^2.0.10", "@smithy/middleware-serde@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/middleware-serde/-/middleware-serde-2.0.11.tgz#89c4433b9b4077e2f71f436cd4f97d613e2cf3bd" + integrity sha512-NuxnjMyf4zQqhwwdh0OTj5RqpnuT6HcH5Xg5GrPijPcKzc2REXVEVK4Yyk8ckj8ez1XSj/bCmJ+oNjmqB02GWA== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/middleware-stack@^2.0.4", "@smithy/middleware-stack@^2.0.5": + version "2.0.5" + resolved "https://registry.yarnpkg.com/@smithy/middleware-stack/-/middleware-stack-2.0.5.tgz#43cd8aa7141b23dfbb64dff9ead8a3983d3acc5c" + integrity sha512-bVQU/rZzBY7CbSxIrDTGZYnBWKtIw+PL/cRc9B7etZk1IKSOe0NvKMJyWllfhfhrTeMF6eleCzOihIQympAvPw== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/node-config-provider@^2.0.13", "@smithy/node-config-provider@^2.1.1": + version "2.1.1" + resolved "https://registry.yarnpkg.com/@smithy/node-config-provider/-/node-config-provider-2.1.1.tgz#34c861b95a4e1b66a2dc1d1aecc2bca08466bd5e" + integrity sha512-1lF6s1YWBi1LBu2O30tD3jyTgMtuvk/Z1twzXM4GPYe4dmZix4nNREPJIPOcfFikNU2o0eTYP80+izx5F2jIJA== + dependencies: + "@smithy/property-provider" "^2.0.12" + "@smithy/shared-ini-file-loader" "^2.2.0" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/node-http-handler@^2.1.6", "@smithy/node-http-handler@^2.1.7": + version "2.1.7" + resolved "https://registry.yarnpkg.com/@smithy/node-http-handler/-/node-http-handler-2.1.7.tgz#a920e0e40fd04e2ea399cb4f06092fea0a1b66da" + integrity sha512-PQIKZXlp3awCDn/xNlCSTFE7aYG/5Tx33M05NfQmWYeB5yV1GZZOSz4dXpwiNJYTXb9jPqjl+ueXXkwtEluFFA== + dependencies: + "@smithy/abort-controller" "^2.0.11" + "@smithy/protocol-http" "^3.0.7" + "@smithy/querystring-builder" "^2.0.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/property-provider@^2.0.0", "@smithy/property-provider@^2.0.12": + version "2.0.12" + resolved "https://registry.yarnpkg.com/@smithy/property-provider/-/property-provider-2.0.12.tgz#09391cae6f336300e88128717ee5fb7cff76c5b4" + integrity sha512-Un/OvvuQ1Kg8WYtoMCicfsFFuHb/TKL3pCA6ZIo/WvNTJTR94RtoRnL7mY4XkkUAoFMyf6KjcQJ76y1FX7S5rw== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/protocol-http@^3.0.6", "@smithy/protocol-http@^3.0.7": + version "3.0.7" + resolved "https://registry.yarnpkg.com/@smithy/protocol-http/-/protocol-http-3.0.7.tgz#4deec17a27f7cc5d2bea962fcb0cdfbfd311b05c" + integrity sha512-HnZW8y+r66ntYueCDbLqKwWcMNWW8o3eVpSrHNluwtBJ/EUWfQHRKSiu6vZZtc6PGfPQWgVfucoCE/C3QufMAA== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/querystring-builder@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/querystring-builder/-/querystring-builder-2.0.11.tgz#7a56bed12474ad46059116d87eb7b81cdba9d7f6" + integrity sha512-b4kEbVMxpmfv2VWUITn2otckTi7GlMteZQxi+jlwedoATOGEyrCJPfRcYQJjbCi3fZ2QTfh3PcORvB27+j38Yg== + dependencies: + "@smithy/types" "^2.3.5" + "@smithy/util-uri-escape" "^2.0.0" + tslib "^2.5.0" + +"@smithy/querystring-parser@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/querystring-parser/-/querystring-parser-2.0.11.tgz#63b7fde68714974c220e386002100ad9b70d91a3" + integrity sha512-YXe7jhi7s3dQ0Fu9dLoY/gLu6NCyy8tBWJL/v2c9i7/RLpHgKT+uT96/OqZkHizCJ4kr0ZD46tzMjql/o60KLg== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/service-error-classification@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@smithy/service-error-classification/-/service-error-classification-2.0.4.tgz#05c0a30eddbf63fb5f27704757da388aec5d66c2" + integrity sha512-77506l12I5gxTZqBkx3Wb0RqMG81bMYLaVQ+EqIWFwQDJRs5UFeXogKxSKojCmz1wLUziHZQXm03MBzPQiumQw== + dependencies: + "@smithy/types" "^2.3.5" + +"@smithy/shared-ini-file-loader@^2.0.6", "@smithy/shared-ini-file-loader@^2.2.0": + version "2.2.0" + resolved "https://registry.yarnpkg.com/@smithy/shared-ini-file-loader/-/shared-ini-file-loader-2.2.0.tgz#9e4a90a29fe3f109875c26e6127802ed0953f43d" + integrity sha512-xFXqs4vAb5BdkzHSRrTapFoaqS4/3m/CGZzdw46fBjYZ0paYuLAoMY60ICCn1FfGirG+PiJ3eWcqJNe4/SkfyA== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/signature-v4@^2.0.0": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/signature-v4/-/signature-v4-2.0.11.tgz#e6d9065c7a73fc6f518f0cbc94039aed49307a1c" + integrity sha512-EFVU1dT+2s8xi227l1A9O27edT/GNKvyAK6lZnIZ0zhIHq/jSLznvkk15aonGAM1kmhmZBVGpI7Tt0odueZK9A== + dependencies: + "@smithy/eventstream-codec" "^2.0.11" + "@smithy/is-array-buffer" "^2.0.0" + "@smithy/types" "^2.3.5" + "@smithy/util-hex-encoding" "^2.0.0" + "@smithy/util-middleware" "^2.0.4" + "@smithy/util-uri-escape" "^2.0.0" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@smithy/smithy-client@^2.1.11", "@smithy/smithy-client@^2.1.9": + version "2.1.11" + resolved "https://registry.yarnpkg.com/@smithy/smithy-client/-/smithy-client-2.1.11.tgz#7e27c9969048952703ae493311245d1af62c73b8" + integrity sha512-okjMbuBBCTiieK665OFN/ap6u9+Z9z55PMphS5FYCsS6Zfp137Q3qlnt0OgBAnUVnH/mNGyoJV0LBX9gkTWptg== + dependencies: + "@smithy/middleware-stack" "^2.0.5" + "@smithy/types" "^2.3.5" + "@smithy/util-stream" "^2.0.16" + tslib "^2.5.0" + +"@smithy/types@^2.3.4", "@smithy/types@^2.3.5": + version "2.3.5" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-2.3.5.tgz#7684a74d4368f323b478bd9e99e7dc3a6156b5e5" + integrity sha512-ehyDt8M9hehyxrLQGoA1BGPou8Js1Ocoh5M0ngDhJMqbFmNK5N6Xhr9/ZExWkyIW8XcGkiMPq3ZUEE0ScrhbuQ== + dependencies: + tslib "^2.5.0" + +"@smithy/url-parser@^2.0.10", "@smithy/url-parser@^2.0.11": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/url-parser/-/url-parser-2.0.11.tgz#19c157f9d47217259e587847101ef6bd83091a5e" + integrity sha512-h89yXMCCF+S5k9XIoKltMIWTYj+FcEkU/IIFZ6RtE222fskOTL4Iak6ZRG+ehSvZDt8yKEcxqheTDq7JvvtK3g== + dependencies: + "@smithy/querystring-parser" "^2.0.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/util-base64@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-base64/-/util-base64-2.0.0.tgz#1beeabfb155471d1d41c8d0603be1351f883c444" + integrity sha512-Zb1E4xx+m5Lud8bbeYi5FkcMJMnn+1WUnJF3qD7rAdXpaL7UjkFQLdmW5fHadoKbdHpwH9vSR8EyTJFHJs++tA== + dependencies: + "@smithy/util-buffer-from" "^2.0.0" + tslib "^2.5.0" + +"@smithy/util-body-length-browser@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-body-length-browser/-/util-body-length-browser-2.0.0.tgz#5447853003b4c73da3bc5f3c5e82c21d592d1650" + integrity sha512-JdDuS4ircJt+FDnaQj88TzZY3+njZ6O+D3uakS32f2VNnDo3vyEuNdBOh/oFd8Df1zSZOuH1HEChk2AOYDezZg== + dependencies: + tslib "^2.5.0" + +"@smithy/util-body-length-node@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@smithy/util-body-length-node/-/util-body-length-node-2.1.0.tgz#313a5f7c5017947baf5fa018bfc22628904bbcfa" + integrity sha512-/li0/kj/y3fQ3vyzn36NTLGmUwAICb7Jbe/CsWCktW363gh1MOcpEcSO3mJ344Gv2dqz8YJCLQpb6hju/0qOWw== + dependencies: + tslib "^2.5.0" + +"@smithy/util-buffer-from@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-buffer-from/-/util-buffer-from-2.0.0.tgz#7eb75d72288b6b3001bc5f75b48b711513091deb" + integrity sha512-/YNnLoHsR+4W4Vf2wL5lGv0ksg8Bmk3GEGxn2vEQt52AQaPSCuaO5PM5VM7lP1K9qHRKHwrPGktqVoAHKWHxzw== + dependencies: + "@smithy/is-array-buffer" "^2.0.0" + tslib "^2.5.0" + +"@smithy/util-config-provider@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-config-provider/-/util-config-provider-2.0.0.tgz#4dd6a793605559d94267312fd06d0f58784b4c38" + integrity sha512-xCQ6UapcIWKxXHEU4Mcs2s7LcFQRiU3XEluM2WcCjjBtQkUN71Tb+ydGmJFPxMUrW/GWMgQEEGipLym4XG0jZg== + dependencies: + tslib "^2.5.0" + +"@smithy/util-defaults-mode-browser@^2.0.13": + version "2.0.15" + resolved "https://registry.yarnpkg.com/@smithy/util-defaults-mode-browser/-/util-defaults-mode-browser-2.0.15.tgz#0ab82d6e88dbebcca5e570678790a0160bd2619c" + integrity sha512-2raMZOYKSuke7QlDg/HDcxQdrp0zteJ8z+S0B9Rn23J55ZFNK1+IjG4HkN6vo/0u3Xy/JOdJ93ibiBSB8F7kOw== + dependencies: + "@smithy/property-provider" "^2.0.12" + "@smithy/smithy-client" "^2.1.11" + "@smithy/types" "^2.3.5" + bowser "^2.11.0" + tslib "^2.5.0" + +"@smithy/util-defaults-mode-node@^2.0.15": + version "2.0.19" + resolved "https://registry.yarnpkg.com/@smithy/util-defaults-mode-node/-/util-defaults-mode-node-2.0.19.tgz#8996479c76dd68baae65fd863180a802a66fdf5d" + integrity sha512-7pScU4jBFADB2MBYKM3zb5onMh6Nn0X3IfaFVLYPyCarTIZDLUtUl1GtruzEUJPmDzP+uGeqOtU589HDY0Ni6g== + dependencies: + "@smithy/config-resolver" "^2.0.14" + "@smithy/credential-provider-imds" "^2.0.16" + "@smithy/node-config-provider" "^2.1.1" + "@smithy/property-provider" "^2.0.12" + "@smithy/smithy-client" "^2.1.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/util-hex-encoding@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-hex-encoding/-/util-hex-encoding-2.0.0.tgz#0aa3515acd2b005c6d55675e377080a7c513b59e" + integrity sha512-c5xY+NUnFqG6d7HFh1IFfrm3mGl29lC+vF+geHv4ToiuJCBmIfzx6IeHLg+OgRdPFKDXIw6pvi+p3CsscaMcMA== + dependencies: + tslib "^2.5.0" + +"@smithy/util-middleware@^2.0.3", "@smithy/util-middleware@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@smithy/util-middleware/-/util-middleware-2.0.4.tgz#2c406efac04e341c3df6435d71fd9c73e03feb46" + integrity sha512-Pbu6P4MBwRcjrLgdTR1O4Y3c0sTZn2JdOiJNcgL7EcIStcQodj+6ZTXtbyU/WTEU3MV2NMA10LxFc3AWHZ3+4A== + dependencies: + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/util-retry@^2.0.3", "@smithy/util-retry@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@smithy/util-retry/-/util-retry-2.0.4.tgz#b3ae28e73b4bdec21480005e76f9eeb9d7279e89" + integrity sha512-b+n1jBBKc77C1E/zfBe1Zo7S9OXGBiGn55N0apfhZHxPUP/fMH5AhFUUcWaJh7NAnah284M5lGkBKuhnr3yK5w== + dependencies: + "@smithy/service-error-classification" "^2.0.4" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + +"@smithy/util-stream@^2.0.14", "@smithy/util-stream@^2.0.16": + version "2.0.16" + resolved "https://registry.yarnpkg.com/@smithy/util-stream/-/util-stream-2.0.16.tgz#8501e14cfcac70913d2c4c01a8cfbf7fc73bc041" + integrity sha512-b5ZSRh1KzUzC7LoJcpfk7+iXGoRr3WylEfmPd4FnBLm90OwxSB9VgK1fDZwicfYxSEvWHdYXgvvjPtenEYBBhw== + dependencies: + "@smithy/fetch-http-handler" "^2.2.3" + "@smithy/node-http-handler" "^2.1.7" + "@smithy/types" "^2.3.5" + "@smithy/util-base64" "^2.0.0" + "@smithy/util-buffer-from" "^2.0.0" + "@smithy/util-hex-encoding" "^2.0.0" + "@smithy/util-utf8" "^2.0.0" + tslib "^2.5.0" + +"@smithy/util-uri-escape@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-uri-escape/-/util-uri-escape-2.0.0.tgz#19955b1a0f517a87ae77ac729e0e411963dfda95" + integrity sha512-ebkxsqinSdEooQduuk9CbKcI+wheijxEb3utGXkCoYQkJnwTnLbH1JXGimJtUkQwNQbsbuYwG2+aFVyZf5TLaw== + dependencies: + tslib "^2.5.0" + +"@smithy/util-utf8@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@smithy/util-utf8/-/util-utf8-2.0.0.tgz#b4da87566ea7757435e153799df9da717262ad42" + integrity sha512-rctU1VkziY84n5OXe3bPNpKR001ZCME2JCaBBFgtiM2hfKbHFudc/BkMuPab8hRbLd0j3vbnBTTZ1igBf0wgiQ== + dependencies: + "@smithy/util-buffer-from" "^2.0.0" + tslib "^2.5.0" + +"@smithy/util-waiter@^2.0.10": + version "2.0.11" + resolved "https://registry.yarnpkg.com/@smithy/util-waiter/-/util-waiter-2.0.11.tgz#1cef055d557675bb187221b510cd666643dc207a" + integrity sha512-8SJWUl9O1YhjC77EccgltI3q4XZQp3vp9DGEW6o0OdkUcwqm/H4qOLnMkA2n+NDojuM5Iia2jWoCdbluIiG7TA== + dependencies: + "@smithy/abort-controller" "^2.0.11" + "@smithy/types" "^2.3.5" + tslib "^2.5.0" + "@socket.io/component-emitter@~3.1.0": version "3.1.0" resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" @@ -3765,6 +4781,21 @@ resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA== +"@types/multer-s3@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@types/multer-s3/-/multer-s3-3.0.1.tgz#b8886c63c26d8567d6d006c4aaaa729b8337ea0b" + integrity sha512-MwWRNbaWPnz3SEQfYlsBf2yp1c+/BTJm14mQYwWHjzIrRd9vYxFlzUtlW2g2sz3AMaV6UhxpOFiGQ92h56maaQ== + dependencies: + "@aws-sdk/client-s3" "^3.0.0" + "@types/multer" "*" + +"@types/multer@*": + version "1.4.8" + resolved "https://registry.yarnpkg.com/@types/multer/-/multer-1.4.8.tgz#8d98c36f6a4e0b228a9f262cd66e881d7cd64039" + integrity sha512-VMZOW6mnmMMhA5m3fsCdXBwFwC+a+27/8gctNMuQC4f7UtWcF79KAFGoIfKZ4iqrElgWIa3j5vhMJDp0iikQ1g== + dependencies: + "@types/express" "*" + "@types/multer@^1.4.7": version "1.4.7" resolved "https://registry.yarnpkg.com/@types/multer/-/multer-1.4.7.tgz#89cf03547c28c7bbcc726f029e2a76a7232cc79e" @@ -4756,6 +5787,11 @@ borc@^3.0.0: json-text-sequence "~0.3.0" readable-stream "^3.6.0" +bowser@^2.11.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/bowser/-/bowser-2.11.0.tgz#5ca3c35757a7aa5771500c70a73a9f91ef420a8f" + integrity sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA== + brace-expansion@^1.1.7: version "1.1.11" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" @@ -4828,6 +5864,14 @@ buffer@4.9.2: ieee754 "^1.1.4" isarray "^1.0.0" +buffer@5.6.0: + version "5.6.0" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" + integrity sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw== + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + buffer@^5.1.0, buffer@^5.5.0: version "5.7.1" resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" @@ -6492,7 +7536,7 @@ events@1.1.1: resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" integrity sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw== -events@^3.3.0: +events@3.3.0, events@^3.3.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== @@ -6697,6 +7741,13 @@ fast-write-atomic@0.2.1, fast-write-atomic@^0.2.0: resolved "https://registry.yarnpkg.com/fast-write-atomic/-/fast-write-atomic-0.2.1.tgz#7ee8ef0ce3c1f531043c09ae8e5143361ab17ede" integrity sha512-WvJe06IfNYlr+6cO3uQkdKdy3Cb1LlCJSF8zRs2eT8yuhdbSlR9nIt+TgQ92RUxiRrQm+/S7RARnMfCs5iuAjw== +fast-xml-parser@4.2.5: + version "4.2.5" + resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-4.2.5.tgz#a6747a09296a6cb34f2ae634019bf1738f3b421f" + integrity sha512-B9/wizE4WngqQftFPmdaMYlXoJlJOYxGQOanC77fq9k8+Z0v5dDSVh+3glErdIROP//s/jgb7ZuxKfB8nVyo0g== + dependencies: + strnum "^1.0.5" + fastq@^1.6.0: version "1.13.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" @@ -6742,6 +7793,11 @@ file-type@^17.1.6: strtok3 "^7.0.0-alpha.9" token-types "^5.0.0-alpha.2" +file-type@^3.3.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/file-type/-/file-type-3.9.0.tgz#257a078384d1db8087bc449d107d52a52672b9e9" + integrity sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA== + file-uri-to-path@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd" @@ -7530,6 +8586,11 @@ hosted-git-info@^4.0.1: dependencies: lru-cache "^6.0.0" +html-comment-regex@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/html-comment-regex/-/html-comment-regex-1.1.2.tgz#97d4688aeb5c81886a364faa0cad1dda14d433a7" + integrity sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ== + html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" @@ -7681,7 +8742,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3, inherits@~2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== @@ -10757,6 +11818,16 @@ ms@2.1.3, ms@^2.1.1, ms@^2.1.3: resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.3.tgz#574c8138ce1d2b5861f0b44579dbadd60c6615b2" integrity sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== +multer-s3@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/multer-s3/-/multer-s3-3.0.1.tgz#4ba7eb8c1eb94a7a64f0de37d414a88c4876d5d8" + integrity sha512-BFwSO80a5EW4GJRBdUuSHblz2jhVSAze33ZbnGpcfEicoT0iRolx4kWR+AJV07THFRCQ78g+kelKFdjkCCaXeQ== + dependencies: + "@aws-sdk/lib-storage" "^3.46.0" + file-type "^3.3.0" + html-comment-regex "^1.1.2" + run-parallel "^1.1.6" + multer@^1.4.5-lts.1: version "1.4.5-lts.1" resolved "https://registry.yarnpkg.com/multer/-/multer-1.4.5-lts.1.tgz#803e24ad1984f58edffbc79f56e305aec5cfd1ac" @@ -12293,6 +13364,15 @@ readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.1.0, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@^3.5.0: + version "3.6.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" + integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readable-stream@^4.0.0: version "4.4.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-4.4.0.tgz#55ce132d60a988c460d75c631e9ccf6a7229b468" @@ -12620,7 +13700,7 @@ run-parallel-limit@^1.1.0: dependencies: queue-microtask "^1.2.2" -run-parallel@^1.1.9: +run-parallel@^1.1.6, run-parallel@^1.1.9: version "1.2.0" resolved "https://registry.yarnpkg.com/run-parallel/-/run-parallel-1.2.0.tgz#66d1368da7bdf921eb9d95bd1a9229e7f21a43ee" integrity sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA== @@ -13141,6 +14221,14 @@ statuses@2.0.1: resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" integrity sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ== +stream-browserify@3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/stream-browserify/-/stream-browserify-3.0.0.tgz#22b0a2850cdf6503e73085da1fc7b7d0c2122f2f" + integrity sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA== + dependencies: + inherits "~2.0.4" + readable-stream "^3.5.0" + stream-to-it@^0.2.2: version "0.2.4" resolved "https://registry.yarnpkg.com/stream-to-it/-/stream-to-it-0.2.4.tgz#d2fd7bfbd4a899b4c0d6a7e6a533723af5749bd0" @@ -13368,6 +14456,11 @@ strip-outer@^2.0.0: resolved "https://registry.yarnpkg.com/strip-outer/-/strip-outer-2.0.0.tgz#c45c724ed9b1ff6be5f660503791404f4714084b" integrity sha512-A21Xsm1XzUkK0qK1ZrytDUvqsQWict2Cykhvi0fBQntGG5JSprESasEyV1EZ/4CiR5WB5KjzLTrP/bO37B0wPg== +strnum@^1.0.5: + version "1.0.5" + resolved "https://registry.yarnpkg.com/strnum/-/strnum-1.0.5.tgz#5c4e829fe15ad4ff0d20c3db5ac97b73c9b072db" + integrity sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA== + strtok3@^7.0.0-alpha.9: version "7.0.0" resolved "https://registry.yarnpkg.com/strtok3/-/strtok3-7.0.0.tgz#868c428b4ade64a8fd8fee7364256001c1a4cbe5" @@ -13861,7 +14954,7 @@ tsconfig@^7.0.0: strip-bom "^3.0.0" strip-json-comments "^2.0.0" -tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: +tslib@^1.11.1, tslib@^1.8.1, tslib@^1.9.0, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== @@ -13871,6 +14964,11 @@ tslib@^2, tslib@^2.1.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.4.0.tgz#7cecaa7f073ce680a05847aa77be941098f36dc3" integrity sha512-d6xOpEDfsi2CZVlPQzGeux8XMwLT9hssAsaPYExaQMuYskwb+x1x7J371tWlbBdWHroy99KnVB6qIkUbs5X3UQ== +tslib@^2.3.1, tslib@^2.5.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + tsutils@^3.21.0: version "3.21.0" resolved "https://registry.yarnpkg.com/tsutils/-/tsutils-3.21.0.tgz#b48717d394cea6c1e096983eed58e9d61715b623" From 0f3ed862ce658b3f087bc087c867378ac62a49fa Mon Sep 17 00:00:00 2001 From: kadami Date: Sat, 14 Oct 2023 03:56:51 +0700 Subject: [PATCH 2/8] big data uploads working, client->server->s3 basic setup --- .env.example | 5 +++++ .env.test | 5 +++++ desci-server/src/controllers/data/update.ts | 2 +- desci-server/src/routes/v1/data.ts | 20 ++++++++++++++++++-- desci-server/src/services/s3.ts | 12 ++++++++---- docker-compose.dev.yml | 1 + 6 files changed, 38 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index c0bdbf727..418c83687 100755 --- a/.env.example +++ b/.env.example @@ -59,6 +59,11 @@ AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= SENDGRID_API_KEY= +# S3 Bucket +AWS_S3_BUCKET_ARN= +AWS_S3_BUCKET_NAME= +AWS_S3_BUCKET_REGION= + # for faucet HOT_WALLET_KEY= # https://cso-classifier.internal diff --git a/.env.test b/.env.test index ea812134c..bf5effb0f 100644 --- a/.env.test +++ b/.env.test @@ -56,6 +56,11 @@ SHOULD_SEND_EMAIL= AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= +# S3 Bucket +AWS_S3_BUCKET_ARN= +AWS_S3_BUCKET_NAME= +AWS_S3_BUCKET_REGION= + # for faucet HOT_WALLET_KEY= # https://cso-classifier.internal diff --git a/desci-server/src/controllers/data/update.ts b/desci-server/src/controllers/data/update.ts index 155173c32..c16f81d34 100644 --- a/desci-server/src/controllers/data/update.ts +++ b/desci-server/src/controllers/data/update.ts @@ -263,7 +263,7 @@ export const update = async (req: Request, res: Response { - const fileStream = await fetchFileStreamFromS3(f.location); + const fileStream = await fetchFileStreamFromS3(f.key); return { path: f.originalname, content: fileStream }; }), ); diff --git a/desci-server/src/routes/v1/data.ts b/desci-server/src/routes/v1/data.ts index b01933a70..06d18ef0c 100644 --- a/desci-server/src/routes/v1/data.ts +++ b/desci-server/src/routes/v1/data.ts @@ -8,11 +8,27 @@ import { diffData } from 'controllers/data/diff'; import { moveData } from 'controllers/data/move'; import { updateExternalCid } from 'controllers/data/updateExternalCid'; import { ensureUser } from 'middleware/ensureUser'; +import { s3Client } from 'services/s3'; -export const s3 = new S3Client(); +// export const s3 = new S3Client(); const router = Router(); -const upload = multer({ preservePath: true, storage: multerS3({ s3, bucket: process.env.AWS_S3_BUCKET_NAME }) }); +const upload = multer({ + preservePath: true, + storage: multerS3({ + s3: s3Client, + bucket: process.env.AWS_S3_BUCKET_NAME, + key: (req, file, cb) => { + const userId = (req as any).user.id; + const { uuid, contextPath } = (req as any).body; + if (!uuid || !contextPath || !userId) { + cb(new Error('Missing required params to form key')); + } + const key = `${userId}*${uuid}*/upload${contextPath}${file.originalname}`; // adjust for dir uploads, doesn't start with '/' + cb(null, key); + }, + }), +}); router.post('/update', [ensureUser, upload.array('files')], update); router.post('/updateExternalCid', [ensureUser], updateExternalCid); diff --git a/desci-server/src/services/s3.ts b/desci-server/src/services/s3.ts index f63cc21a9..1f49de102 100644 --- a/desci-server/src/services/s3.ts +++ b/desci-server/src/services/s3.ts @@ -6,11 +6,15 @@ const logger = parentLogger.child({ module: 'Services::S3', }); -export const s3Client = new S3Client({ region: process.env.AWS_S3_REGION }); - -export async function fetchFileStreamFromS3(url: string): Promise { - const key = url.replace(`https://${process.env.AWS_S3_BUCKET_NAME}.s3.amazonaws.com/`, ''); +export const s3Client = new S3Client({ + region: process.env.AWS_S3_BUCKET_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, +}); +export async function fetchFileStreamFromS3(key: string): Promise { const params = { Bucket: process.env.AWS_S3_BUCKET_NAME, Key: key, diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 012a17aa9..44c1d7579 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -76,6 +76,7 @@ services: - db_postgres volumes: - ./local-data/yarn_cache:/root/.yarn + mem_limit: 2g block_explorer_dev: image: sinaiman/expedition-dev:latest From 07076c227493fe805ea9e7a90ee0cffe8a3ff5e3 Mon Sep 17 00:00:00 2001 From: kadami Date: Sat, 14 Oct 2023 03:58:01 +0700 Subject: [PATCH 3/8] comment out memory limit on backend container for testing --- docker-compose.dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 44c1d7579..5bf48f871 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -76,7 +76,7 @@ services: - db_postgres volumes: - ./local-data/yarn_cache:/root/.yarn - mem_limit: 2g + # mem_limit: 2g uncomment to test large data with limited memory block_explorer_dev: image: sinaiman/expedition-dev:latest From 8a4f8fe3e24b63d6fb7866d2d4891dc884c13f44 Mon Sep 17 00:00:00 2001 From: kadami Date: Thu, 19 Oct 2023 18:38:44 +0700 Subject: [PATCH 4/8] upload changes for prod --- .env.example | 5 ++--- desci-server/src/routes/v1/data.ts | 3 ++- docker-compose.dev.yml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env.example b/.env.example index 418c83687..ff68c2b89 100755 --- a/.env.example +++ b/.env.example @@ -55,14 +55,13 @@ THEGRAPH_API_URL=http://host.docker.internal:8000/subgraphs/name/nodes # set to true if need to send email SHOULD_SEND_EMAIL= -AWS_ACCESS_KEY_ID= -AWS_SECRET_ACCESS_KEY= SENDGRID_API_KEY= # S3 Bucket -AWS_S3_BUCKET_ARN= AWS_S3_BUCKET_NAME= AWS_S3_BUCKET_REGION= +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= # for faucet HOT_WALLET_KEY= diff --git a/desci-server/src/routes/v1/data.ts b/desci-server/src/routes/v1/data.ts index 06d18ef0c..37c866565 100644 --- a/desci-server/src/routes/v1/data.ts +++ b/desci-server/src/routes/v1/data.ts @@ -2,6 +2,7 @@ import { S3Client } from '@aws-sdk/client-s3'; import { Router } from 'express'; import multer = require('multer'); import multerS3 from 'multer-s3'; +import { v4 } from 'uuid'; import { pubTree, retrieveTree, deleteData, update, renameData } from 'controllers/data'; import { diffData } from 'controllers/data/diff'; @@ -24,7 +25,7 @@ const upload = multer({ if (!uuid || !contextPath || !userId) { cb(new Error('Missing required params to form key')); } - const key = `${userId}*${uuid}*/upload${contextPath}${file.originalname}`; // adjust for dir uploads, doesn't start with '/' + const key = `${userId}*${uuid}/${v4()}`; // adjust for dir uploads, doesn't start with '/' cb(null, key); }, }), diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5bf48f871..e79944a55 100755 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -76,7 +76,7 @@ services: - db_postgres volumes: - ./local-data/yarn_cache:/root/.yarn - # mem_limit: 2g uncomment to test large data with limited memory + # mem_limit: 2g #uncomment to test large data with limited memory block_explorer_dev: image: sinaiman/expedition-dev:latest From 3efc747000f89b8ff211563e3fc7238a28126618 Mon Sep 17 00:00:00 2001 From: kadami Date: Thu, 19 Oct 2023 18:46:56 +0700 Subject: [PATCH 5/8] add envs to kubernetes deployment configs --- desci-server/kubernetes/deployment.yaml | 2 ++ desci-server/kubernetes/deployment_demo.yaml | 2 ++ desci-server/kubernetes/deployment_dev.yaml | 2 ++ 3 files changed, 6 insertions(+) diff --git a/desci-server/kubernetes/deployment.yaml b/desci-server/kubernetes/deployment.yaml index cc182be1a..c8281bb37 100755 --- a/desci-server/kubernetes/deployment.yaml +++ b/desci-server/kubernetes/deployment.yaml @@ -56,6 +56,8 @@ spec: export SHOULD_SEND_EMAIL=true export AWS_ACCESS_KEY_ID={{ .Data.AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY={{ .Data.AWS_SECRET_ACCESS_KEY }} + export AWS_S3_BUCKET_NAME={{ .Data.AWS_S3_BUCKET_NAME }} + export AWS_S3_BUCKET_REGION={{ .Data.AWS_S3_BUCKET_REGION }} export THEGRAPH_API_URL={{ .Data.THEGRAPH_API_URL }} export HOT_WALLET_KEY={{ .Data.HOT_WALLET_KEY }} export CSO_CLASSIFIER_API={{ .Data.CSO_CLASSIFIER_API }} diff --git a/desci-server/kubernetes/deployment_demo.yaml b/desci-server/kubernetes/deployment_demo.yaml index 08eae94cc..854ae0b1b 100644 --- a/desci-server/kubernetes/deployment_demo.yaml +++ b/desci-server/kubernetes/deployment_demo.yaml @@ -56,6 +56,8 @@ spec: export SHOULD_SEND_EMAIL=true export AWS_ACCESS_KEY_ID={{ .Data.AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY={{ .Data.AWS_SECRET_ACCESS_KEY }} + export AWS_S3_BUCKET_NAME={{ .Data.AWS_S3_BUCKET_NAME }} + export AWS_S3_BUCKET_REGION={{ .Data.AWS_S3_BUCKET_REGION }} export THEGRAPH_API_URL={{ .Data.THEGRAPH_API_URL }} export HOT_WALLET_KEY={{ .Data.HOT_WALLET_KEY }} export CSO_CLASSIFIER_API={{ .Data.CSO_CLASSIFIER_API }} diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml index 1b9e7b656..1108167ec 100644 --- a/desci-server/kubernetes/deployment_dev.yaml +++ b/desci-server/kubernetes/deployment_dev.yaml @@ -56,6 +56,8 @@ spec: export SHOULD_SEND_EMAIL=true export AWS_ACCESS_KEY_ID={{ .Data.AWS_ACCESS_KEY_ID }} export AWS_SECRET_ACCESS_KEY={{ .Data.AWS_SECRET_ACCESS_KEY }} + export AWS_S3_BUCKET_NAME={{ .Data.AWS_S3_BUCKET_NAME }} + export AWS_S3_BUCKET_REGION={{ .Data.AWS_S3_BUCKET_REGION }} export THEGRAPH_API_URL={{ .Data.THEGRAPH_API_URL }} export HOT_WALLET_KEY={{ .Data.HOT_WALLET_KEY }} export CSO_CLASSIFIER_API={{ .Data.CSO_CLASSIFIER_API }} From e687235908d5a068edde27e726623f4015b78c8a Mon Sep 17 00:00:00 2001 From: kadami Date: Thu, 19 Oct 2023 19:16:13 +0700 Subject: [PATCH 6/8] make s3 optional for dev exp and tests --- desci-server/src/controllers/data/update.ts | 9 +++-- desci-server/src/routes/v1/data.ts | 40 ++++++++++----------- desci-server/src/services/s3.ts | 18 ++++++---- 3 files changed, 37 insertions(+), 30 deletions(-) diff --git a/desci-server/src/controllers/data/update.ts b/desci-server/src/controllers/data/update.ts index c16f81d34..cc4848c26 100644 --- a/desci-server/src/controllers/data/update.ts +++ b/desci-server/src/controllers/data/update.ts @@ -28,7 +28,7 @@ import { pinDirectory, RecursiveLsResult, } from 'services/ipfs'; -import { fetchFileStreamFromS3 } from 'services/s3'; +import { fetchFileStreamFromS3, isS3Configured } from 'services/s3'; import { arrayXor, calculateTotalZipUncompressedSize, @@ -263,8 +263,11 @@ export const update = async (req: Request, res: Response { - const fileStream = await fetchFileStreamFromS3(f.key); - return { path: f.originalname, content: fileStream }; + if (isS3Configured) { + const fileStream = await fetchFileStreamFromS3(f.key); + return { path: f.originalname, content: fileStream }; + } + return { path: f.originalname, content: f.buffer }; }), ); diff --git a/desci-server/src/routes/v1/data.ts b/desci-server/src/routes/v1/data.ts index 37c866565..6b50816b8 100644 --- a/desci-server/src/routes/v1/data.ts +++ b/desci-server/src/routes/v1/data.ts @@ -1,4 +1,3 @@ -import { S3Client } from '@aws-sdk/client-s3'; import { Router } from 'express'; import multer = require('multer'); import multerS3 from 'multer-s3'; @@ -9,27 +8,28 @@ import { diffData } from 'controllers/data/diff'; import { moveData } from 'controllers/data/move'; import { updateExternalCid } from 'controllers/data/updateExternalCid'; import { ensureUser } from 'middleware/ensureUser'; -import { s3Client } from 'services/s3'; - -// export const s3 = new S3Client(); +import { isS3Configured, s3Client } from 'services/s3'; const router = Router(); -const upload = multer({ - preservePath: true, - storage: multerS3({ - s3: s3Client, - bucket: process.env.AWS_S3_BUCKET_NAME, - key: (req, file, cb) => { - const userId = (req as any).user.id; - const { uuid, contextPath } = (req as any).body; - if (!uuid || !contextPath || !userId) { - cb(new Error('Missing required params to form key')); - } - const key = `${userId}*${uuid}/${v4()}`; // adjust for dir uploads, doesn't start with '/' - cb(null, key); - }, - }), -}); + +const upload = isS3Configured + ? multer({ + preservePath: true, + storage: multerS3({ + s3: s3Client, + bucket: process.env.AWS_S3_BUCKET_NAME, + key: (req, file, cb) => { + const userId = (req as any).user.id; + const { uuid, contextPath } = (req as any).body; + if (!uuid || !contextPath || !userId) { + cb(new Error('Missing required params to form key')); + } + const key = `${userId}*${uuid}/${v4()}`; // adjust for dir uploads, doesn't start with '/' + cb(null, key); + }, + }), + }) + : multer({ preservePath: true }); router.post('/update', [ensureUser, upload.array('files')], update); router.post('/updateExternalCid', [ensureUser], updateExternalCid); diff --git a/desci-server/src/services/s3.ts b/desci-server/src/services/s3.ts index 1f49de102..4773a7082 100644 --- a/desci-server/src/services/s3.ts +++ b/desci-server/src/services/s3.ts @@ -6,13 +6,17 @@ const logger = parentLogger.child({ module: 'Services::S3', }); -export const s3Client = new S3Client({ - region: process.env.AWS_S3_BUCKET_REGION, - credentials: { - accessKeyId: process.env.AWS_ACCESS_KEY_ID, - secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, - }, -}); +export const isS3Configured = process.env.AWS_S3_BUCKET_NAME && process.env.AWS_S3_BUCKET_REGION; + +export const s3Client = isS3Configured + ? new S3Client({ + region: process.env.AWS_S3_BUCKET_REGION, + credentials: { + accessKeyId: process.env.AWS_ACCESS_KEY_ID, + secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY, + }, + }) + : null; export async function fetchFileStreamFromS3(key: string): Promise { const params = { From 5cad91e837d1886aa8bc0f1dffaabee8825f16fc Mon Sep 17 00:00:00 2001 From: Sina Iman Date: Mon, 23 Oct 2023 11:27:09 +0200 Subject: [PATCH 7/8] orcid api updates --- .env.example | 6 +++++- desci-server/kubernetes/deployment.yaml | 1 + desci-server/kubernetes/deployment_dev.yaml | 1 + desci-server/src/controllers/auth/orcid.ts | 9 ++++++--- 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index ff68c2b89..dc64a9e7a 100755 --- a/.env.example +++ b/.env.example @@ -82,4 +82,8 @@ DISCORD_NOTIFICATIONS_WEBHOOK_URL= # for sending nodes cover cid to media servers # must match value in nodes-media/.env -MEDIA_SECRET_KEY=supersecret \ No newline at end of file +MEDIA_SECRET_KEY=supersecret + +ORCID_API_DOMAIN=https://api.sandbox.orcid.org +ORCID_CLIENT_ID= +ORCID_CLIENT_SECRET= \ No newline at end of file diff --git a/desci-server/kubernetes/deployment.yaml b/desci-server/kubernetes/deployment.yaml index c8281bb37..954df1077 100755 --- a/desci-server/kubernetes/deployment.yaml +++ b/desci-server/kubernetes/deployment.yaml @@ -39,6 +39,7 @@ spec: export JWT_SECRET={{ .Data.JWT_SECRET }} export JWT_EXPIRATION=15m export SESSION_KEY={{ .Data.SESSION_KEY }} + export ORCID_API_DOMAIN={{ .Data.ORCID_API_DOMAIN }} export ORCID_CLIENT_ID={{ .Data.ORCID_CLIENT_ID }} export ORCID_CLIENT_SECRET={{ .Data.ORCID_CLIENT_SECRET }} export ARWEAVE_ENABLED=0 diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml index 1108167ec..eba35c790 100644 --- a/desci-server/kubernetes/deployment_dev.yaml +++ b/desci-server/kubernetes/deployment_dev.yaml @@ -39,6 +39,7 @@ spec: export JWT_SECRET={{ .Data.JWT_SECRET }} export JWT_EXPIRATION=15m export SESSION_KEY={{ .Data.SESSION_KEY }} + export ORCID_API_DOMAIN={{ .Data.ORCID_API_DOMAIN }} export ORCID_CLIENT_ID={{ .Data.ORCID_CLIENT_ID }} export ORCID_CLIENT_SECRET={{ .Data.ORCID_CLIENT_SECRET }} export ARWEAVE_ENABLED=0 diff --git a/desci-server/src/controllers/auth/orcid.ts b/desci-server/src/controllers/auth/orcid.ts index 834b935be..686c893c5 100755 --- a/desci-server/src/controllers/auth/orcid.ts +++ b/desci-server/src/controllers/auth/orcid.ts @@ -29,7 +29,7 @@ export const orcidAuthClose = async (req: Request, res: Response) => { export const validateOrcid = async (req: Request, res: Response) => { // console.log('TOK', req.query.token); try { - const url = `https://pub.sandbox.orcid.org/v3.0/${req.query.orcid}/record`; + const url = `https://api.${process.env.ORCID_API_DOMAIN}/v3.0/${req.query.orcid}/record`; const { data } = await axios.get(url, { headers: { Authorization: `Bearer ${req.query.token}`, 'Content-Type': 'application/json', Accept: '*/*' }, }); @@ -57,9 +57,12 @@ export interface OrcIdRecordData { }; } export const getOrcidRecord = async (orcid: string, accessToken: string): Promise => { + /** + * this will fail if the orcid doesn't match the accessToken + */ const config: AxiosRequestConfig = { method: 'get', - url: `https://pub.sandbox.orcid.org/v3.0/${orcid}/record`, + url: `https://api.${process.env.ORCID_API_DOMAIN}/v3.0/${orcid}/record`, headers: { 'Content-Type': 'application/json', Accept: 'application/json', @@ -88,7 +91,7 @@ const getAllOrcData = async ({ queryCode, redirectUri }: { queryCode: string; re access_token: string; refresh_token: string; expires_in: number; - }>('https://sandbox.orcid.org/oauth/token', data, { + }>(`https://${process.env.ORCID_API_DOMAIN}/oauth/token`, data, { headers: { 'Content-Type': 'application/x-www-form-urlencoded', }, From b3648586dbc39ea658e81e2ea4f89b6e5ae490da Mon Sep 17 00:00:00 2001 From: Sina Iman Date: Mon, 23 Oct 2023 11:29:12 +0200 Subject: [PATCH 8/8] auto apply kubeconfig per env --- .github/workflows/build-server.yaml | 2 ++ desci-server/kubernetes/deployment.yaml | 4 ++-- desci-server/kubernetes/deployment_dev.yaml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-server.yaml b/.github/workflows/build-server.yaml index 3684c076e..52ccfec22 100644 --- a/.github/workflows/build-server.yaml +++ b/.github/workflows/build-server.yaml @@ -195,6 +195,7 @@ jobs: - name: Verify EKS Deployment (DEV) if: github.ref == 'refs/heads/develop' run: | + kubectl apply -f desci-server/kubernetes/deployment_dev.yaml kubectl rollout status deployment/desci-server-dev - name: Verify EKS Deployment (DEMO) @@ -205,4 +206,5 @@ jobs: - name: Verify EKS Deployment (PROD) if: github.ref == 'refs/heads/main' run: | + kubectl apply -f desci-server/kubernetes/deployment.yaml kubectl rollout status deployment/desci-server diff --git a/desci-server/kubernetes/deployment.yaml b/desci-server/kubernetes/deployment.yaml index 954df1077..df8fd12d0 100755 --- a/desci-server/kubernetes/deployment.yaml +++ b/desci-server/kubernetes/deployment.yaml @@ -5,8 +5,8 @@ metadata: labels: App: DesciServer spec: - replicas: 16 - revisionHistoryLimit: 2 + replicas: 12 + revisionHistoryLimit: 8 selector: matchLabels: App: DesciServer diff --git a/desci-server/kubernetes/deployment_dev.yaml b/desci-server/kubernetes/deployment_dev.yaml index eba35c790..8d87ff790 100644 --- a/desci-server/kubernetes/deployment_dev.yaml +++ b/desci-server/kubernetes/deployment_dev.yaml @@ -6,7 +6,7 @@ metadata: App: DesciServerDev spec: replicas: 6 - revisionHistoryLimit: 2 + revisionHistoryLimit: 8 selector: matchLabels: App: DesciServerDev