make s3 optional for dev exp and tests

desci-labs · Oct 19, 2023 · e687235 · e687235
1 parent 3efc747
commit e687235
Show file tree

Hide file tree

Showing 3 changed files with 37 additions and 30 deletions.
diff --git a/desci-server/src/controllers/data/update.ts b/desci-server/src/controllers/data/update.ts
@@ -28,7 +28,7 @@ import {
   pinDirectory,
   RecursiveLsResult,
 } from 'services/ipfs';
-import { fetchFileStreamFromS3 } from 'services/s3';
+import { fetchFileStreamFromS3, isS3Configured } from 'services/s3';
 import {
   arrayXor,
   calculateTotalZipUncompressedSize,
@@ -263,8 +263,11 @@ export const update = async (req: Request, res: Response<UpdateResponse | ErrorR
   //Pin the new files
   const structuredFilesForPinning: IpfsDirStructuredInput[] = await Promise.all(
     files.map(async (f: any) => {
-      const fileStream = await fetchFileStreamFromS3(f.key);
-      return { path: f.originalname, content: fileStream };
+      if (isS3Configured) {
+        const fileStream = await fetchFileStreamFromS3(f.key);
+        return { path: f.originalname, content: fileStream };
+      }
+      return { path: f.originalname, content: f.buffer };
     }),
   );
 

diff --git a/desci-server/src/routes/v1/data.ts b/desci-server/src/routes/v1/data.ts
@@ -1,4 +1,3 @@
-import { S3Client } from '@aws-sdk/client-s3';
 import { Router } from 'express';
 import multer = require('multer');
 import multerS3 from 'multer-s3';
@@ -9,27 +8,28 @@ import { diffData } from 'controllers/data/diff';
 import { moveData } from 'controllers/data/move';
 import { updateExternalCid } from 'controllers/data/updateExternalCid';
 import { ensureUser } from 'middleware/ensureUser';
-import { s3Client } from 'services/s3';
-
-// export const s3 = new S3Client();
+import { isS3Configured, s3Client } from 'services/s3';
 
 const router = Router();
-const upload = multer({
-  preservePath: true,
-  storage: multerS3({
-    s3: s3Client,
-    bucket: process.env.AWS_S3_BUCKET_NAME,
-    key: (req, file, cb) => {
-      const userId = (req as any).user.id;
-      const { uuid, contextPath } = (req as any).body;
-      if (!uuid || !contextPath || !userId) {
-        cb(new Error('Missing required params to form key'));
-      }
-      const key = `${userId}*${uuid}/${v4()}`; // adjust for dir uploads, doesn't start with '/'
-      cb(null, key);
-    },
-  }),
-});
+
+const upload = isS3Configured
+  ? multer({
+      preservePath: true,
+      storage: multerS3({
+        s3: s3Client,
+        bucket: process.env.AWS_S3_BUCKET_NAME,
+        key: (req, file, cb) => {
+          const userId = (req as any).user.id;
+          const { uuid, contextPath } = (req as any).body;
+          if (!uuid || !contextPath || !userId) {
+            cb(new Error('Missing required params to form key'));
+          }
+          const key = `${userId}*${uuid}/${v4()}`; // adjust for dir uploads, doesn't start with '/'
+          cb(null, key);
+        },
+      }),
+    })
+  : multer({ preservePath: true });
 
 router.post('/update', [ensureUser, upload.array('files')], update);
 router.post('/updateExternalCid', [ensureUser], updateExternalCid);

diff --git a/desci-server/src/services/s3.ts b/desci-server/src/services/s3.ts
@@ -6,13 +6,17 @@ const logger = parentLogger.child({
   module: 'Services::S3',
 });
 
-export const s3Client = new S3Client({
-  region: process.env.AWS_S3_BUCKET_REGION,
-  credentials: {
-    accessKeyId: process.env.AWS_ACCESS_KEY_ID,
-    secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
-  },
-});
+export const isS3Configured = process.env.AWS_S3_BUCKET_NAME && process.env.AWS_S3_BUCKET_REGION;
+
+export const s3Client = isS3Configured
+  ? new S3Client({
+      region: process.env.AWS_S3_BUCKET_REGION,
+      credentials: {
+        accessKeyId: process.env.AWS_ACCESS_KEY_ID,
+        secretAccessKey: process.env.AWS_SECRET_ACCESS_KEY,
+      },
+    })
+  : null;
 
 export async function fetchFileStreamFromS3(key: string): Promise<ReadableStream | null> {
   const params = {