Merge pull request #236 from desci-labs/gd-integration

Google APIs integration, import via gdrive

.env.example

@@ -107,6 +107,10 @@ IPFS_READ_ONLY_GATEWAY_SERVER=http://host.docker.internal:8089/ipfs # Used to pr
 # SET TO 1 to run communities seed script
 RUN=1
 
+# Enable google api functionalities
+GOOGLE_CLIENT_ID=
+GOOGLE_CLIENT_SECRET= # Unnecessary for now, not doing serverside 2step
+GOOGLE_DEV_API_KEY= # Unnecessary for now, not doing serverside 2step
 
 ## Configure RPC nodes (open an issue/ping us to access DeSci Labs' nodes)
 ETHEREUM_RPC_URL=http://host.docker.internal:8545
@@ -115,4 +119,4 @@ ETHEREUM_RPC_URL=http://host.docker.internal:8545
 # ETHEREUM_RPC_URL=https://eth-goerli.g.alchemy.com/v2/demo
 
 # Use this for Sepolia testnet
-# ETHEREUM_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/demo
+# ETHEREUM_RPC_URL=https://eth-sepolia.g.alchemy.com/v2/demo

desci-server/kubernetes/deployment.yaml

@@ -77,6 +77,7 @@ spec:
           export ISOLATED_MEDIA_SERVER_URL={{ .Data.ISOLATED_MEDIA_SERVER_URL }}
           export IPFS_READ_ONLY_GATEWAY_SERVER_URL={{ .Data.IPFS_READ_ONLY_GATEWAY_SERVER_URL }}
           export ETHEREUM_RPC_URL={{ .Data.ETHEREUM_RPC_URL }}
+          export GOOGLE_CLIENT_ID={{ .Data.GOOGLE_CLIENT_ID }}
           export DEBUG_TEST=0;
           echo "appfinish";
           {{- end -}}

desci-server/kubernetes/deployment_dev.yaml

@@ -77,6 +77,7 @@ spec:
           export ISOLATED_MEDIA_SERVER_URL={{ .Data.ISOLATED_MEDIA_SERVER_URL }}
           export IPFS_READ_ONLY_GATEWAY_SERVER_URL={{ .Data.IPFS_READ_ONLY_GATEWAY_SERVER_URL }}
           export ETHEREUM_RPC_URL={{ .Data.ETHEREUM_RPC_URL }}
+          export GOOGLE_CLIENT_ID={{ .Data.GOOGLE_CLIENT_ID }}
           export DEBUG_TEST=0;
           echo "appfinish";
           {{- end -}}

desci-server/package.json

@@ -81,6 +81,7 @@
     "express": "^4.17.1",
     "form-data": "^4.0.0",
     "google-protobuf": "^3.20.0-rc.2",
+    "googleapis": "^133.0.0",
     "grpc": "^1.24.11",
     "helmet": "^4.6.0",
     "http-proxy-middleware": "3.0.0-beta.0",

desci-server/src/controllers/data/google/import.ts

@@ -0,0 +1,67 @@
+import { Request, Response } from 'express';
+
+import { logger as parentLogger } from '../../../logger.js';
+import { processS3DataToIpfs } from '../../../services/data/processing.js';
+import { GoogleApiService } from '../../../services/googleApiService.js';
+import { ErrorResponse, UpdateResponse } from '../update.js';
+
+interface GoogleImportReqBody {
+  uuid: string;
+  contextPath: string;
+  googleFileId: string;
+  gAuthAccessToken: string; // We can change this to use the oauth backend flow in the future
+}
+
+export const googleImport = async (
+  req: Request<any, any, GoogleImportReqBody>,
+  res: Response<UpdateResponse | ErrorResponse>,
+) => {
+  const owner = (req as any).user;
+  const node = (req as any).node;
+
+  const { uuid, contextPath, googleFileId, gAuthAccessToken } = req.body;
+  if (contextPath === undefined) return res.status(400).json({ error: 'contextPath is required' });
+  if (googleFileId === undefined) return res.status(400).json({ error: 'googleFileId is required' });
+  if (gAuthAccessToken === undefined) return res.status(400).json({ error: 'gAuthAccessToken is required' });
+
+  const logger = parentLogger.child({
+    module: 'DATA::GoogleImportController',
+    uuid: uuid,
+    user: owner.id,
+    contextPath: contextPath,
+    googleFileId,
+  });
+  const googleService = new GoogleApiService(gAuthAccessToken);
+  // googleService.exchangeCodeForToken(gAuthAccessToken);
+  const fileMd = await googleService.getFileMetadata(googleFileId);
+  const fileStream = await googleService.getFileStream(googleFileId);
+  // debugger;
+  const files = [{ originalname: '/' + fileMd.name, content: fileStream, size: fileMd.size }];
+  const { ok, value } = await processS3DataToIpfs({
+    files,
+    user: owner,
+    node,
+    contextPath,
+  });
+  if (ok) {
+    const {
+      rootDataCid: newRootCidString,
+      manifest: updatedManifest,
+      manifestCid: persistedManifestCid,
+      tree: tree,
+      date: date,
+    } = value as UpdateResponse;
+    return res.status(200).json({
+      rootDataCid: newRootCidString,
+      manifest: updatedManifest,
+      manifestCid: persistedManifestCid,
+      tree: tree,
+      date: date,
+    });
+  } else {
+    if (!('message' in value)) return res.status(500);
+    logger.error({ value }, 'processing error occured');
+    return res.status(value.status).json({ status: value.status, error: value.message });
+  }
+  // return res.status(400);
+};

desci-server/src/routes/v1/data.ts

@@ -4,6 +4,7 @@ import multerS3 from 'multer-s3';
 import { v4 } from 'uuid';
 
 import { diffData } from '../../controllers/data/diff.js';
+import { googleImport } from '../../controllers/data/google/import.js';
 import { pubTree, retrieveTree, deleteData, update, renameData } from '../../controllers/data/index.js';
 import { moveData } from '../../controllers/data/move.js';
 import { updateExternalCid } from '../../controllers/data/updateExternalCid.js';
@@ -75,10 +76,8 @@ router.post('/move', [ensureUser, ensureNodeAccess], moveData);
 router.get('/retrieveTree/:nodeUuid/:manifestCid', [ensureUser], retrieveTree);
 router.get('/retrieveTree/:nodeUuid/:manifestCid/:shareId?', retrieveTree);
 router.get('/pubTree/:nodeUuid/:manifestCid/:rootCid?', pubTree);
-// router.get('/downloadDataset/:nodeUuid/:cid', [ensureUser], downloadDataset);
 router.get('/diff/:nodeUuid/:manifestCidA/:manifestCidB?', [attachUser], diffData);
 
-// must be last
-// router.get('/*', [ensureUser], list);
+router.post('/google/import', [ensureUser, ensureNodeAccess], googleImport);
 
 export default router;

desci-server/src/services/googleApiService.ts

@@ -0,0 +1,69 @@
+import { Readable } from 'stream';
+
+import { GaxiosResponse } from 'gaxios';
+import { google, drive_v3 } from 'googleapis';
+
+import { logger as parentLogger } from '../logger.js';
+
+export class GoogleApiService {
+  private oauth2Client;
+  private driveClient: drive_v3.Drive;
+  private logger;
+
+  constructor(accessToken: string) {
+    this.oauth2Client = new google.auth.OAuth2({
+      clientId: process.env.GOOGLE_CLIENT_ID,
+      // clientSecret: process.env.GOOGLE_CLIENT_SECRET, Unnecessary unless we switch to 2step server-side OAuth flow
+    });
+    this.oauth2Client.setCredentials({ access_token: accessToken });
+    this.driveClient = google.drive({ version: 'v3', auth: this.oauth2Client });
+    this.logger = parentLogger.child({ module: 'Services::GoogleApiService' });
+  }
+
+  async getFileMetadata(docId: string): Promise<drive_v3.Schema$File> {
+    try {
+      const fileMetadata = await this.driveClient.files.get({ fileId: docId, fields: 'id, name, mimeType, size' });
+
+      return fileMetadata.data;
+    } catch (error) {
+      this.logger.error({ docId, error }, 'Failed to get file metadata');
+      throw error;
+    }
+  }
+
+  async getFileStream(docId: string): Promise<Readable> {
+    try {
+      const response = await this.driveClient.files.get({ fileId: docId, alt: 'media' }, { responseType: 'stream' });
+
+      return response.data;
+    } catch (error) {
+      this.logger.error({ docId, error }, 'Failed to get file stream');
+      throw error;
+    }
+  }
+
+  async authenticateWithAccessToken(accessToken: string): Promise<void> {
+    try {
+      this.oauth2Client.setCredentials({ access_token: accessToken });
+      this.logger.info('Successfully authenticated with access token');
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to authenticate with access token');
+      throw error;
+    }
+  }
+
+  /**
+   * Can be used later if we switch to 2-step, server-side OAuth flow, useful if we need >60 minutes of access.
+   */
+  async exchangeCodeForToken(code: string): Promise<void> {
+    try {
+      const { tokens } = await this.oauth2Client.getToken(code);
+      debugger;
+      this.oauth2Client.setCredentials(tokens);
+      this.logger.info('Successfully exchanged code for tokens');
+    } catch (error) {
+      this.logger.error({ error }, 'Failed to exchange code for tokens');
+      throw error;
+    }
+  }
+}