Merge pull request #257 from desci-labs/m0ar/media-service-fixes

nodes-media fixes

.github/workflows/check-media.yml

@@ -0,0 +1,29 @@
+on:
+  pull_request:
+    paths:
+      - .github/workflows/check-media.yml
+      - nodes-media/**
+jobs:
+  check-media:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: ".nvmrc"
+          check-latest: false
+          cache: "yarn"
+          cache-dependency-path: "nodes-media/yarn.lock"
+
+      - name: Install dependencies
+        run: |
+          cd nodes-media
+          yarn
+
+      - name: Build project
+        run: |
+          cd nodes-media
+          npx tsc --noEmit

desci-media-isolated/README.md

@@ -0,0 +1,3 @@
+# DeSci Media service
+
+Unpermissioned service responsible for generating thumbnails and other media transformations.

nodes-media/.estlintrc.js

@@ -1,50 +1,49 @@
 module.exports = {
-    parser: '@typescript-eslint/parser',
-
-    extends: [
-      'plugin:@typescript-eslint/recommended',
-      'prettier',
-      'plugin:prettier/recommended',
-      'plugin:no-array-reduce/recommended',
-    ],
-
-    plugins: ['@typescript-eslint', 'import'],
-
-    rules: {
-      // General
-      '@typescript-eslint/no-unused-vars': 0,
-      '@typescript-eslint/explicit-module-boundary-types': 0,
-      '@typescript-eslint/no-explicit-any': 0,
-      '@typescript-eslint/no-non-null-assertion': 0,
-      '@typescript-eslint/ban-ts-comment': 0,
-      '@typescript-eslint/no-empty-interface': 0,
-
-      // Import
-      'import/order': [
-        'error',
-        {
-          groups: ['builtin', 'external', 'internal', 'parent', 'sibling'],
-          'newlines-between': 'always',
-          alphabetize: {
-            order: 'asc',
-            caseInsensitive: true,
-          },
-        },
-      ],
-    },
-
-    parserOptions: {
-      ecmaVersion: 2018,
-      sourceType: 'module',
-    },
-
-    settings: {
-      'import/resolver': {
-        node: {
-          extensions: ['.js', '.jsx', '.ts', '.tsx'],
-          moduleDirectory: ['node_modules', 'src/'],
+  parser: '@typescript-eslint/parser',
+
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'prettier',
+    'plugin:prettier/recommended',
+    'plugin:no-array-reduce/recommended',
+  ],
+
+  plugins: ['@typescript-eslint', 'import'],
+
+  rules: {
+    // General
+    '@typescript-eslint/no-unused-vars': 0,
+    '@typescript-eslint/explicit-module-boundary-types': 0,
+    '@typescript-eslint/no-explicit-any': 0,
+    '@typescript-eslint/no-non-null-assertion': 0,
+    '@typescript-eslint/ban-ts-comment': 0,
+    '@typescript-eslint/no-empty-interface': 0,
+
+    // Import
+    'import/order': [
+      'error',
+      {
+        groups: ['builtin', 'external', 'internal', 'parent', 'sibling'],
+        'newlines-between': 'always',
+        alphabetize: {
+          order: 'asc',
+          caseInsensitive: true,
         },
       },
+    ],
+  },
+
+  parserOptions: {
+    ecmaVersion: 2018,
+    sourceType: 'module',
+  },
+
+  settings: {
+    'import/resolver': {
+      node: {
+        extensions: ['.js', '.jsx', '.ts', '.tsx'],
+        moduleDirectory: ['node_modules', 'src/'],
+      },
     },
-  };
-  
+  },
+};

nodes-media/kubernetes/deployment_dev.yaml

@@ -38,7 +38,7 @@ spec:
           name: nodes-media-server-dev
           command: ['/bin/bash', '-c']
           args:
-            - echo "SOURCING ENV"; source /vault/secrets/config; NODE_PATH=./dist node ./dist/index.js;
+            - echo "SOURCING ENV"; source /vault/secrets/config; node ./dist/index.js;
           ports:
             - containerPort: 5454
           resources:

nodes-media/kubernetes/deployment_prod.yaml

@@ -38,7 +38,7 @@ spec:
           name: nodes-media-server
           command: ['/bin/bash', '-c']
           args:
-            - echo "SOURCING ENV"; source /vault/secrets/config; NODE_PATH=./dist node ./dist/index.js;
+            - echo "SOURCING ENV"; source /vault/secrets/config; node ./dist/index.js;
           ports:
             - containerPort: 5454
           resources:

nodes-media/kubernetes/deployment_staging.yaml

@@ -38,7 +38,7 @@ spec:
           name: nodes-media-server-staging
           command: ['/bin/bash', '-c']
           args:
-            - echo "SOURCING ENV"; source /vault/secrets/config; NODE_PATH=./dist node ./dist/index.js;
+            - echo "SOURCING ENV"; source /vault/secrets/config; node ./dist/index.js;
           ports:
             - containerPort: 5454
           resources:

nodes-media/package.json

@@ -1,14 +1,12 @@
 {
   "name": "@desci-labs/nodes-media",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "Resolve open access data, transcode media, translate LaTeX to PDF",
-  "engines": {
-    "node": ">=16"
-  },
+  "type": "module",
   "scripts": {
-    "dev": "debug=* NODE_PATH=./src ts-node-dev --inspect=0.0.0.0:9230 --respawn --exit-child ./src/index.ts",
+    "dev": "tsx watch --inspect=0.0.0.0:9230 ./src/index.ts",
     "build": "rimraf dist && tsc",
-    "start": "NODE_PATH=./dist node ./dist/index.js",
+    "start": "node ./dist/index.js",
     "docker:dev": "docker-compose --file docker-compose.yml --file docker-compose.dev.yml --compatibility up --build",
     "prepare": "",
     "lint": "eslint --max-warnings 0 --ext .js,ts src",
@@ -22,32 +20,31 @@
     "@sentry/tracing": "^7.12.0",
     "aws-sdk": "^2.1186.0",
     "axios": "^1.3.5",
-    "cors": "^2.8.5",
+    "body-parser": "^1.20.2",
     "express": "^4.17.1",
     "express-fileupload": "^1.4.0",
     "imagemagick": "^0.1.3",
-    "jsonwebtoken": "^8.5.1",
     "kubo-rpc-client": "^3.0.4",
     "morgan": "^1.10.0",
     "node-latex": "^3.1.0",
-    "reflect-metadata": "^0.1.13"
+    "reflect-metadata": "^0.1.13",
+    "temp": "^0.9.4"
   },
   "devDependencies": {
     "@commitlint/cli": "^13.1.0",
     "@commitlint/config-conventional": "^13.1.0",
-    "@types/body-parser": "^1.19.1",
-    "@types/cors": "^2.8.12",
+    "@types/body-parser": "^1.19.5",
     "@types/express": "^4.17.13",
+    "@types/express-fileupload": "^1.5.0",
     "@types/imagemagick": "^0.0.35",
-    "@types/jsonwebtoken": "^8.5.4",
     "@types/morgan": "^1.9.3",
-    "@types/node": "^16.4.13",
+    "@types/node": "18.17.0",
+    "@types/temp": "^0.9.4",
     "@types/validator": "^13.6.3",
     "@typescript-eslint/eslint-plugin": "^4.29.1",
     "@typescript-eslint/parser": "^4.29.1",
     "commitizen": "^4.2.4",
     "dotenv": "^10.0.0",
-    "dotenv-cli": "^4.0.0",
     "eslint": "^7.32.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-import": "^2.24.0",
@@ -57,8 +54,7 @@
     "prettier": "^2.3.2",
     "pretty-quick": "^3.1.1",
     "rimraf": "^3.0.2",
-    "ts-node": "10.2.0",
-    "ts-node-dev": "^1.1.8",
+    "tsx": "^4.7.1",
     "typescript": "^5.0.0"
   },
   "lint-staged": {

nodes-media/src/controllers/latex/compile.ts

@@ -1,11 +1,11 @@
 import fs from 'fs';
 import { join } from 'path';
 
-import { Request, Response } from 'express';
+import type { Request, Response } from 'express';
 import temp from 'temp';
 
-import parser from './log-parser';
-import compileTex from './tex-compiler';
+import parser from './log-parser.js';
+import compileTex from './tex-compiler.js';
 const compile = function (req: Request, res: Response) {
   try {
     const buf = new Buffer(req.body.foo.toString('utf8'), 'base64');
@@ -19,7 +19,7 @@ const compile = function (req: Request, res: Response) {
 
     compileTex(path + '/' + name, 'pdflatex')
       .catch((error) => {
-        /** */
+        console.error(`tex compilation failed: ${JSON.stringify(error)}`);
       })
       .then(function (results) {
         const start = async () => {

nodes-media/src/controllers/latex/upload.ts

@@ -1,6 +1,6 @@
 import { join, resolve } from 'path';
 
-import { Request, Response } from 'express';
+import type { Request, Response } from 'express';
 import latex from 'node-latex';
 
 const upload = function (req: Request, res: Response) {
@@ -15,7 +15,7 @@ const upload = function (req: Request, res: Response) {
   const buf = new Buffer(req.body.foo.toString('utf8'), 'base64');
   const text = buf.toString();
 
-  const pdf = latex(text, options);
+  const pdf = latex.default(text, options);
 
   pdf.pipe(res);
   pdf.on('error', (err) => {

nodes-media/src/controllers/nodes/cover.ts

@@ -1,15 +1,15 @@
-import { Request, Response } from 'express';
-import { cleanupManifestUrl } from 'utils';
+import type { Request, Response } from 'express';
 // import { fromPath } from 'pdf2pic';
 import axios from 'axios';
 import { createWriteStream, existsSync, mkdirSync } from 'fs';
 import { promisify } from 'util';
 import * as stream from 'stream';
 import path from 'path';
 import { create } from 'kubo-rpc-client';
-import { PUBLIC_IPFS_PATH } from 'config';
 import { readFile } from 'fs/promises';
 import * as im from 'imagemagick';
+import { cleanupManifestUrl } from '../../utils.js';
+import { PUBLIC_IPFS_PATH } from '../../config/index.js';
 
 const client = create({ url: process.env.IPFS_NODE_URL });
 
@@ -24,26 +24,25 @@ if (!existsSync(TMP_DIR)) {
   mkdirSync(TMP_DIR);
 }
 
-
-console.log("TMPDIR", TMP_DIR);
+console.log('TMPDIR', TMP_DIR);
 
 const cover = async function (req: Request, res: Response) {
-  console.log("REQ", req.params, req.query)
+  console.log('REQ', req.params, req.query);
   try {
     const url = cleanupManifestUrl(req.params.cid, req.query?.g as string);
-    console.log("URL", url)
+    console.log('URL', url);
 
     const downloaded = await downloadFile(url, TMP_FILE);
 
     if (downloaded === false) {
-      console.log("cover not found",url);
+      console.log('cover not found', url);
       res.status(400).send({ ok: false, message: 'Cover not found' });
       return;
     }
 
-    console.log("starting convert", url)
-    await convertAsync([`${TMP_FILE}[0]`, '-quality', '100', TARGET_IMG])
-    console.log("done convert", url)
+    console.log('starting convert', url);
+    await convertAsync([`${TMP_FILE}[0]`, '-quality', '100', TARGET_IMG]);
+    console.log('done convert', url);
     const buffer = await readFile(TARGET_IMG);
     const storedCover = await client.add(buffer, { cidVersion: 1 });
 

nodes-media/src/index.ts

@@ -1,16 +1,19 @@
 import 'dotenv/config';
 import 'reflect-metadata';
 import fs from 'fs';
-import path from 'path';
+import path, { dirname } from 'path';
 
 import * as Sentry from '@sentry/node';
 import bodyParser from 'body-parser';
 import express from 'express';
 import fileupload from 'express-fileupload';
 import morgan from 'morgan';
 
-import routes from './routes';
+import routes from './routes/index.js';
+import { fileURLToPath } from 'url';
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
 export const app = express();
 
 const ENABLE_SENTRY = process.env.NODE_ENV === 'production';

nodes-media/src/middleware/ensureApiKey.ts

@@ -1,4 +1,4 @@
-import { Request, Response, NextFunction } from 'express';
+import type { Request, Response, NextFunction } from 'express';
 const MEDIA_SERVER_API_KEY = process.env.MEDIA_SECRET_KEY;
 
 export const ensureApiKey = async (req: Request, res: Response, next: NextFunction) => {

nodes-media/src/routes/index.ts

@@ -1,8 +1,8 @@
 import { Router } from 'express';
 
-import page404 from './pages/404';
-import pageRoot from './pages/root';
-import v1 from './v1/';
+import page404 from './pages/404.js';
+import pageRoot from './pages/root.js';
+import v1 from './v1/index.js';
 
 const router = Router();
 

nodes-media/src/routes/v1/index.ts

@@ -1,7 +1,7 @@
 import { Router } from 'express';
 
-import latex from './latex';
-import nodes from './nodes';
+import latex from './latex.js';
+import nodes from './nodes.js';
 
 const router = Router();
 

nodes-media/src/routes/v1/latex.ts

@@ -1,7 +1,7 @@
 import { Router } from 'express';
+import upload from '../../controllers/latex/upload.js';
+import compile from '../../controllers/latex/compile.js';
 
-import compile from 'controllers/latex/compile';
-import upload from 'controllers/latex/upload';
 const router = Router();
 
 router.post('/upload', upload);

nodes-media/src/routes/v1/nodes.ts

@@ -1,6 +1,6 @@
 import { Router } from 'express';
-import cover from 'controllers/nodes/cover';
-import { ensureApiKey } from 'middleware/ensureApiKey';
+import { ensureApiKey } from '../../middleware/ensureApiKey.js';
+import cover from '../../controllers/nodes/cover.js';
 
 const router = Router();
 

nodes-media/src/utils.ts

@@ -1,4 +1,4 @@
-import { PUBLIC_IPFS_PATH } from 'config';
+import { PUBLIC_IPFS_PATH } from './config/index.js';
 
 export const cleanupManifestUrl = (url: string, gateway?: string) => {
   if (url && (PUBLIC_IPFS_PATH || gateway)) {