From eb03f9ea2cc72fbeec8d41c2de29c1df5e1bcee0 Mon Sep 17 00:00:00 2001 From: Sajera Date: Wed, 14 Sep 2022 17:45:43 +0300 Subject: [PATCH] feature/URL -- allow more urls including non English characters -- RENDER_FALLBACK to allow disabling fall back from render to refresh --- .env | 1 + docker-compose.yaml | 1 + package-lock.json | 13 +------------ package.json | 3 +-- src/config.js | 4 +++- src/index.js | 18 +++++++++++------- 6 files changed, 18 insertions(+), 22 deletions(-) diff --git a/.env b/.env index 388e8e9..30ce87c 100644 --- a/.env +++ b/.env @@ -4,6 +4,7 @@ DEBUG=true # API local configuration PORT=3000 HOST=localhost +RENDER_FALLBACK=true # Redis local configuration REDIS_URL=redis://127.0.0.1:6379 diff --git a/docker-compose.yaml b/docker-compose.yaml index f5ee463..8405184 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,6 +11,7 @@ services: DEBUG: 1 PORT: 3000 HOST: 0.0.0.0 + RENDER_FALLBACK: 0 REDIS_URL: redis://cache:6379/0 CHROME_FORWARD_HEADERS: 1 CHROME_DEBUGGING_PORT: 9222 diff --git a/package-lock.json b/package-lock.json index ea9c2d6..a803c05 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,8 +12,7 @@ "chrome-remote-interface": "^0.31.3", "dotenv": "^16.0.1", "redis": "^4.2.0", - "uuid": "^8.3.2", - "valid-url": "^1.0.9" + "uuid": "^8.3.2" }, "devDependencies": { "nodemon": "^2.0.19" @@ -504,11 +503,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/valid-url": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", - "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA==" - }, "node_modules/ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", @@ -893,11 +887,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" }, - "valid-url": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/valid-url/-/valid-url-1.0.9.tgz", - "integrity": "sha512-QQDsV8OnSf5Uc30CKSwG9lnhMPe6exHtTXLRYX8uMwKENy640pU+2BgBL0LRbDh/eYRahNCS7aewCx0wf3NYVA==" - }, "ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", diff --git a/package.json b/package.json index e8e6181..d14f08f 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,7 @@ "chrome-remote-interface": "^0.31.3", "dotenv": "^16.0.1", "redis": "^4.2.0", - "uuid": "^8.3.2", - "valid-url": "^1.0.9" + "uuid": "^8.3.2" }, "devDependencies": { "nodemon": "^2.0.19" diff --git a/src/config.js b/src/config.js index bc87636..9bc88de 100644 --- a/src/config.js +++ b/src/config.js @@ -10,6 +10,7 @@ export const DEBUG = varBoolean(process.env.DEBUG); export const API = { port: varNumber(process.env.PORT) || 80, host: varString(process.env.HOST) || '0.0.0.0', + renderFallback: varBoolean(process.env.RENDER_FALLBACK), }; // NOTE for now Redis only export const CACHE = { @@ -59,7 +60,6 @@ export function defaultCleanupHtmlScript () { } })(['noscript', 'script', 'style'])`; } - /****************************************************** * ¯\(ヅ)/¯ helpers ᕦ(ツ)ᕤ *****************************************************/ @@ -73,3 +73,5 @@ const logWithTime = (text, data) => console.log( text, data === undefined ? '' : DEBUG ? JSON.stringify(data, null, 4) : JSON.stringify(data), ); +const urlRegExp = /(?:https?):\/\/(\w+:?\w*)?(\S+)(:\d+)?(\/|\/([\w#!:.?+=&%!\-\/]))?/; +export const isUrl = url => urlRegExp.test(url); diff --git a/src/index.js b/src/index.js index c744d89..0809eeb 100644 --- a/src/index.js +++ b/src/index.js @@ -1,13 +1,12 @@ // outsource dependencies import qs from 'node:querystring'; -import { isWebUri } from 'valid-url'; // local dependencies import api from './api/index.js'; import cache from './cache/index.js'; import prerender from './prerender/index.js'; -import { logError, log, API, CACHE, PRERENDER } from './config.js'; +import { isUrl, logError, log, API, CACHE, PRERENDER } from './config.js'; // NOTE log unhandled promise exception process.on('unhandledRejection', error => logError('[service:unhandledRejection]', error && { @@ -43,19 +42,24 @@ render.contentType = 'text/html'; async function render (request) { if (!cache.isReady()) { throw { code: 503, message: 'Service not ready yet' }; } const url = qs.unescape(qs.parse(request.url.query).url); - if (!isWebUri(url)) { throw { code: 400, message: `Invalid query parameter url "${url}"` }; } + if (!isUrl(url)) { throw { code: 400, message: `Invalid query parameter url "${url}"` }; } const results = await cache.get(url); - results && log('[api:cache]', url); - return results || refresh(request); + if (results) { + log('[api:cache]', url); + return results; + } + if (API.renderFallback) { + return refresh(request); + } + throw { code: 404, message: `Cache empty for "${url}"` }; } api.middleware['/refresh'] = refresh; refresh.contentType = 'text/html'; async function refresh (request) { - console.log(request.url.query); if (!prerender.isReady() || !cache.isReady()) { throw { code: 503, message: 'Service not ready yet' }; } const url = qs.unescape(qs.parse(request.url.query).url); - if (!isWebUri(url)) { throw { code: 400, message: `Invalid query parameter url "${url}"` }; } + if (!isUrl(url)) { throw { code: 400, message: `Invalid query parameter url "${url}"` }; } const results = await prerender.render(url); log('[api:generate]', url); await cache.set(url, results);