From 2148f2747a31e921c6dc79971ede49e5a6f65cd2 Mon Sep 17 00:00:00 2001 From: Sajera Date: Fri, 23 Sep 2022 01:34:31 +0300 Subject: [PATCH 01/10] feature/RELEASE-1.0.9 -- split local and docker environments --- .env | 2 +- Dockerfile | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index c3443fb..1c51821 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ DEBUG=true # API local configuration PORT=3636 -HOST=0.0.0.0 +HOST=localhost ALLOW_DOMAINS=. # Redis local configuration diff --git a/Dockerfile b/Dockerfile index b04d4f1..ea39dc4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ RUN apk add --update-cache chromium && \ rm -rf /var/cache/apk/* /tmp/* ENV DEBUG=false \ + HOST=0.0.0.0 \ CHROME_DEBUGGING_PORT=9222 \ CHROME_BIN=/usr/bin/chromium-browser \ CHROME_FLAGS="--no-sandbox,--headless,--disable-gpu,--remote-debugging-port=9222,--hide-scrollbars,--disable-dev-shm-usage" From 06938188ba97e5163e87ad80a0eed8abc2f4dad7 Mon Sep 17 00:00:00 2001 From: Sajera Date: Fri, 23 Sep 2022 01:41:54 +0300 Subject: [PATCH 02/10] feature/RELEASE-1.0.9 -- roadmap --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index e380557..399cc40 100644 --- a/README.md +++ b/README.md @@ -68,3 +68,17 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - **DELETE /cached** - `curl -X 'DELETE' 'http://localhost:3636/cached?url=http://example.com/'` +# TODO +- [x] Render SPA page to get HTML +- [x] Cache HTML +- [x] Refresh cached HTML +- [x] Cache unlimited but controlled via API +- [x] Health status +- [x] Base environment +- [x] Docker image +- [x] Docker for local development +- [x] Domain limitation +- [ ] Queue for rendering +- [ ] Accumulate Sitemap +- [ ] Different cache technology +- [ ] Different queue technology From 39f630d4cb852267b9e1c0a1013249826b1998a0 Mon Sep 17 00:00:00 2001 From: Sajera Date: Fri, 23 Sep 2022 01:49:51 +0300 Subject: [PATCH 03/10] feature/RELEASE-1.0.9 -- update doc -- package.json version 1.0.9 --- README.md | 6 ++---- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 399cc40..52aea78 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ Before start development locally, please make sure you have installed [Chrome br `npm run start:dev` -> NOTE Starting Redis on MAC using Brew `brew services start redis` ## Development using [Docker](https://www.docker.com/) Please, take in mind, that the `Dockerfile` isn't for local usage. It exists to simplify inserting the app into complex infrastructures. @@ -41,7 +40,6 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - `HOST=0.0.0.0` - `ALLOW_DOMAINS=.` - `CHROME_DEBUGGING_PORT=9222` -- `CHROME_FORWARD_HEADERS=true` - `CHROME_BIN=/usr/bin/chromium-browser` - `CHROME_FLAGS=--no-sandbox,--headless,--disable-gpu,--remote-debugging-port=9222,--hide-scrollbars,--disable-dev-shm-usage` @@ -52,11 +50,11 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - `curl 'http://localhost:3636/health'` - `{ status: "UP" | "DOWN" }` -- Will render URL in `Chrome` browser then return `HTML` only the first time. After providing `HTML` content from the cache. +- Will render URL in `Chromium` browser then return `HTML` only the first time. After that, provide `HTML` content from the cache. - **GET /render** - `curl 'http://localhost:3636/render?url=http://example.com/'` -- Force reset cache and render url in `Chrome` browser then return `html`. +- Force reset the cache and render URL in `Chromium` browser, returns `HTML`. - **GET /refresh** - `curl 'http://localhost:3636/refresh?url=http://example.com/'` diff --git a/package-lock.json b/package-lock.json index ebad10b..578fd7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "s-prerender", - "version": "1.0.8", + "version": "1.0.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "s-prerender", - "version": "1.0.8", + "version": "1.0.9", "license": "MIT", "dependencies": { "chrome-remote-interface": "^0.31.3", diff --git a/package.json b/package.json index 59aa214..242b923 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s-prerender", - "version": "1.0.8", + "version": "1.0.9", "type": "module", "description": "s-prerender is the Service to solve SPA problems with SEO", "main": "src/index.js", From dc1e93ff824bdf75951b9af49a6acb464499a87b Mon Sep 17 00:00:00 2001 From: Sajera Date: Sat, 24 Sep 2022 14:21:56 +0300 Subject: [PATCH 04/10] feature/RELEASE-1.0.9 -- process point -- more logs -- queue interface -- amqp connect --- .env | 3 + package-lock.json | 205 ++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/api/index.js | 2 +- src/cache/index.js | 16 +++- src/cache/redis.js | 8 +- src/config.js | 31 +++--- src/index.js | 54 +++++------ src/prerender/chrome.js | 2 +- src/prerender/index.js | 10 +- src/queue/index.js | 19 ++++ src/queue/rabbitmq.js | 35 +++++++ 12 files changed, 331 insertions(+), 55 deletions(-) create mode 100644 src/queue/index.js create mode 100644 src/queue/rabbitmq.js diff --git a/.env b/.env index 1c51821..a831469 100644 --- a/.env +++ b/.env @@ -9,6 +9,9 @@ ALLOW_DOMAINS=. # Redis local configuration REDIS_URL=redis://127.0.0.1:6379 +# Redis local configuration +RABBITMQ_URL=amqp://127.0.0.1:5672 + # Prerender local configuration CHROME_DEBUGGING_PORT=9222 #CHROME_BIN ===== for local enw will take your Chrome automatically diff --git a/package-lock.json b/package-lock.json index 578fd7d..5772b21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.9", "license": "MIT", "dependencies": { + "amqplib": "^0.10.3", "chrome-remote-interface": "^0.31.3", "dotenv": "^16.0.1", "redis": "^4.2.0" @@ -21,6 +22,40 @@ "npm": ">=7" } }, + "node_modules/@acuminous/bitsyntax": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz", + "integrity": "sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==", + "dependencies": { + "buffer-more-ints": "~1.0.0", + "debug": "^4.3.4", + "safe-buffer": "~5.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/@acuminous/bitsyntax/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@acuminous/bitsyntax/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/@redis/bloom": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", @@ -80,6 +115,20 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "node_modules/amqplib": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", + "integrity": "sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw==", + "dependencies": { + "@acuminous/bitsyntax": "^0.1.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "url-parse": "~1.5.10" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -130,6 +179,11 @@ "node": ">=8" } }, + "node_modules/buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -188,6 +242,11 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, "node_modules/debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -266,6 +325,11 @@ "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", "dev": true }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -308,6 +372,11 @@ "node": ">=0.12.0" } }, + "node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -397,6 +466,22 @@ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", "dev": true }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -422,6 +507,16 @@ "@redis/time-series": "1.0.3" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -452,6 +547,11 @@ "semver": "bin/semver.js" } }, + "node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -494,6 +594,15 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", @@ -521,6 +630,31 @@ } }, "dependencies": { + "@acuminous/bitsyntax": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz", + "integrity": "sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==", + "requires": { + "buffer-more-ints": "~1.0.0", + "debug": "^4.3.4", + "safe-buffer": "~5.1.2" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "@redis/bloom": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", @@ -567,6 +701,17 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "amqplib": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", + "integrity": "sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw==", + "requires": { + "@acuminous/bitsyntax": "^0.1.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "url-parse": "~1.5.10" + } + }, "anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -608,6 +753,11 @@ "fill-range": "^7.0.1" } }, + "buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, "chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -649,6 +799,11 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, "debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -705,6 +860,11 @@ "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", "dev": true }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -735,6 +895,11 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -795,6 +960,22 @@ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", "dev": true }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -817,6 +998,16 @@ "@redis/time-series": "1.0.3" } }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -840,6 +1031,11 @@ } } }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -873,6 +1069,15 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", diff --git a/package.json b/package.json index 242b923..db5c3a2 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "homepage": "https://github.com/sajera/s-prerender#s-prerender", "license": "MIT", "dependencies": { + "amqplib": "^0.10.3", "chrome-remote-interface": "^0.31.3", "dotenv": "^16.0.1", "redis": "^4.2.0" diff --git a/src/api/index.js b/src/api/index.js index 58eef07..ba5a417 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -4,7 +4,7 @@ import url from 'node:url'; import { createServer } from 'node:http'; // local dependencies -import { logError, log, suid, DEBUG } from '../config.js'; +import { logError, log, suid } from '../config.js'; // NOTE required interface for "api" export default { start, isReady, middleware }; diff --git a/src/cache/index.js b/src/cache/index.js index 6b8a573..1019966 100644 --- a/src/cache/index.js +++ b/src/cache/index.js @@ -1,7 +1,19 @@ +// local dependencies import redis from './redis.js'; -// TODO switching caches based on environment +// NOTE module interface +const noop = () => null; +const cache = { start, isReady: noop, set: noop, get: noop, del: noop }; + +// TODO switching based on environment variables // NOTE for now implemented only Redis ¯\_(ツ)_/¯ +function start (config) { + if (config.redis) { + Object.assign(cache, redis); + return redis.start(config); + } + throw new Error('No useful CACHE configuration found'); +} -export default redis; +export default cache; diff --git a/src/cache/redis.js b/src/cache/redis.js index 3c50c9e..1fe5a80 100644 --- a/src/cache/redis.js +++ b/src/cache/redis.js @@ -18,12 +18,12 @@ export function isReady () { return CONNECTED; } export async function start (config) { log('[redis:connecting]', config); - client = createClient(config); - client.on('connect', () => debug('[redis:start]', CONNECTED = true)); - client.on('ready', () => debug('[redis:ready]')); + client = createClient({ url: config.redisUrl }); + client.on('connect', () => debug('[redis:start]')); + client.on('ready', () => debug('[redis:connected]', CONNECTED = true)); client.on('end', () => debug('[redis:stopped]', CONNECTED = false)); client.on('error', error => logError('REDIS', { message: error.message, stack: error.stack })); await client.connect(); - log('[redis:connected]'); + log('[redis:started]', config.redisUrl); } diff --git a/src/config.js b/src/config.js index 0b2ee51..02194a5 100644 --- a/src/config.js +++ b/src/config.js @@ -2,7 +2,18 @@ // outsource dependencies import dotenv from 'dotenv'; -// local dependencies +// NOTE log unhandled promise exception +process.on('unhandledRejection', error => logError('[service:unhandledRejection]', error && { + message: error.message, + stack: error.stack, + code: error.code, +})); +// NOTE log process exception +process.on('uncaughtException', error => logError('[service:uncaughtException]', error && { + message: error.message, + stack: error.stack, + code: error.code, +}) || process.exit(1)); dotenv.config({ override: false, debug: varBoolean(process.env.DEBUG) }); export const DEBUG = varBoolean(process.env.DEBUG); @@ -12,18 +23,15 @@ export const API = { host: varString(process.env.HOST), allowDomains: varArray(process.env.ALLOW_DOMAINS) || ['.'], }; +// NOTE for now RabbitMQ only +export const QUEUE = { + rabbitmq: Boolean(process.env.RABBITMQ_URL), + rabbitmqUrl: varString(process.env.RABBITMQ_URL), +}; // NOTE for now Redis only export const CACHE = { - url: varString(process.env.REDIS_URL), - // name: process.env.REDIS_NAME, - // database: process.env.REDIS_DB, - // username: process.env.REDIS_USERNAME, - // password: process.env.REDIS_PASSWORD, - // commandsQueueMaxLength?: number; - // disableOfflineQueue?: boolean; - // readonly?: boolean; - // legacyMode?: boolean; - // isolationPoolOptions?: PoolOptions; + redis: Boolean(process.env.REDIS_URL), + redisUrl: varString(process.env.REDIS_URL), }; export const PRERENDER = { browserDebuggingPort: varNumber(process.env.CHROME_DEBUGGING_PORT), @@ -37,6 +45,7 @@ export const PRERENDER = { followRedirects: varBoolean(process.env.CHROME_FOLLOW_REDIREC), // Weather to follow redirect cleanupHtmlScript: varString(process.env.CHROME_CLEANUP_HTML_SCRIPT) || defaultCleanupHtmlScript(), // ability to pass string with JS to execute on all pages }; +export const config = { API, CACHE, PRERENDER, QUEUE }; /****************************************************** * variables parsers *****************************************************/ diff --git a/src/index.js b/src/index.js index 58dc3bc..8b2506f 100644 --- a/src/index.js +++ b/src/index.js @@ -5,30 +5,21 @@ import qs from 'node:querystring'; // local dependencies import api from './api/index.js'; import cache from './cache/index.js'; +import queue from './queue/index.js'; import prerender from './prerender/index.js'; -import { isUrl, logError, log, API, CACHE, PRERENDER } from './config.js'; - -// NOTE log unhandled promise exception -process.on('unhandledRejection', error => logError('[service:unhandledRejection]', error && { - message: error.message, - stack: error.stack, - code: error.code, -})); -// NOTE log process exception -process.on('uncaughtException', error => logError('[service:uncaughtException]', error && { - message: error.message, - stack: error.stack, - code: error.code, -}) || process.exit(1)); +import { isUrl, logError, log, config } from './config.js'; // configure let READY; +const services = { API: api, CACHE: cache, PRERENDER: prerender, QUEUE: queue }; +const sids = Object.keys(services); // run all services -Promise.all([ - api.start(API), - cache.start(CACHE), - prerender.start(PRERENDER), -]).then(() => log('[service:ready]', READY = true)); +Promise.all(sids.map(id => services[id].start(config[id]))) + .then(() => log('[service:started]', READY = true)) + .catch(error => { + logError('SERVICES', error.message); + process.exit(100500); + }); /****************************************************** * GET /health @@ -50,7 +41,7 @@ function health () { api.middleware.GET['/render'] = render; render.contentType = 'text/html'; async function render (request) { - checkReadyState(); + checkReadyState(['CACHE']); const url = validUrl(request); const results = await cache.get(url); if (results) { @@ -67,7 +58,7 @@ async function render (request) { api.middleware.GET['/refresh'] = refresh; refresh.contentType = 'text/html'; async function refresh (request) { - checkReadyState(); + checkReadyState(['CACHE', 'PRERENDER']); const url = validUrl(request); const results = await prerender.render(url); log('[api:generate]', url); @@ -82,7 +73,7 @@ async function refresh (request) { api.middleware.GET['/cached'] = getCached; getCached.contentType = 'text/html'; async function getCached (request) { - checkReadyState(); + checkReadyState(['CACHE']); const url = validUrl(request); const results = await cache.get(url); if (!results) { throw { code: 404, message: `Cache empty for "${url}"` }; } @@ -95,20 +86,21 @@ async function getCached (request) { api.middleware.DELETE['/cached'] = deleteCached; deleteCached.contentType = 'text/plain'; async function deleteCached (request) { - checkReadyState(); + checkReadyState(['CACHE']); const url = validUrl(request); await cache.del(url); return 'OK'; } -/****************************************************** - * /////////////////// - *****************************************************/ -function checkReadyState () { - if (!api.isReady()) { throw { code: 503, message: 'Service(API) not ready yet' }; } - if (!cache.isReady()) { throw { code: 503, message: 'Service(CACHE) not ready yet' }; } - if (!prerender.isReady()) { throw { code: 503, message: 'Service(PRERENDER) not ready yet' }; } - return true; +/********************************************* + * //////// THROW /////////// * + *********************************************/ +function checkReadyState (required = sids) { + return required.map(id => { + if (!services[id].isReady()) { + throw { code: 503, message: `Service(${id}) not ready yet` }; + } + }); } function validUrl (request) { diff --git a/src/prerender/chrome.js b/src/prerender/chrome.js index cd5f508..9046bd6 100644 --- a/src/prerender/chrome.js +++ b/src/prerender/chrome.js @@ -44,7 +44,7 @@ chrome.connect = () => new Promise((resolve, reject) => { clearTimeout(timeout); connected = true; debug('[prerender:ready]', info); - resolve(info); + resolve(info['User-Agent']); }).catch(error => debug('[prerender:connect] Retrying connection to browser...', error, setTimeout(retry, 4e3))); setTimeout(retry, 0); }); diff --git a/src/prerender/index.js b/src/prerender/index.js index 0312729..40173cd 100644 --- a/src/prerender/index.js +++ b/src/prerender/index.js @@ -7,7 +7,7 @@ import { chrome as browser } from './chrome.js'; import { DEBUG, debug, delay, log, suid } from '../config.js'; // NOTE required interface for "prerender" -export default { start, render, isReady }; +export default { start, isReady, render }; // configure let CONNECTED; @@ -19,13 +19,13 @@ process.on('SIGINT', () => { export function isReady () { return CONNECTED; } export async function start (config) { - log('[prerender:starting]', config); + log('[prerender:connecting]', config); browser.kill(); await browser.spawn(config); - browser.onClose(() => log('[prerender:stopped]', CONNECTED = false)); - await browser.connect(); + browser.onClose(() => log('[prerender:stopped]', CONNECTED = false, start(config))); + const info = await browser.connect(); CONNECTED = true; - log('[prerender:started]'); + log('[prerender:started]', info); } // TODO ERROR:[service:unhandledRejection] diff --git a/src/queue/index.js b/src/queue/index.js new file mode 100644 index 0000000..cf281c0 --- /dev/null +++ b/src/queue/index.js @@ -0,0 +1,19 @@ + +// local dependencies +import rabbitmq from './rabbitmq.js'; + +// NOTE module interface +const noop = () => null; +const queue = { start, isReady: noop }; + +// TODO switching based on environment variables +// NOTE for now implemented only Redis ¯\_(ツ)_/¯ +function start (config) { + if (config.rabbitmq) { + Object.assign(queue, rabbitmq); + return rabbitmq.start(config); + } + throw new Error('No useful QUEUE configuration found'); +} + +export default queue; diff --git a/src/queue/rabbitmq.js b/src/queue/rabbitmq.js new file mode 100644 index 0000000..b264cd3 --- /dev/null +++ b/src/queue/rabbitmq.js @@ -0,0 +1,35 @@ + +// outsource dependencies +import amqp from 'amqplib'; + +// local dependencies +import { logError, debug, log } from '../config.js'; + +// NOTE required interface for "queue" +export default { start, set, isReady }; + +// configure +let client; +let CONNECTED; +export function set (key, value) { return ; } +export function isReady () { return CONNECTED; } + +export async function start (config) { + log('[rabbitmq:connecting]', config); + await connect(config.rabbitmqUrl); + client.on('close', () => debug('[rabbitmq:stopped]', CONNECTED = false, start(config))); + client.on('error', error => logError('RABBITMQ', { message: error.message, stack: error.stack })); + // TODO channel ? + log('[rabbitmq:started]', config.rabbitmqUrl); +} + + +const connect = url => new Promise(resolve => { + const retry = () => amqp.connect(url) + .then(connection => { + debug('[rabbitmq:connected]', CONNECTED = true); + resolve(client = connection); + }) + .catch(error => logError('RABBITMQ', { message: error.message, stack: error.stack }, setTimeout(retry, 4e3))); + retry(); +}); From 1271216ac729a37f7d0c67b57135dc48c5150542 Mon Sep 17 00:00:00 2001 From: Sajera Date: Sun, 25 Sep 2022 11:47:43 +0300 Subject: [PATCH 05/10] feature/RELEASE-1.0.9 -- process point -- queue configuration -- POST /refresh --- .env | 3 +- docker-compose.yaml | 12 ++++++++ src/api/index.js | 2 +- src/cache/index.js | 9 +++--- src/config.js | 8 ++++-- src/index.js | 66 +++++++++++++++++++++++++++++++------------ src/queue/index.js | 11 ++++---- src/queue/rabbitmq.js | 32 +++++++++++++++++---- 8 files changed, 107 insertions(+), 36 deletions(-) diff --git a/.env b/.env index a831469..e0eb443 100644 --- a/.env +++ b/.env @@ -9,7 +9,8 @@ ALLOW_DOMAINS=. # Redis local configuration REDIS_URL=redis://127.0.0.1:6379 -# Redis local configuration +# RabbitMQ local configuration +RABBITMQ_QUEUE=PRERENDER RABBITMQ_URL=amqp://127.0.0.1:5672 # Prerender local configuration diff --git a/docker-compose.yaml b/docker-compose.yaml index fbc7e99..344dcad 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,8 +10,10 @@ services: DEBUG: 1 PORT: 3636 REDIS_URL: redis://cache:6379/0 + RABBITMQ_URL: amqp://queue:5672 depends_on: - cache + - queue ports: - '3636:3636' command: npm run start:dev @@ -27,6 +29,16 @@ services: command: redis-server --save 20 1 --loglevel warning volumes: - cache:/data/cache + - + queue: + image: rabbitmq:management + restart: always + ports: + - '5672:5672' + - '15672:15672' + volumes: + - queue:/data/queue + - volumes: cache: diff --git a/src/api/index.js b/src/api/index.js index ba5a417..1f3d0b5 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -18,7 +18,7 @@ export function start (config) { isReady() && api.close(() => log('[api:stopped]')); return new Promise(resolve => { api = createServer(middleware); - log('[api:starting]', config); + log('[api:connecting]', config); api.listen(config.port, config.host, () => { log('[api:started]', `http://${config.host}:${config.port}/`); resolve(); diff --git a/src/cache/index.js b/src/cache/index.js index 1019966..6e3f431 100644 --- a/src/cache/index.js +++ b/src/cache/index.js @@ -2,18 +2,19 @@ // local dependencies import redis from './redis.js'; -// NOTE module interface +// configure const noop = () => null; +// NOTE module interface const cache = { start, isReady: noop, set: noop, get: noop, del: noop }; -// TODO switching based on environment variables -// NOTE for now implemented only Redis ¯\_(ツ)_/¯ +// NOTE switching based on environment variables function start (config) { if (config.redis) { Object.assign(cache, redis); return redis.start(config); } - throw new Error('No useful CACHE configuration found'); + // NOTE for now implemented only Redis ¯\_(ツ)_/¯ + // throw new Error('No useful CACHE configuration found'); } export default cache; diff --git a/src/config.js b/src/config.js index 02194a5..ca1215c 100644 --- a/src/config.js +++ b/src/config.js @@ -14,10 +14,11 @@ process.on('uncaughtException', error => logError('[service:uncaughtException]', stack: error.stack, code: error.code, }) || process.exit(1)); - +// NOTE strict dotenv rules to avoid unexpected process environment - .env is defaults with minimal priority dotenv.config({ override: false, debug: varBoolean(process.env.DEBUG) }); +// NOTE reading any variables only after reading defaults to make sure the minimal required data was set export const DEBUG = varBoolean(process.env.DEBUG); - +// NOTE for base node server export const API = { port: varNumber(process.env.PORT), host: varString(process.env.HOST), @@ -27,12 +28,14 @@ export const API = { export const QUEUE = { rabbitmq: Boolean(process.env.RABBITMQ_URL), rabbitmqUrl: varString(process.env.RABBITMQ_URL), + rabbitmqQueue: varString(process.env.RABBITMQ_QUEUE), }; // NOTE for now Redis only export const CACHE = { redis: Boolean(process.env.REDIS_URL), redisUrl: varString(process.env.REDIS_URL), }; +// NOTE Chrome/Chromium only export const PRERENDER = { browserDebuggingPort: varNumber(process.env.CHROME_DEBUGGING_PORT), chromeLocation: varString(process.env.CHROME_BIN), @@ -61,6 +64,7 @@ export function varArray (value) { export function varString (value) { return /^(null|undefined)$/i.test(value) ? void 0 : value; } +// TODO way to setup js for rendered page export function defaultCleanupHtmlScript () { return `(tags => { for(const tag of tags) { diff --git a/src/index.js b/src/index.js index 8b2506f..7dd907f 100644 --- a/src/index.js +++ b/src/index.js @@ -12,9 +12,9 @@ import { isUrl, logError, log, config } from './config.js'; // configure let READY; const services = { API: api, CACHE: cache, PRERENDER: prerender, QUEUE: queue }; -const sids = Object.keys(services); +const servicesIds = Object.keys(services); // run all services -Promise.all(sids.map(id => services[id].start(config[id]))) +Promise.all(servicesIds.map(id => services[id].start(config[id]))) .then(() => log('[service:started]', READY = true)) .catch(error => { logError('SERVICES', error.message); @@ -29,7 +29,7 @@ health.contentType = 'application/json'; function health () { let ready = false; try { - checkReadyState(); + checkReadyState(servicesIds); ready = true; } catch (e) { } return JSON.stringify({ status: ready ? 'UP' : 'DOWN' }); @@ -41,30 +41,40 @@ function health () { api.middleware.GET['/render'] = render; render.contentType = 'text/html'; async function render (request) { - checkReadyState(['CACHE']); - const url = validUrl(request); - const results = await cache.get(url); - if (results) { - log('[api:from-cache]', url); - return results; + try { + return await getCached(request); + } catch (error) { + return getRefresh(request); } - log('[api:no-cache]', url); - return refresh(request); } /****************************************************** * GET /refresh?url=http://example.com/ *****************************************************/ -api.middleware.GET['/refresh'] = refresh; -refresh.contentType = 'text/html'; -async function refresh (request) { +api.middleware.GET['/refresh'] = getRefresh; +getRefresh.contentType = 'text/html'; +async function getRefresh (request) { checkReadyState(['CACHE', 'PRERENDER']); const url = validUrl(request); const results = await prerender.render(url); log('[api:generate]', url); await cache.set(url, results); log('[api:to-cache]', url); - return results; + return 'OK'; +} + +/****************************************************** + * POST /refresh + *****************************************************/ +api.middleware.POST['/refresh'] = postRefresh; +postRefresh.contentType = 'text/html'; +async function postRefresh (request) { + checkReadyState(['CACHE', 'QUEUE']); + const urls = await parseUrls(request); + // TODO message schema + await queue.sendToQueue(urls); + log('[api:to-queue]', urls.length); + return 'OK'; } /****************************************************** @@ -77,6 +87,7 @@ async function getCached (request) { const url = validUrl(request); const results = await cache.get(url); if (!results) { throw { code: 404, message: `Cache empty for "${url}"` }; } + log('[api:from-cache]', url); return results; } @@ -95,16 +106,35 @@ async function deleteCached (request) { /********************************************* * //////// THROW /////////// * *********************************************/ -function checkReadyState (required = sids) { +function checkReadyState (required = servicesIds) { return required.map(id => { if (!services[id].isReady()) { - throw { code: 503, message: `Service(${id}) not ready yet` }; + throw { code: 503, message: `Service(${id}) unavailable` }; } }); } function validUrl (request) { const url = qs.unescape(qs.parse(request.url.query).url); - if (!isUrl(url)) { throw { code: 400, message: `Invalid query parameter url "${url}"` }; } + if (!isUrl(url)) { throw { code: 422, message: `Unsupported url "${url}"` }; } return url; } + +const parseUrls = request => new Promise((resolve, reject) => { + let data = ''; + request.on('data', chunk => data += chunk); + request.on('end', () => { + let urls = []; + try { urls = JSON.parse(data); } catch (error) { + logError('[api:parseUrls]', { message: error.message, stack: error.stack }); + } + if (!Array.isArray(urls)) { reject({ code: 422, message: 'Invalid data, expected an Array with URLs' }); } + const obj = {}; + for (const url of urls) { obj[url] = 1; } + urls = []; + // NOTE only uniq and valid url + for (const url in obj) { isUrl(url) && urls.push(url); } + if (!urls.length) { reject({ code: 422, message: 'Invalid data, expected an Array with URLs' }); } + resolve(urls); + }); +}); diff --git a/src/queue/index.js b/src/queue/index.js index cf281c0..40316f3 100644 --- a/src/queue/index.js +++ b/src/queue/index.js @@ -2,18 +2,19 @@ // local dependencies import rabbitmq from './rabbitmq.js'; -// NOTE module interface +// configure const noop = () => null; -const queue = { start, isReady: noop }; +// NOTE module interface +const queue = { start, isReady: noop, sendToQueue: noop }; -// TODO switching based on environment variables -// NOTE for now implemented only Redis ¯\_(ツ)_/¯ +// NOTE switching based on environment variables function start (config) { if (config.rabbitmq) { Object.assign(queue, rabbitmq); return rabbitmq.start(config); } - throw new Error('No useful QUEUE configuration found'); + // NOTE for now implemented only RabbitMQ ¯\_(ツ)_/¯ + // throw new Error('No useful QUEUE configuration found'); } export default queue; diff --git a/src/queue/rabbitmq.js b/src/queue/rabbitmq.js index b264cd3..1ef2e61 100644 --- a/src/queue/rabbitmq.js +++ b/src/queue/rabbitmq.js @@ -1,29 +1,39 @@ // outsource dependencies import amqp from 'amqplib'; +import { Buffer } from 'node:buffer'; // local dependencies -import { logError, debug, log } from '../config.js'; +import { logError, debug, log, delay } from '../config.js'; // NOTE required interface for "queue" -export default { start, set, isReady }; +export default { start, isReady, sendToQueue }; // configure let client; let CONNECTED; -export function set (key, value) { return ; } export function isReady () { return CONNECTED; } +export async function sendToQueue (messages) { + const channel = await client.createChannel(); + await channel.assertQueue(client.rabbitmqQueue); + debug(`[rabbitmq:sendToQueue] ${client.rabbitmqQueue}`, messages); + for (const msg of messages) { + channel.sendToQueue(client.rabbitmqQueue, Buffer.from(msg)); + } +} export async function start (config) { log('[rabbitmq:connecting]', config); await connect(config.rabbitmqUrl); + client.rabbitmqQueue = config.rabbitmqQueue; client.on('close', () => debug('[rabbitmq:stopped]', CONNECTED = false, start(config))); client.on('error', error => logError('RABBITMQ', { message: error.message, stack: error.stack })); - // TODO channel ? + const channel = await client.createConfirmChannel(); + await channel.assertQueue(client.rabbitmqQueue); + channel.consume(client.rabbitmqQueue, message(channel), { durable: false }); log('[rabbitmq:started]', config.rabbitmqUrl); } - const connect = url => new Promise(resolve => { const retry = () => amqp.connect(url) .then(connection => { @@ -33,3 +43,15 @@ const connect = url => new Promise(resolve => { .catch(error => logError('RABBITMQ', { message: error.message, stack: error.stack }, setTimeout(retry, 4e3))); retry(); }); + +const message = channel => async message => { + try { + await delay(5e3); + debug(`[rabbitmq:message] TODO handle`, message.content.toString()); + + channel.ack(message); + } catch (error) { + channel.reject(message, true); + logError('[rabbitmq:message]', { message: error.message, stack: error.stack }); + } +} From 7590ab31db95da45047326cb5690832d99b46537 Mon Sep 17 00:00:00 2001 From: Sajera Date: Sun, 25 Sep 2022 13:11:37 +0300 Subject: [PATCH 06/10] feature/RELEASE-1.0.9 -- process point -- allow to ignore refreshing html results --- src/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.js b/src/index.js index 7dd907f..7ea651d 100644 --- a/src/index.js +++ b/src/index.js @@ -60,7 +60,7 @@ async function getRefresh (request) { log('[api:generate]', url); await cache.set(url, results); log('[api:to-cache]', url); - return 'OK'; + return qs.parse(request.url.query).ignoreResults ? 'OK' : results; } /****************************************************** From 2b5ad97edeee6e56f109f14ae66b3cb44fea054e Mon Sep 17 00:00:00 2001 From: Sajera Date: Sun, 25 Sep 2022 15:59:04 +0300 Subject: [PATCH 07/10] feature/RELEASE-1.0.9 -- process point -- queue controls via environment -- RABBITMQ_CHANNELS -- RABBITMQ_QUEUE -- RABBITMQ_URL --- .env | 1 + src/config.js | 1 + src/index.js | 2 +- src/queue/rabbitmq.js | 40 ++++++++++++++++++++++++++++++---------- 4 files changed, 33 insertions(+), 11 deletions(-) diff --git a/.env b/.env index e0eb443..26086ba 100644 --- a/.env +++ b/.env @@ -10,6 +10,7 @@ ALLOW_DOMAINS=. REDIS_URL=redis://127.0.0.1:6379 # RabbitMQ local configuration +RABBITMQ_CHANNELS=1 RABBITMQ_QUEUE=PRERENDER RABBITMQ_URL=amqp://127.0.0.1:5672 diff --git a/src/config.js b/src/config.js index ca1215c..2b12e78 100644 --- a/src/config.js +++ b/src/config.js @@ -29,6 +29,7 @@ export const QUEUE = { rabbitmq: Boolean(process.env.RABBITMQ_URL), rabbitmqUrl: varString(process.env.RABBITMQ_URL), rabbitmqQueue: varString(process.env.RABBITMQ_QUEUE), + rabbitmqChannels: varNumber(process.env.RABBITMQ_CHANNELS), }; // NOTE for now Redis only export const CACHE = { diff --git a/src/index.js b/src/index.js index 7ea651d..5c1921d 100644 --- a/src/index.js +++ b/src/index.js @@ -130,7 +130,7 @@ const parseUrls = request => new Promise((resolve, reject) => { } if (!Array.isArray(urls)) { reject({ code: 422, message: 'Invalid data, expected an Array with URLs' }); } const obj = {}; - for (const url of urls) { obj[url] = 1; } + for (const url of urls) { obj[qs.unescape(url)] = 1; } urls = []; // NOTE only uniq and valid url for (const url in obj) { isUrl(url) && urls.push(url); } diff --git a/src/queue/rabbitmq.js b/src/queue/rabbitmq.js index 1ef2e61..6387ab3 100644 --- a/src/queue/rabbitmq.js +++ b/src/queue/rabbitmq.js @@ -1,10 +1,11 @@ // outsource dependencies import amqp from 'amqplib'; +import http from 'node:http'; import { Buffer } from 'node:buffer'; // local dependencies -import { logError, debug, log, delay } from '../config.js'; +import { logError, debug, log, config } from '../config.js'; // NOTE required interface for "queue" export default { start, isReady, sendToQueue }; @@ -15,22 +16,28 @@ let CONNECTED; export function isReady () { return CONNECTED; } export async function sendToQueue (messages) { const channel = await client.createChannel(); - await channel.assertQueue(client.rabbitmqQueue); - debug(`[rabbitmq:sendToQueue] ${client.rabbitmqQueue}`, messages); + await channel.assertQueue(client.queue, client.queueOptions); + debug(`[rabbitmq:sendToQueue] ${client.queue}`, messages); for (const msg of messages) { - channel.sendToQueue(client.rabbitmqQueue, Buffer.from(msg)); + // TODO message schema + channel.sendToQueue(client.queue, Buffer.from(msg), client.sendOptions); } } export async function start (config) { log('[rabbitmq:connecting]', config); await connect(config.rabbitmqUrl); - client.rabbitmqQueue = config.rabbitmqQueue; + // NOTE configuration + client.queue = config.rabbitmqQueue; + client.queueOptions = { durable: true }; + client.sendOptions = { persistent: true }; + client.consumeOptions = { noAck: false }; client.on('close', () => debug('[rabbitmq:stopped]', CONNECTED = false, start(config))); client.on('error', error => logError('RABBITMQ', { message: error.message, stack: error.stack })); const channel = await client.createConfirmChannel(); - await channel.assertQueue(client.rabbitmqQueue); - channel.consume(client.rabbitmqQueue, message(channel), { durable: false }); + channel.prefetch(config.rabbitmqChannels); + await channel.assertQueue(client.queue, client.queueOptions); + channel.consume(client.queue, message(channel), client.consumeOptions); log('[rabbitmq:started]', config.rabbitmqUrl); } @@ -46,12 +53,25 @@ const connect = url => new Promise(resolve => { const message = channel => async message => { try { - await delay(5e3); - debug(`[rabbitmq:message] TODO handle`, message.content.toString()); - + // TODO message schema from Estative + // TODO limit attempts to handle message + if (message.fields.deliveryTag < 5) { + log('[rabbitmq:message]', message.content.toString()); + await refresh(message.content.toString()); + } channel.ack(message); } catch (error) { channel.reject(message, true); logError('[rabbitmq:message]', { message: error.message, stack: error.stack }); } } + +const refresh = url => new Promise((resolve, reject) => { + http.get(new URL(`http://${config.API.host}:${config.API.port}/refresh?ignoreResults=true&url=${url}`), response => { + // NOTE consume response data to free up memory + response.resume(); + // NOTE no addition data or explanations need due to debug from API logs. + return response.statusCode === 200 ? resolve('OK') + : reject(new Error(`Request Failed. [${response.statusCode}] Failed to refresh "${url}"`)); + }); +}); From 81453392891ffe5e6e64ab321c3cb5b7711d29e2 Mon Sep 17 00:00:00 2001 From: Sajera Date: Sun, 25 Sep 2022 16:01:27 +0300 Subject: [PATCH 08/10] feature/RELEASE-1.0.9 -- process point --- src/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.js b/src/config.js index 2b12e78..959cc0e 100644 --- a/src/config.js +++ b/src/config.js @@ -18,7 +18,7 @@ process.on('uncaughtException', error => logError('[service:uncaughtException]', dotenv.config({ override: false, debug: varBoolean(process.env.DEBUG) }); // NOTE reading any variables only after reading defaults to make sure the minimal required data was set export const DEBUG = varBoolean(process.env.DEBUG); -// NOTE for base node server +// NOTE export const API = { port: varNumber(process.env.PORT), host: varString(process.env.HOST), From e854a41fc8be1c10f20d6621241491311b025171 Mon Sep 17 00:00:00 2001 From: Sajera Date: Mon, 26 Sep 2022 13:13:13 +0300 Subject: [PATCH 09/10] feature/RELEASE-1.0.9 -- update README -- queue implementation -- queue message schema -- clear local defaults for Docker image --- Dockerfile | 2 ++ README.md | 14 +++++++++- src/queue/rabbitmq.js | 64 ++++++++++++++++++++++++++----------------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea39dc4..acd05e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,6 +5,8 @@ RUN apk add --update-cache chromium && \ ENV DEBUG=false \ HOST=0.0.0.0 \ + REDIS_URL="" \ + RABBITMQ_URL="" \ CHROME_DEBUGGING_PORT=9222 \ CHROME_BIN=/usr/bin/chromium-browser \ CHROME_FLAGS="--no-sandbox,--headless,--disable-gpu,--remote-debugging-port=9222,--hide-scrollbars,--disable-dev-shm-usage" diff --git a/README.md b/README.md index 52aea78..f0ecf1c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ Deployment using `Dockerfile` require only `REDIS_URL`. `REDIS_URL=` connection to Redis within your environment +`RABBITMQ_URL=` connection to RabbitMQ within your environment (optional) + > Defaults - `PORT=3636` - `DEBUG=false` @@ -42,6 +44,11 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - `CHROME_DEBUGGING_PORT=9222` - `CHROME_BIN=/usr/bin/chromium-browser` - `CHROME_FLAGS=--no-sandbox,--headless,--disable-gpu,--remote-debugging-port=9222,--hide-scrollbars,--disable-dev-shm-usage` +- `REDIS_URL=` +- `RABBITMQ_URL=` +- `RABBITMQ_CHANNELS=1` +- `RABBITMQ_QUEUE=PRERENDER` + --- ### API @@ -57,6 +64,11 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - Force reset the cache and render URL in `Chromium` browser, returns `HTML`. - **GET /refresh** - `curl 'http://localhost:3636/refresh?url=http://example.com/'` + - Optional query `ignoreResults=true` to avoid html results + +- Force reset the cache and render URLs in `Chromium` browser + - **POST /refresh** + - `curl 'http://localhost:3636/refresh' -X 'POST' --data-raw '["not a link","http://example.com/","http://example.com/"]'` - Will get URL `HTML` content from the cache. - **GET /cached** @@ -76,7 +88,7 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - [x] Docker image - [x] Docker for local development - [x] Domain limitation -- [ ] Queue for rendering +- [x] Queue for rendering - [ ] Accumulate Sitemap - [ ] Different cache technology - [ ] Different queue technology diff --git a/src/queue/rabbitmq.js b/src/queue/rabbitmq.js index 6387ab3..a757709 100644 --- a/src/queue/rabbitmq.js +++ b/src/queue/rabbitmq.js @@ -14,13 +14,12 @@ export default { start, isReady, sendToQueue }; let client; let CONNECTED; export function isReady () { return CONNECTED; } -export async function sendToQueue (messages) { +export async function sendToQueue (urls) { const channel = await client.createChannel(); await channel.assertQueue(client.queue, client.queueOptions); - debug(`[rabbitmq:sendToQueue] ${client.queue}`, messages); - for (const msg of messages) { - // TODO message schema - channel.sendToQueue(client.queue, Buffer.from(msg), client.sendOptions); + debug(`[rabbitmq:sendToQueue] ${client.queue}`, urls); + for (const url of urls) { + channel.sendToQueue(client.queue, formatMessage({ url }), client.sendOptions); } } @@ -41,31 +40,27 @@ export async function start (config) { log('[rabbitmq:started]', config.rabbitmqUrl); } -const connect = url => new Promise(resolve => { - const retry = () => amqp.connect(url) - .then(connection => { - debug('[rabbitmq:connected]', CONNECTED = true); - resolve(client = connection); - }) - .catch(error => logError('RABBITMQ', { message: error.message, stack: error.stack }, setTimeout(retry, 4e3))); - retry(); -}); - const message = channel => async message => { + const { url, attempt = 0 } = parseMessage(message.content.toString()); try { - // TODO message schema from Estative - // TODO limit attempts to handle message - if (message.fields.deliveryTag < 5) { - log('[rabbitmq:message]', message.content.toString()); - await refresh(message.content.toString()); - } - channel.ack(message); + log(`[rabbitmq:message] ${attempt}`, message.content.toString()); + await refresh(url); } catch (error) { - channel.reject(message, true); - logError('[rabbitmq:message]', { message: error.message, stack: error.stack }); + logError('[rabbitmq:message]', error.message); + // NOTE try again later + if (url && attempt > 0) { + try { + // NOTE create new message in a queue + channel.sendToQueue(client.queue, formatMessage({ url, attempt: attempt - 1 }), client.sendOptions); + } catch (error) { /* NOTE unbelievable, but just in case */ } + } } + // NOTE at any case message was handled + channel.ack(message); } - +/****************************************************** + * HELPERS + *****************************************************/ const refresh = url => new Promise((resolve, reject) => { http.get(new URL(`http://${config.API.host}:${config.API.port}/refresh?ignoreResults=true&url=${url}`), response => { // NOTE consume response data to free up memory @@ -75,3 +70,22 @@ const refresh = url => new Promise((resolve, reject) => { : reject(new Error(`Request Failed. [${response.statusCode}] Failed to refresh "${url}"`)); }); }); + +const formatMessage = ({ url, attempt = 3 }) => Buffer.from(JSON.stringify({ attempt, url })); +const parseMessage = message => { + try { + return JSON.parse(message.toString()) || {}; + } catch (error) { + return {}; + } +} + +const connect = url => new Promise(resolve => { + const retry = () => amqp.connect(url) + .then(connection => { + debug('[rabbitmq:connected]', CONNECTED = true); + resolve(client = connection); + }) + .catch(error => logError('RABBITMQ', { message: error.message, stack: error.stack }, setTimeout(retry, 4e3))); + retry(); +}); From 36979263e175f9be7a713c16d56ef48cc9b5b605 Mon Sep 17 00:00:00 2001 From: Sajera Date: Mon, 26 Sep 2022 14:02:03 +0300 Subject: [PATCH 10/10] feature/RELEASE-1.0.9 -- fix docker-compose --- docker-compose.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 344dcad..444b023 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -29,7 +29,7 @@ services: command: redis-server --save 20 1 --loglevel warning volumes: - cache:/data/cache - - + queue: image: rabbitmq:management restart: always @@ -37,8 +37,7 @@ services: - '5672:5672' - '15672:15672' volumes: - - queue:/data/queue - - + - cache:/data/queue volumes: cache: