diff --git a/.env b/.env index c3443fb..26086ba 100644 --- a/.env +++ b/.env @@ -3,12 +3,17 @@ DEBUG=true # API local configuration PORT=3636 -HOST=0.0.0.0 +HOST=localhost ALLOW_DOMAINS=. # Redis local configuration REDIS_URL=redis://127.0.0.1:6379 +# RabbitMQ local configuration +RABBITMQ_CHANNELS=1 +RABBITMQ_QUEUE=PRERENDER +RABBITMQ_URL=amqp://127.0.0.1:5672 + # Prerender local configuration CHROME_DEBUGGING_PORT=9222 #CHROME_BIN ===== for local enw will take your Chrome automatically diff --git a/Dockerfile b/Dockerfile index b04d4f1..acd05e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ RUN apk add --update-cache chromium && \ rm -rf /var/cache/apk/* /tmp/* ENV DEBUG=false \ + HOST=0.0.0.0 \ + REDIS_URL="" \ + RABBITMQ_URL="" \ CHROME_DEBUGGING_PORT=9222 \ CHROME_BIN=/usr/bin/chromium-browser \ CHROME_FLAGS="--no-sandbox,--headless,--disable-gpu,--remote-debugging-port=9222,--hide-scrollbars,--disable-dev-shm-usage" diff --git a/README.md b/README.md index e380557..f0ecf1c 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,6 @@ Before start development locally, please make sure you have installed [Chrome br `npm run start:dev` -> NOTE Starting Redis on MAC using Brew `brew services start redis` ## Development using [Docker](https://www.docker.com/) Please, take in mind, that the `Dockerfile` isn't for local usage. It exists to simplify inserting the app into complex infrastructures. @@ -35,15 +34,21 @@ Deployment using `Dockerfile` require only `REDIS_URL`. `REDIS_URL=` connection to Redis within your environment +`RABBITMQ_URL=` connection to RabbitMQ within your environment (optional) + > Defaults - `PORT=3636` - `DEBUG=false` - `HOST=0.0.0.0` - `ALLOW_DOMAINS=.` - `CHROME_DEBUGGING_PORT=9222` -- `CHROME_FORWARD_HEADERS=true` - `CHROME_BIN=/usr/bin/chromium-browser` - `CHROME_FLAGS=--no-sandbox,--headless,--disable-gpu,--remote-debugging-port=9222,--hide-scrollbars,--disable-dev-shm-usage` +- `REDIS_URL=` +- `RABBITMQ_URL=` +- `RABBITMQ_CHANNELS=1` +- `RABBITMQ_QUEUE=PRERENDER` + --- ### API @@ -52,13 +57,18 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - `curl 'http://localhost:3636/health'` - `{ status: "UP" | "DOWN" }` -- Will render URL in `Chrome` browser then return `HTML` only the first time. After providing `HTML` content from the cache. +- Will render URL in `Chromium` browser then return `HTML` only the first time. After that, provide `HTML` content from the cache. - **GET /render** - `curl 'http://localhost:3636/render?url=http://example.com/'` -- Force reset cache and render url in `Chrome` browser then return `html`. +- Force reset the cache and render URL in `Chromium` browser, returns `HTML`. - **GET /refresh** - `curl 'http://localhost:3636/refresh?url=http://example.com/'` + - Optional query `ignoreResults=true` to avoid html results + +- Force reset the cache and render URLs in `Chromium` browser + - **POST /refresh** + - `curl 'http://localhost:3636/refresh' -X 'POST' --data-raw '["not a link","http://example.com/","http://example.com/"]'` - Will get URL `HTML` content from the cache. - **GET /cached** @@ -68,3 +78,17 @@ Deployment using `Dockerfile` require only `REDIS_URL`. - **DELETE /cached** - `curl -X 'DELETE' 'http://localhost:3636/cached?url=http://example.com/'` +# TODO +- [x] Render SPA page to get HTML +- [x] Cache HTML +- [x] Refresh cached HTML +- [x] Cache unlimited but controlled via API +- [x] Health status +- [x] Base environment +- [x] Docker image +- [x] Docker for local development +- [x] Domain limitation +- [x] Queue for rendering +- [ ] Accumulate Sitemap +- [ ] Different cache technology +- [ ] Different queue technology diff --git a/docker-compose.yaml b/docker-compose.yaml index fbc7e99..444b023 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,8 +10,10 @@ services: DEBUG: 1 PORT: 3636 REDIS_URL: redis://cache:6379/0 + RABBITMQ_URL: amqp://queue:5672 depends_on: - cache + - queue ports: - '3636:3636' command: npm run start:dev @@ -28,5 +30,14 @@ services: volumes: - cache:/data/cache + queue: + image: rabbitmq:management + restart: always + ports: + - '5672:5672' + - '15672:15672' + volumes: + - cache:/data/queue + volumes: cache: diff --git a/package-lock.json b/package-lock.json index ebad10b..5772b21 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,14 +1,15 @@ { "name": "s-prerender", - "version": "1.0.8", + "version": "1.0.9", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "s-prerender", - "version": "1.0.8", + "version": "1.0.9", "license": "MIT", "dependencies": { + "amqplib": "^0.10.3", "chrome-remote-interface": "^0.31.3", "dotenv": "^16.0.1", "redis": "^4.2.0" @@ -21,6 +22,40 @@ "npm": ">=7" } }, + "node_modules/@acuminous/bitsyntax": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz", + "integrity": "sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==", + "dependencies": { + "buffer-more-ints": "~1.0.0", + "debug": "^4.3.4", + "safe-buffer": "~5.1.2" + }, + "engines": { + "node": ">=0.8" + } + }, + "node_modules/@acuminous/bitsyntax/node_modules/debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/@acuminous/bitsyntax/node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "node_modules/@redis/bloom": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", @@ -80,6 +115,20 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "node_modules/amqplib": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", + "integrity": "sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw==", + "dependencies": { + "@acuminous/bitsyntax": "^0.1.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "url-parse": "~1.5.10" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -130,6 +179,11 @@ "node": ">=8" } }, + "node_modules/buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -188,6 +242,11 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "node_modules/core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, "node_modules/debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -266,6 +325,11 @@ "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", "dev": true }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -308,6 +372,11 @@ "node": ">=0.12.0" } }, + "node_modules/isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + }, "node_modules/minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -397,6 +466,22 @@ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", "dev": true }, + "node_modules/querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "node_modules/readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, "node_modules/readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -422,6 +507,16 @@ "@redis/time-series": "1.0.3" } }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -452,6 +547,11 @@ "semver": "bin/semver.js" } }, + "node_modules/string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -494,6 +594,15 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "node_modules/url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "dependencies": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "node_modules/ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", @@ -521,6 +630,31 @@ } }, "dependencies": { + "@acuminous/bitsyntax": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/@acuminous/bitsyntax/-/bitsyntax-0.1.2.tgz", + "integrity": "sha512-29lUK80d1muEQqiUsSo+3A0yP6CdspgC95EnKBMi22Xlwt79i/En4Vr67+cXhU+cZjbti3TgGGC5wy1stIywVQ==", + "requires": { + "buffer-more-ints": "~1.0.0", + "debug": "^4.3.4", + "safe-buffer": "~5.1.2" + }, + "dependencies": { + "debug": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "@redis/bloom": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.0.2.tgz", @@ -567,6 +701,17 @@ "integrity": "sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==", "dev": true }, + "amqplib": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/amqplib/-/amqplib-0.10.3.tgz", + "integrity": "sha512-UHmuSa7n8vVW/a5HGh2nFPqAEr8+cD4dEZ6u9GjP91nHfr1a54RyAKyra7Sb5NH7NBKOUlyQSMXIp0qAixKexw==", + "requires": { + "@acuminous/bitsyntax": "^0.1.2", + "buffer-more-ints": "~1.0.0", + "readable-stream": "1.x >=1.1.9", + "url-parse": "~1.5.10" + } + }, "anymatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.2.tgz", @@ -608,6 +753,11 @@ "fill-range": "^7.0.1" } }, + "buffer-more-ints": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/buffer-more-ints/-/buffer-more-ints-1.0.0.tgz", + "integrity": "sha512-EMetuGFz5SLsT0QTnXzINh4Ksr+oo4i+UGTXEshiGCQWnsgSs7ZhJ8fzlwQ+OzEMs0MpDAMr1hxnblp5a4vcHg==" + }, "chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -649,6 +799,11 @@ "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", "dev": true }, + "core-util-is": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.3.tgz", + "integrity": "sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ==" + }, "debug": { "version": "3.2.7", "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.7.tgz", @@ -705,6 +860,11 @@ "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", "dev": true }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -735,6 +895,11 @@ "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", "dev": true }, + "isarray": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz", + "integrity": "sha512-D2S+3GLxWH+uhrNEcoh/fnmYeP8E8/zHl644d/jdA0g2uyXvy3sb0qxotE+ne0LtccHknQzWwZEzhak7oJ0COQ==" + }, "minimatch": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", @@ -795,6 +960,22 @@ "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", "dev": true }, + "querystringify": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", + "integrity": "sha512-FIqgj2EUvTa7R50u0rGsyTftzjYmv/a3hO345bZNrqabNqjtgiDMgmo4mkUjd+nzU5oF3dClKqFIPUKybUyqoQ==" + }, + "readable-stream": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-1.1.14.tgz", + "integrity": "sha512-+MeVjFf4L44XUkhM1eYbD8fyEsxcV81pqMSR5gblfcLCHfZvbrqy4/qYHE+/R5HoBUT11WV5O08Cr1n3YXkWVQ==", + "requires": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.1", + "isarray": "0.0.1", + "string_decoder": "~0.10.x" + } + }, "readdirp": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", @@ -817,6 +998,16 @@ "@redis/time-series": "1.0.3" } }, + "requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==" + }, + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "semver": { "version": "5.7.1", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", @@ -840,6 +1031,11 @@ } } }, + "string_decoder": { + "version": "0.10.31", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-0.10.31.tgz", + "integrity": "sha512-ev2QzSzWPYmy9GuqfIVildA4OdcGLeFZQrq5ys6RtiuF+RQQiZWr8TZNyAcuVXyQRYfEO+MsoB/1BuQVhOJuoQ==" + }, "supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -873,6 +1069,15 @@ "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", "dev": true }, + "url-parse": { + "version": "1.5.10", + "resolved": "https://registry.npmjs.org/url-parse/-/url-parse-1.5.10.tgz", + "integrity": "sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ==", + "requires": { + "querystringify": "^2.1.1", + "requires-port": "^1.0.0" + } + }, "ws": { "version": "7.5.9", "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.9.tgz", diff --git a/package.json b/package.json index 59aa214..db5c3a2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s-prerender", - "version": "1.0.8", + "version": "1.0.9", "type": "module", "description": "s-prerender is the Service to solve SPA problems with SEO", "main": "src/index.js", @@ -13,6 +13,7 @@ "homepage": "https://github.com/sajera/s-prerender#s-prerender", "license": "MIT", "dependencies": { + "amqplib": "^0.10.3", "chrome-remote-interface": "^0.31.3", "dotenv": "^16.0.1", "redis": "^4.2.0" diff --git a/src/api/index.js b/src/api/index.js index 58eef07..1f3d0b5 100644 --- a/src/api/index.js +++ b/src/api/index.js @@ -4,7 +4,7 @@ import url from 'node:url'; import { createServer } from 'node:http'; // local dependencies -import { logError, log, suid, DEBUG } from '../config.js'; +import { logError, log, suid } from '../config.js'; // NOTE required interface for "api" export default { start, isReady, middleware }; @@ -18,7 +18,7 @@ export function start (config) { isReady() && api.close(() => log('[api:stopped]')); return new Promise(resolve => { api = createServer(middleware); - log('[api:starting]', config); + log('[api:connecting]', config); api.listen(config.port, config.host, () => { log('[api:started]', `http://${config.host}:${config.port}/`); resolve(); diff --git a/src/cache/index.js b/src/cache/index.js index 6b8a573..6e3f431 100644 --- a/src/cache/index.js +++ b/src/cache/index.js @@ -1,7 +1,20 @@ +// local dependencies import redis from './redis.js'; -// TODO switching caches based on environment -// NOTE for now implemented only Redis ¯\_(ツ)_/¯ +// configure +const noop = () => null; +// NOTE module interface +const cache = { start, isReady: noop, set: noop, get: noop, del: noop }; -export default redis; +// NOTE switching based on environment variables +function start (config) { + if (config.redis) { + Object.assign(cache, redis); + return redis.start(config); + } + // NOTE for now implemented only Redis ¯\_(ツ)_/¯ + // throw new Error('No useful CACHE configuration found'); +} + +export default cache; diff --git a/src/cache/redis.js b/src/cache/redis.js index 3c50c9e..1fe5a80 100644 --- a/src/cache/redis.js +++ b/src/cache/redis.js @@ -18,12 +18,12 @@ export function isReady () { return CONNECTED; } export async function start (config) { log('[redis:connecting]', config); - client = createClient(config); - client.on('connect', () => debug('[redis:start]', CONNECTED = true)); - client.on('ready', () => debug('[redis:ready]')); + client = createClient({ url: config.redisUrl }); + client.on('connect', () => debug('[redis:start]')); + client.on('ready', () => debug('[redis:connected]', CONNECTED = true)); client.on('end', () => debug('[redis:stopped]', CONNECTED = false)); client.on('error', error => logError('REDIS', { message: error.message, stack: error.stack })); await client.connect(); - log('[redis:connected]'); + log('[redis:started]', config.redisUrl); } diff --git a/src/config.js b/src/config.js index 0b2ee51..959cc0e 100644 --- a/src/config.js +++ b/src/config.js @@ -2,29 +2,41 @@ // outsource dependencies import dotenv from 'dotenv'; -// local dependencies - +// NOTE log unhandled promise exception +process.on('unhandledRejection', error => logError('[service:unhandledRejection]', error && { + message: error.message, + stack: error.stack, + code: error.code, +})); +// NOTE log process exception +process.on('uncaughtException', error => logError('[service:uncaughtException]', error && { + message: error.message, + stack: error.stack, + code: error.code, +}) || process.exit(1)); +// NOTE strict dotenv rules to avoid unexpected process environment - .env is defaults with minimal priority dotenv.config({ override: false, debug: varBoolean(process.env.DEBUG) }); +// NOTE reading any variables only after reading defaults to make sure the minimal required data was set export const DEBUG = varBoolean(process.env.DEBUG); - +// NOTE export const API = { port: varNumber(process.env.PORT), host: varString(process.env.HOST), allowDomains: varArray(process.env.ALLOW_DOMAINS) || ['.'], }; +// NOTE for now RabbitMQ only +export const QUEUE = { + rabbitmq: Boolean(process.env.RABBITMQ_URL), + rabbitmqUrl: varString(process.env.RABBITMQ_URL), + rabbitmqQueue: varString(process.env.RABBITMQ_QUEUE), + rabbitmqChannels: varNumber(process.env.RABBITMQ_CHANNELS), +}; // NOTE for now Redis only export const CACHE = { - url: varString(process.env.REDIS_URL), - // name: process.env.REDIS_NAME, - // database: process.env.REDIS_DB, - // username: process.env.REDIS_USERNAME, - // password: process.env.REDIS_PASSWORD, - // commandsQueueMaxLength?: number; - // disableOfflineQueue?: boolean; - // readonly?: boolean; - // legacyMode?: boolean; - // isolationPoolOptions?: PoolOptions; + redis: Boolean(process.env.REDIS_URL), + redisUrl: varString(process.env.REDIS_URL), }; +// NOTE Chrome/Chromium only export const PRERENDER = { browserDebuggingPort: varNumber(process.env.CHROME_DEBUGGING_PORT), chromeLocation: varString(process.env.CHROME_BIN), @@ -37,6 +49,7 @@ export const PRERENDER = { followRedirects: varBoolean(process.env.CHROME_FOLLOW_REDIREC), // Weather to follow redirect cleanupHtmlScript: varString(process.env.CHROME_CLEANUP_HTML_SCRIPT) || defaultCleanupHtmlScript(), // ability to pass string with JS to execute on all pages }; +export const config = { API, CACHE, PRERENDER, QUEUE }; /****************************************************** * variables parsers *****************************************************/ @@ -52,6 +65,7 @@ export function varArray (value) { export function varString (value) { return /^(null|undefined)$/i.test(value) ? void 0 : value; } +// TODO way to setup js for rendered page export function defaultCleanupHtmlScript () { return `(tags => { for(const tag of tags) { diff --git a/src/index.js b/src/index.js index 58dc3bc..5c1921d 100644 --- a/src/index.js +++ b/src/index.js @@ -5,30 +5,21 @@ import qs from 'node:querystring'; // local dependencies import api from './api/index.js'; import cache from './cache/index.js'; +import queue from './queue/index.js'; import prerender from './prerender/index.js'; -import { isUrl, logError, log, API, CACHE, PRERENDER } from './config.js'; - -// NOTE log unhandled promise exception -process.on('unhandledRejection', error => logError('[service:unhandledRejection]', error && { - message: error.message, - stack: error.stack, - code: error.code, -})); -// NOTE log process exception -process.on('uncaughtException', error => logError('[service:uncaughtException]', error && { - message: error.message, - stack: error.stack, - code: error.code, -}) || process.exit(1)); +import { isUrl, logError, log, config } from './config.js'; // configure let READY; +const services = { API: api, CACHE: cache, PRERENDER: prerender, QUEUE: queue }; +const servicesIds = Object.keys(services); // run all services -Promise.all([ - api.start(API), - cache.start(CACHE), - prerender.start(PRERENDER), -]).then(() => log('[service:ready]', READY = true)); +Promise.all(servicesIds.map(id => services[id].start(config[id]))) + .then(() => log('[service:started]', READY = true)) + .catch(error => { + logError('SERVICES', error.message); + process.exit(100500); + }); /****************************************************** * GET /health @@ -38,7 +29,7 @@ health.contentType = 'application/json'; function health () { let ready = false; try { - checkReadyState(); + checkReadyState(servicesIds); ready = true; } catch (e) { } return JSON.stringify({ status: ready ? 'UP' : 'DOWN' }); @@ -50,30 +41,40 @@ function health () { api.middleware.GET['/render'] = render; render.contentType = 'text/html'; async function render (request) { - checkReadyState(); - const url = validUrl(request); - const results = await cache.get(url); - if (results) { - log('[api:from-cache]', url); - return results; + try { + return await getCached(request); + } catch (error) { + return getRefresh(request); } - log('[api:no-cache]', url); - return refresh(request); } /****************************************************** * GET /refresh?url=http://example.com/ *****************************************************/ -api.middleware.GET['/refresh'] = refresh; -refresh.contentType = 'text/html'; -async function refresh (request) { - checkReadyState(); +api.middleware.GET['/refresh'] = getRefresh; +getRefresh.contentType = 'text/html'; +async function getRefresh (request) { + checkReadyState(['CACHE', 'PRERENDER']); const url = validUrl(request); const results = await prerender.render(url); log('[api:generate]', url); await cache.set(url, results); log('[api:to-cache]', url); - return results; + return qs.parse(request.url.query).ignoreResults ? 'OK' : results; +} + +/****************************************************** + * POST /refresh + *****************************************************/ +api.middleware.POST['/refresh'] = postRefresh; +postRefresh.contentType = 'text/html'; +async function postRefresh (request) { + checkReadyState(['CACHE', 'QUEUE']); + const urls = await parseUrls(request); + // TODO message schema + await queue.sendToQueue(urls); + log('[api:to-queue]', urls.length); + return 'OK'; } /****************************************************** @@ -82,10 +83,11 @@ async function refresh (request) { api.middleware.GET['/cached'] = getCached; getCached.contentType = 'text/html'; async function getCached (request) { - checkReadyState(); + checkReadyState(['CACHE']); const url = validUrl(request); const results = await cache.get(url); if (!results) { throw { code: 404, message: `Cache empty for "${url}"` }; } + log('[api:from-cache]', url); return results; } @@ -95,24 +97,44 @@ async function getCached (request) { api.middleware.DELETE['/cached'] = deleteCached; deleteCached.contentType = 'text/plain'; async function deleteCached (request) { - checkReadyState(); + checkReadyState(['CACHE']); const url = validUrl(request); await cache.del(url); return 'OK'; } -/****************************************************** - * /////////////////// - *****************************************************/ -function checkReadyState () { - if (!api.isReady()) { throw { code: 503, message: 'Service(API) not ready yet' }; } - if (!cache.isReady()) { throw { code: 503, message: 'Service(CACHE) not ready yet' }; } - if (!prerender.isReady()) { throw { code: 503, message: 'Service(PRERENDER) not ready yet' }; } - return true; +/********************************************* + * //////// THROW /////////// * + *********************************************/ +function checkReadyState (required = servicesIds) { + return required.map(id => { + if (!services[id].isReady()) { + throw { code: 503, message: `Service(${id}) unavailable` }; + } + }); } function validUrl (request) { const url = qs.unescape(qs.parse(request.url.query).url); - if (!isUrl(url)) { throw { code: 400, message: `Invalid query parameter url "${url}"` }; } + if (!isUrl(url)) { throw { code: 422, message: `Unsupported url "${url}"` }; } return url; } + +const parseUrls = request => new Promise((resolve, reject) => { + let data = ''; + request.on('data', chunk => data += chunk); + request.on('end', () => { + let urls = []; + try { urls = JSON.parse(data); } catch (error) { + logError('[api:parseUrls]', { message: error.message, stack: error.stack }); + } + if (!Array.isArray(urls)) { reject({ code: 422, message: 'Invalid data, expected an Array with URLs' }); } + const obj = {}; + for (const url of urls) { obj[qs.unescape(url)] = 1; } + urls = []; + // NOTE only uniq and valid url + for (const url in obj) { isUrl(url) && urls.push(url); } + if (!urls.length) { reject({ code: 422, message: 'Invalid data, expected an Array with URLs' }); } + resolve(urls); + }); +}); diff --git a/src/prerender/chrome.js b/src/prerender/chrome.js index cd5f508..9046bd6 100644 --- a/src/prerender/chrome.js +++ b/src/prerender/chrome.js @@ -44,7 +44,7 @@ chrome.connect = () => new Promise((resolve, reject) => { clearTimeout(timeout); connected = true; debug('[prerender:ready]', info); - resolve(info); + resolve(info['User-Agent']); }).catch(error => debug('[prerender:connect] Retrying connection to browser...', error, setTimeout(retry, 4e3))); setTimeout(retry, 0); }); diff --git a/src/prerender/index.js b/src/prerender/index.js index 0312729..40173cd 100644 --- a/src/prerender/index.js +++ b/src/prerender/index.js @@ -7,7 +7,7 @@ import { chrome as browser } from './chrome.js'; import { DEBUG, debug, delay, log, suid } from '../config.js'; // NOTE required interface for "prerender" -export default { start, render, isReady }; +export default { start, isReady, render }; // configure let CONNECTED; @@ -19,13 +19,13 @@ process.on('SIGINT', () => { export function isReady () { return CONNECTED; } export async function start (config) { - log('[prerender:starting]', config); + log('[prerender:connecting]', config); browser.kill(); await browser.spawn(config); - browser.onClose(() => log('[prerender:stopped]', CONNECTED = false)); - await browser.connect(); + browser.onClose(() => log('[prerender:stopped]', CONNECTED = false, start(config))); + const info = await browser.connect(); CONNECTED = true; - log('[prerender:started]'); + log('[prerender:started]', info); } // TODO ERROR:[service:unhandledRejection] diff --git a/src/queue/index.js b/src/queue/index.js new file mode 100644 index 0000000..40316f3 --- /dev/null +++ b/src/queue/index.js @@ -0,0 +1,20 @@ + +// local dependencies +import rabbitmq from './rabbitmq.js'; + +// configure +const noop = () => null; +// NOTE module interface +const queue = { start, isReady: noop, sendToQueue: noop }; + +// NOTE switching based on environment variables +function start (config) { + if (config.rabbitmq) { + Object.assign(queue, rabbitmq); + return rabbitmq.start(config); + } + // NOTE for now implemented only RabbitMQ ¯\_(ツ)_/¯ + // throw new Error('No useful QUEUE configuration found'); +} + +export default queue; diff --git a/src/queue/rabbitmq.js b/src/queue/rabbitmq.js new file mode 100644 index 0000000..a757709 --- /dev/null +++ b/src/queue/rabbitmq.js @@ -0,0 +1,91 @@ + +// outsource dependencies +import amqp from 'amqplib'; +import http from 'node:http'; +import { Buffer } from 'node:buffer'; + +// local dependencies +import { logError, debug, log, config } from '../config.js'; + +// NOTE required interface for "queue" +export default { start, isReady, sendToQueue }; + +// configure +let client; +let CONNECTED; +export function isReady () { return CONNECTED; } +export async function sendToQueue (urls) { + const channel = await client.createChannel(); + await channel.assertQueue(client.queue, client.queueOptions); + debug(`[rabbitmq:sendToQueue] ${client.queue}`, urls); + for (const url of urls) { + channel.sendToQueue(client.queue, formatMessage({ url }), client.sendOptions); + } +} + +export async function start (config) { + log('[rabbitmq:connecting]', config); + await connect(config.rabbitmqUrl); + // NOTE configuration + client.queue = config.rabbitmqQueue; + client.queueOptions = { durable: true }; + client.sendOptions = { persistent: true }; + client.consumeOptions = { noAck: false }; + client.on('close', () => debug('[rabbitmq:stopped]', CONNECTED = false, start(config))); + client.on('error', error => logError('RABBITMQ', { message: error.message, stack: error.stack })); + const channel = await client.createConfirmChannel(); + channel.prefetch(config.rabbitmqChannels); + await channel.assertQueue(client.queue, client.queueOptions); + channel.consume(client.queue, message(channel), client.consumeOptions); + log('[rabbitmq:started]', config.rabbitmqUrl); +} + +const message = channel => async message => { + const { url, attempt = 0 } = parseMessage(message.content.toString()); + try { + log(`[rabbitmq:message] ${attempt}`, message.content.toString()); + await refresh(url); + } catch (error) { + logError('[rabbitmq:message]', error.message); + // NOTE try again later + if (url && attempt > 0) { + try { + // NOTE create new message in a queue + channel.sendToQueue(client.queue, formatMessage({ url, attempt: attempt - 1 }), client.sendOptions); + } catch (error) { /* NOTE unbelievable, but just in case */ } + } + } + // NOTE at any case message was handled + channel.ack(message); +} +/****************************************************** + * HELPERS + *****************************************************/ +const refresh = url => new Promise((resolve, reject) => { + http.get(new URL(`http://${config.API.host}:${config.API.port}/refresh?ignoreResults=true&url=${url}`), response => { + // NOTE consume response data to free up memory + response.resume(); + // NOTE no addition data or explanations need due to debug from API logs. + return response.statusCode === 200 ? resolve('OK') + : reject(new Error(`Request Failed. [${response.statusCode}] Failed to refresh "${url}"`)); + }); +}); + +const formatMessage = ({ url, attempt = 3 }) => Buffer.from(JSON.stringify({ attempt, url })); +const parseMessage = message => { + try { + return JSON.parse(message.toString()) || {}; + } catch (error) { + return {}; + } +} + +const connect = url => new Promise(resolve => { + const retry = () => amqp.connect(url) + .then(connection => { + debug('[rabbitmq:connected]', CONNECTED = true); + resolve(client = connection); + }) + .catch(error => logError('RABBITMQ', { message: error.message, stack: error.stack }, setTimeout(retry, 4e3))); + retry(); +});