From 39ebb4a83d51938b80b287550900fc86540bceed Mon Sep 17 00:00:00 2001 From: zaanposni Date: Sun, 1 Dec 2024 12:12:52 +0100 Subject: [PATCH 1/3] removed avif due to heavy cpu load in convertion --- src/imageresizer/src/index.ts | 40 ++++++------------- .../src/lib/components/CDNImage.svelte | 25 +----------- .../src/lib/components/InstagramPost.svelte | 2 +- .../InstagramStoriesContainer.svelte | 3 +- 4 files changed, 16 insertions(+), 54 deletions(-) diff --git a/src/imageresizer/src/index.ts b/src/imageresizer/src/index.ts index 73e47e2..019810e 100644 --- a/src/imageresizer/src/index.ts +++ b/src/imageresizer/src/index.ts @@ -21,7 +21,7 @@ app.get("/_health", (_, res) => { app.get("/cdn/:dir/:filename", async (req: Request<{ dir: string; filename: string }>, res: Response) => { let { dir, filename } = req.params; - const { width, format } = req.query; + const { width } = req.query; if (!filename) { logger.error("Missing filename parameter"); @@ -46,7 +46,6 @@ app.get("/cdn/:dir/:filename", async (req: Request<{ dir: string; filename: stri } // Validate parameters - const validFormats = ["avif", "jpg"]; const validWidths = ["300", "768", "original"]; if (width && !validWidths.includes(width.toString())) { @@ -55,36 +54,21 @@ app.get("/cdn/:dir/:filename", async (req: Request<{ dir: string; filename: stri return; } - if (format && !validFormats.includes(format.toString())) { - logger.error(`Invalid format parameter: "${format}"`); - res.status(400).send("Invalid format parameter, must be one of: avif, jpg"); - return; - } - - const targetFormat = (format as "avif" | "jpg") || "jpg"; const targetWidth = width ? (width === "original" ? undefined : parseInt(width.toString())) : undefined; - const originalFilePath = path.join(cdnFiles, filename); - if (!fs.existsSync(originalFilePath)) { - logger.error(`Original Resource not found: "${originalFilePath}"`); - res.status(404).send("Resource not found"); - return; - } - // convert image to specified format and size - let specificFileName = filename; - if (targetWidth !== undefined || targetFormat !== "jpg") { - if (targetWidth) { - specificFileName = `${filename.split(".")[0]}-w${targetWidth}.${targetFormat}`; - } else { - specificFileName = `${filename.split(".")[0]}.${targetFormat}`; - } - } - + const originalFilePath = path.join(cdnFiles, filename); + const specificFileName = targetWidth === undefined ? filename : `${filename.split(".")[0]}-w${targetWidth}.jpg`; const specificFilePath = path.join(cdnFiles, specificFileName); logger.info(`Serving image: "${specificFilePath}"`); if (!fs.existsSync(specificFilePath)) { + if (specificFileName === filename && !fs.existsSync(originalFilePath)) { + logger.error(`Original Resource not found: "${originalFilePath}"`); + res.status(404).send("Resource not found"); + return; + } + logger.info(`Generating image: "${specificFilePath}" from "${originalFilePath}"`); try { @@ -93,17 +77,17 @@ app.get("/cdn/:dir/:filename", async (req: Request<{ dir: string; filename: stri image.resize(targetWidth); } - image.toFormat(targetFormat, { quality: 100 }); + image.toFormat("jpg", { quality: 100 }); await image.toFile(specificFilePath); } catch (error) { - logger.error(`Failed to generate image: "${specificFilePath}"`); + logger.error(`Failed to generate image: "${specificFilePath}"`, error); res.status(500).send("Failed to generate image"); return; } } - res.contentType(`image/${targetFormat}`); + res.contentType(`image/jpg`); res.sendFile(specificFilePath); }); diff --git a/src/psaggregator/src/lib/components/CDNImage.svelte b/src/psaggregator/src/lib/components/CDNImage.svelte index 1fd2900..2505e4d 100644 --- a/src/psaggregator/src/lib/components/CDNImage.svelte +++ b/src/psaggregator/src/lib/components/CDNImage.svelte @@ -13,29 +13,6 @@ let imageSrc = ""; - function supportsAVIF() { - return new Promise((resolve) => { - const avifImage = - "data:image/avif;base64,AAAAIGZ0eXBhdmlmAAAAAGF2aWZtaWYxbWlhZk1BMUIAAADybWV0YQAAAAAAAAAoaGRscgAAAAAAAAAAcGljdAAAAAAAAAAAAAAAAGxpYmF2aWYAAAAADnBpdG0AAAAAAAEAAAAeaWxvYwAAAABEAAABAAEAAAABAAABGgAAAB0AAAAoaWluZgAAAAAAAQAAABppbmZlAgAAAAABAABhdjAxQ29sb3IAAAAAamlwcnAAAABLaXBjbwAAABRpc3BlAAAAAAAAAAIAAAACAAAAEHBpeGkAAAAAAwgICAAAAAxhdjFDgQ0MAAAAABNjb2xybmNseAACAAIAAYAAAAAXaXBtYQAAAAAAAAABAAEEAQKDBAAAACVtZGF0EgAKCBgANogQEAwgMg8f8D///8WfhwB8+ErK42A="; - const img = new Image(); - img.onload = () => resolve(true); - img.onerror = () => resolve(false); - img.src = avifImage; - }); - } - - async function getPreferredImageFormat() { - if (!browser || !window || !window.matchMedia) { - return "jpg"; - } - - if (await supportsAVIF()) { - return "avif"; - } - - return "jpg"; // fallback - } - function getPreferredImageSize(): "300" | "768" | "original" { if (!browser || !window || !window.matchMedia) { return "300"; @@ -63,7 +40,7 @@ } async function calculateImageSrc(src: string | null | undefined) { - imageSrc = `${src}?format=${await getPreferredImageFormat()}&width=${getPreferredImageSize()}`; + imageSrc = `${src}?width=${getPreferredImageSize()}`; } $: browser && calculateImageSrc(src); diff --git a/src/psaggregator/src/lib/components/InstagramPost.svelte b/src/psaggregator/src/lib/components/InstagramPost.svelte index 26ff395..a9ae447 100644 --- a/src/psaggregator/src/lib/components/InstagramPost.svelte +++ b/src/psaggregator/src/lib/components/InstagramPost.svelte @@ -55,7 +55,7 @@ src={resource.imageUri} alt={resource.remoteId} title={resource.remoteId} - loading="lazy" /> + {loading} /> {/each} diff --git a/src/psaggregator/src/lib/components/InstagramStoriesContainer.svelte b/src/psaggregator/src/lib/components/InstagramStoriesContainer.svelte index bb398ed..b3bc8e7 100644 --- a/src/psaggregator/src/lib/components/InstagramStoriesContainer.svelte +++ b/src/psaggregator/src/lib/components/InstagramStoriesContainer.svelte @@ -268,7 +268,8 @@ src={selectedStory.imageUri} alt={selectedStory.text} title={selectedStory.text} - class="rounded-[20px] object-contain" /> + class="rounded-[20px] object-contain" + loading="eager" /> {/if} {/if} From c2f9adc7651d6fcd8a83d2a85a0019312684b653 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Sun, 1 Dec 2024 12:19:20 +0100 Subject: [PATCH 2/3] fix twitter import --- src/dataimport/Dockerfile | 23 +++++++++-------------- src/dataimport/requirements.txt | 2 +- src/dataimport/twitter.py | 2 +- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/dataimport/Dockerfile b/src/dataimport/Dockerfile index 462019d..01867e4 100644 --- a/src/dataimport/Dockerfile +++ b/src/dataimport/Dockerfile @@ -8,30 +8,25 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Install Firefox RUN apt-get update && \ - apt-get install -y --no-install-recommends firefox-esr && \ - rm -rf /var/lib/apt/lists/* + apt-get install -y --no-install-recommends firefox-esr # Install GeckoDriver -RUN apt-get update && \ - apt-get install -y --no-install-recommends wget unzip && \ +RUN apt-get install -y --no-install-recommends wget unzip && \ wget https://github.com/mozilla/geckodriver/releases/download/v0.30.0/geckodriver-v0.30.0-linux64.tar.gz && \ tar -xvzf geckodriver-v0.30.0-linux64.tar.gz && \ rm geckodriver-v0.30.0-linux64.tar.gz && \ chmod +x geckodriver && \ mv geckodriver /usr/local/bin/ && \ - apt-get remove -y wget unzip && \ - rm -rf /var/lib/apt/lists/* + apt-get remove -y wget unzip -# Install rsyslog -RUN apt-get update && apt-get install -y rsyslog -COPY rsyslog.conf /etc/rsyslog.conf -RUN chmod 644 /etc/rsyslog.conf +# Install deps +RUN apt-get install -y rsyslog cron dos2unix libmagic1 -# Install cron -RUN apt-get update && apt-get install -y cron +# Cleanup +RUN apt-get clean && rm -rf /var/lib/apt/lists/* -# Install dos2unix -RUN apt-get update && apt-get install -y dos2unix +COPY rsyslog.conf /etc/rsyslog.conf +RUN chmod 644 /etc/rsyslog.conf COPY . /app diff --git a/src/dataimport/requirements.txt b/src/dataimport/requirements.txt index a4a9a2c..f791ad9 100644 --- a/src/dataimport/requirements.txt +++ b/src/dataimport/requirements.txt @@ -13,7 +13,7 @@ pyotp==2.9.0 instagrapi==2.0.1 pillow==10.3.0 blinker==1.7.0 -tweety-ns==1.1.9 +tweety-ns==2.0.6 httpx==0.27.2 h2==4.1.0 httpx[http2] diff --git a/src/dataimport/twitter.py b/src/dataimport/twitter.py index 4ba9b90..fbca462 100644 --- a/src/dataimport/twitter.py +++ b/src/dataimport/twitter.py @@ -138,7 +138,7 @@ async def twitter(): console.log("Fetching last tweets of list...") - contents = app.get_list_tweets(LIST_ID) + contents = await app.get_list_tweets(LIST_ID) for content in contents: if isinstance(content, Tweet): await handle_tweet(content, db) From 549255a5f8ae043725ed46aec52352863250b240 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Sun, 1 Dec 2024 12:20:12 +0100 Subject: [PATCH 3/3] bump version --- src/psaggregator/package-lock.json | 4 ++-- src/psaggregator/package.json | 2 +- src/psaggregator/src/routes/changelog/+page.svelte | 9 +++++++++ 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/psaggregator/package-lock.json b/src/psaggregator/package-lock.json index 62dd91c..53564c4 100644 --- a/src/psaggregator/package-lock.json +++ b/src/psaggregator/package-lock.json @@ -1,12 +1,12 @@ { "name": "psaggregator", - "version": "1.19.0", + "version": "1.19.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "psaggregator", - "version": "1.19.0", + "version": "1.19.1", "dependencies": { "@internationalized/date": "^3.5.5", "@sentry/sveltekit": "^8.41.0", diff --git a/src/psaggregator/package.json b/src/psaggregator/package.json index c65c790..d3955f0 100644 --- a/src/psaggregator/package.json +++ b/src/psaggregator/package.json @@ -1,6 +1,6 @@ { "name": "psaggregator", - "version": "1.19.0", + "version": "1.19.1", "scripts": { "dev": "vite dev", "build": "vite build", diff --git a/src/psaggregator/src/routes/changelog/+page.svelte b/src/psaggregator/src/routes/changelog/+page.svelte index e3585d3..b5baa92 100644 --- a/src/psaggregator/src/routes/changelog/+page.svelte +++ b/src/psaggregator/src/routes/changelog/+page.svelte @@ -16,6 +16,15 @@
+

Version 1.19.1

+
+

Überarbeitungen

+
+ Twitter-Import + Ein Problem beim Twitter-Import wurde behoben. Tweets sollten nun wieder korrekt importiert werden. +
+
+

Version 1.19.0

Neue Features