diff --git a/.env.example b/.env.example index 6d994cd..0eff420 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,10 @@ MYSQL_DATABASE=psaggregator MYSQL_ROOT_PASSWORD=psaggregator LEGAL_URL= -PUBLIC_KOFI_USERNAME=zaanposni +KOFI_USERNAME=zaanposni OPENAI_API_KEY= + +INSTAGRAM_USERNAME = +INSTAGRAM_PASSWORD = +INSTAGRAM_2FA_SECRET = diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 05a33a0..51e8dc4 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -49,8 +49,13 @@ services: - OPENAI_API_KEY=${OPENAI_API_KEY} - SQLALCHEMY_SILENCE_UBER_WARNING=1 - YT_SERVER_BASE_URL=http://youtube-api + - INSTAGRAM_USERNAME=${INSTAGRAM_USERNAME} + - INSTAGRAM_PASSWORD=${INSTAGRAM_PASSWORD} + - INSTAGRAM_2FA_SECRET=${INSTAGRAM_2FA_SECRET} + - INSTAGRAM_CONFIG_PATH=/app/config/instagram.json volumes: - shared-data:/app/cdn + - config:/app/config depends_on: - db - frontend @@ -90,6 +95,7 @@ networks: volumes: mysql: + config: shared-data: driver: local driver_opts: diff --git a/docker-compose.yml b/docker-compose.yml index b22fabc..d4c00ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,8 +43,13 @@ services: - OPENAI_API_KEY=${OPENAI_API_KEY} - SQLALCHEMY_SILENCE_UBER_WARNING=1 - YT_SERVER_BASE_URL=http://youtube-api + - INSTAGRAM_USERNAME=${INSTAGRAM_USERNAME} + - INSTAGRAM_PASSWORD=${INSTAGRAM_PASSWORD} + - INSTAGRAM_2FA_SECRET=${INSTAGRAM_2FA_SECRET} + - INSTAGRAM_CONFIG_PATH=/app/config/instagram.json volumes: - shared-data:/app/cdn + - config:/app/config depends_on: - db - frontend @@ -82,6 +87,7 @@ networks: volumes: mysql: + config: shared-data: driver: local driver_opts: diff --git a/readme.md b/readme.md index 2ccd87e..cfcca55 100644 --- a/readme.md +++ b/readme.md @@ -7,6 +7,15 @@ PietSmiet Aggregator is a selfhostable web application that aggregates all the videos, streams and additional content from PietSmiet and displays them in a nice overview. +⭐ **Dashboard** - Display all PietSmiet videos and streams in a nice overview.\ +⭐ **API** - Free and Public JSON HTTP API.\ +⭐ **PietSmietDE Import** - Import all videos and news from the PietSmiet website.\ +⭐ **YouTube Import** - Import all community posts from the PietSmiet YouTube channel.\ +⭐ **Reddit Import** - Import trending posts from r/pietsmiet.\ +⭐ **Instagram Import** - Import all posts.\ +⭐ **OpenAI Streamingplan Analysis** - Analyze the Streamingplan with OpenAI Vision and import scheduled streams into the database.\ +⭐ **Full Control** - Selfhostablem, Open Source and Dockerized. + ## ⚡️ Quick Start Self Deployment First, [download][docker_download_url] and install **Docker**. diff --git a/src/dataimport/.env.example b/src/dataimport/.env.example index be51dc4..73b19c9 100644 --- a/src/dataimport/.env.example +++ b/src/dataimport/.env.example @@ -7,3 +7,7 @@ TWITCH_CLIENT_ID= TWITCH_CLIENT_SECRET= OPENAI_API_KEY= + +INSTAGRAM_USERNAME = +INSTAGRAM_PASSWORD = +INSTAGRAM_2FA_SECRET = diff --git a/src/dataimport/.gitignore b/src/dataimport/.gitignore index 55b1e95..1b1d690 100644 --- a/src/dataimport/.gitignore +++ b/src/dataimport/.gitignore @@ -1,7 +1,7 @@ geckodriver.exe *.json -instagram.py openai_test.py .env videos.sql __pycache__/ +threads.py diff --git a/src/dataimport/Dockerfile b/src/dataimport/Dockerfile index 37d42d2..462019d 100644 --- a/src/dataimport/Dockerfile +++ b/src/dataimport/Dockerfile @@ -37,6 +37,7 @@ COPY . /app # Install any needed packages specified in requirements.txt RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --no-deps --force-reinstall pydantic==1.10.13 RUN touch /var/log/cron.log COPY hello-cron /etc/cron.d/hello-cron diff --git a/src/dataimport/hello-cron b/src/dataimport/hello-cron index bd12615..52268d7 100644 --- a/src/dataimport/hello-cron +++ b/src/dataimport/hello-cron @@ -3,9 +3,10 @@ 35 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdevideoimporter.py >> /var/log/cron.log 2>&1 */15 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdevideoimporter.py >> /var/log/cron.log 2>&1 5 * * * * . /root/project_env.sh; /usr/local/bin/python /app/youtube.py >> /var/log/cron.log 2>&1 +5 * * * * . /root/project_env.sh; /usr/local/bin/python /app/instagram.py >> /var/log/cron.log 2>&1 */15 * * * * . /root/project_env.sh; /usr/local/bin/python /app/reddit.py >> /var/log/cron.log 2>&1 */1 * * * * . /root/project_env.sh; /usr/local/bin/python /app/twitch.py >> /var/log/cron.log 2>&1 -1 1 * * * . /root/project_env.sh; /usr/local/bin/python /app/informationopenaianalyze.py >> /var/log/cron.log 2>&1 +10 1 * * * . /root/project_env.sh; /usr/local/bin/python /app/informationopenaianalyze.py >> /var/log/cron.log 2>&1 2 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdeuploadplan.py >> /var/log/cron.log 2>&1 5 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdeuploadplan.py >> /var/log/cron.log 2>&1 30 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdeuploadplan.py >> /var/log/cron.log 2>&1 diff --git a/src/dataimport/instagram.py b/src/dataimport/instagram.py new file mode 100644 index 0000000..15d2efe --- /dev/null +++ b/src/dataimport/instagram.py @@ -0,0 +1,207 @@ +import asyncio +import os +import requests +from databases import Database +from uuid import uuid4 + +from rich.console import Console +from instagrapi import Client +from instagrapi.exceptions import LoginRequired +import pyotp + + +console = Console() + +USERNAME = os.getenv("INSTAGRAM_USERNAME") +PASSWORD = os.getenv("INSTAGRAM_PASSWORD") +KEY_2FA = os.getenv("INSTAGRAM_2FA_SECRET") +CONFIG_PATH = os.getenv("INSTAGRAM_CONFIG_PATH") +if not CONFIG_PATH: + CONFIG_PATH = "session.json" +console.log(f"Using config path {CONFIG_PATH}") + +if not USERNAME or not PASSWORD: + raise Exception("No Instagram username or password provided") + + +def login_user(): + """ + Attempts to login to Instagram using either the provided session information + or the provided username and password. + """ + + console.log("Attempting to login user...") + if not os.path.exists(CONFIG_PATH): + console.log("No session file found, creating empty session file") + with open(CONFIG_PATH, "w") as f: + f.write("{}") + + cl = Client() + session = cl.load_settings(CONFIG_PATH) + + login_via_session = False + login_via_pw = False + + if session: + try: + cl.set_settings(session) + + code_2fa = "" + if KEY_2FA: + code_2fa = pyotp.TOTP(KEY_2FA.replace(" ", "")).now() + cl.login(USERNAME, PASSWORD, verification_code=code_2fa) + + # check if session is valid + try: + cl.get_timeline_feed() + except LoginRequired: + console.log( + "Session is invalid, need to login via username and password" + ) + + old_session = cl.get_settings() + + # use the same device uuids across logins + cl.set_settings({}) + cl.set_uuids(old_session["uuids"]) + + code_2fa = "" + if KEY_2FA: + code_2fa = pyotp.TOTP(KEY_2FA.replace(" ", "")).now() + cl.login(USERNAME, PASSWORD, verification_code=code_2fa) + login_via_session = True + except Exception as e: + console.log("Couldn't login user using session information: %s" % e) + + if not login_via_session: + try: + console.log( + "Attempting to login via username and password. username: %s" % USERNAME + ) + + code_2fa = "" + if KEY_2FA: + code_2fa = pyotp.TOTP(KEY_2FA.replace(" ", "")).now() + + if cl.login(USERNAME, PASSWORD, verification_code=code_2fa): + login_via_pw = True + except Exception as e: + console.log("Couldn't login user using username and password: %s" % e) + + if not login_via_pw and not login_via_session: + raise Exception("Couldn't login user with either password or session") + + cl.dump_settings(CONFIG_PATH) + return cl + + +cl = login_user() +console.log("Successfully logged in user") + +user_dict = { + "peter": 344058897, + "brammen": 1588473759, + "jay": 2030403724, + "sep": 1609561808, + "chris": 1057433625, +} + +console.log("Fetching last 50 media items for each user") + +INSERT_QUERY_INFORMATION = """ + INSERT INTO Information (id, remoteId, text, additionalInfo, imageUri, href, date, analyzedAt, importedAt, importedFrom) + VALUES (:id, :remoteId, :text, :additionalInfo, :imageUri, :href, :date, NULL, now(), 'Instagram')""" +INSERT_QUERY_RESOURCE = """ + INSERT INTO InformationResource (id, remoteId, informationId, imageUri, videoUri, importedAt, importedFrom) + VALUES (:id, :remoteId, :informationId, :imageUri, :videoUri, now(), 'Instagram')""" +SELECT_QUERY_INFORMATION = """ + SELECT id FROM Information WHERE remoteId = :remoteId AND importedFrom = 'Instagram'""" + + +async def instagram(): + console.log("Connecting to database...", style="bold green") + db = Database(url=os.getenv("DATABASE_URL")) + await db.connect() + for user, user_id in user_dict.items(): + console.log(f"Fetching last 50 media items for {user}") + last_media = cl.user_medias(user_id, amount=50) + console.log(f"Found {len(last_media)} media items for {user}") + for media in last_media: + remote_id = f"{user}_{str(media.id)}" + media_db_id = uuid4() + console.log(f"Processing media item {remote_id}") + if await db.fetch_one( + SELECT_QUERY_INFORMATION, {"remoteId": str(remote_id)} + ): + console.log( + f"Media item {remote_id} already in database", style="bold red" + ) + continue + console.log(f"Media item {remote_id} not in database, inserting") + + thumbnail_url = media.thumbnail_url + if not thumbnail_url: + for resource in media.resources: + if resource.thumbnail_url: + thumbnail_url = resource.thumbnail_url + break + + if not thumbnail_url: + console.log( + f"Media item {remote_id} has no thumbnail, skipping", + style="bold red", + ) + continue + + console.log(f"Downloading thumbnail for {remote_id}") + try: + thumbnail = requests.get(thumbnail_url).content + filename = f"instagram_{uuid4()}.jpg" + with open(f"/app/cdn/{filename}", "wb") as f: + f.write(thumbnail) + thumbnail_url = f"/cdn/{filename}" + except Exception as e: + console.log(f"Error downloading thumbnail: {e}", style="bold red") + continue + + await db.execute( + INSERT_QUERY_INFORMATION, + { + "id": media_db_id, + "remoteId": remote_id, + "text": media.caption_text, + "additionalInfo": user, + "imageUri": thumbnail_url, + "href": f"https://www.instagram.com/p/{media.code}", + "date": media.taken_at.strftime("%Y-%m-%d %H:%M:%S"), + }, + ) + for resource in media.resources: + thumbnail_url = resource.thumbnail_url + if thumbnail_url: + console.log(f"Downloading thumbnail for resource {resource.pk}") + try: + thumbnail = requests.get(thumbnail_url).content + filename = f"instagramr_{uuid4()}.jpg" + with open(f"/app/cdn/{filename}", "wb") as f: + f.write(thumbnail) + thumbnail_url = f"/cdn/{filename}" + except Exception as e: + console.log( + f"Error downloading thumbnail: {e}", style="bold red" + ) + continue + + await db.execute( + INSERT_QUERY_RESOURCE, + { + "id": uuid4(), + "remoteId": str(resource.pk), + "informationId": media_db_id, + "imageUri": thumbnail_url, + "videoUri": resource.video_url, + }, + ) + + +asyncio.run(instagram()) diff --git a/src/dataimport/requirements.txt b/src/dataimport/requirements.txt index bdbb56c..41cb515 100644 --- a/src/dataimport/requirements.txt +++ b/src/dataimport/requirements.txt @@ -9,3 +9,6 @@ praw==7.7.1 rich==12.4.4 requests openai==1.7.2 +pyotp==2.9.0 +instagrapi==2.0.1 +pillow==10.2.0 diff --git a/src/dataimport/youtube.py b/src/dataimport/youtube.py index d433fbc..8e8e43e 100644 --- a/src/dataimport/youtube.py +++ b/src/dataimport/youtube.py @@ -40,7 +40,7 @@ async def youtube(): yt_data = requests.get(collection_url).json()["items"][0]["community"] for yt in yt_data: - query = f"SELECT * FROM Information WHERE remoteId='{yt['id']}'" + query = f"SELECT * FROM Information WHERE remoteId='{yt['id']}' AND importedFrom='YouTube'" result = await db.fetch_one(query=query) if result: console.log(f"{yt['id']} already in database", style="bold red") diff --git a/src/psaggregator/package-lock.json b/src/psaggregator/package-lock.json index 46bd363..0a50b58 100644 --- a/src/psaggregator/package-lock.json +++ b/src/psaggregator/package-lock.json @@ -1,21 +1,21 @@ { "name": "psaggregator", - "version": "1.2.3", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "psaggregator", - "version": "1.2.3", + "version": "1.3.0", "devDependencies": { "@fontsource/fira-mono": "^4.5.10", "@neoconfetti/svelte": "^1.0.0", - "@prisma/client": "^5.8.0", + "@prisma/client": "4.15.0-integration-feat-client-esm.27", "@skeletonlabs/skeleton": "^2.7.0", "@skeletonlabs/tw-plugin": "^0.3.1", "@sveltejs/adapter-auto": "^3.0.0", - "@sveltejs/adapter-node": "^2.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-node": "^4.0.1", + "@sveltejs/kit": "^2.4.3", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.7", "@types/eslint": "8.56.0", @@ -34,7 +34,7 @@ "prettier": "^3.1.1", "prettier-plugin-svelte": "^3.1.2", "prettier-plugin-tailwindcss": "^0.5.9", - "prisma": "^5.8.0", + "prisma": "4.15.0-integration-feat-client-esm.27", "svelte": "^4.2.7", "svelte-check": "^3.6.0", "tailwindcss": "^3", @@ -734,13 +734,16 @@ "dev": true }, "node_modules/@prisma/client": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.8.0.tgz", - "integrity": "sha512-QxO6C4MaA/ysTIbC+EcAH1aX/YkpymhXtO6zPdk+FvA7+59tNibIYpd+7koPdViLg2iKES4ojsxWNUGNJaEcbA==", + "version": "4.15.0-integration-feat-client-esm.27", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-4.15.0-integration-feat-client-esm.27.tgz", + "integrity": "sha512-d1+EXkJpNaLOFH4x2FIRj66rkdW8g8wAwDcw7ayPNzix+1MKW4IBWncZLZDf5by9gSwHJlCb5N2BTRJmUnE3qQ==", "dev": true, "hasInstallScript": true, + "dependencies": { + "@prisma/engines-version": "4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944" + }, "engines": { - "node": ">=16.13" + "node": ">=14.17" }, "peerDependencies": { "prisma": "*" @@ -751,51 +754,19 @@ } } }, - "node_modules/@prisma/debug": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.8.0.tgz", - "integrity": "sha512-ZqPpkvbovu/kQJ1bvy57NO4dw97fpQGcbQSCtsqlwSE1UNKJP75R3BKxdznk8ZPMY+GJdMRetWNv4oAvSbWn8Q==", - "dev": true - }, "node_modules/@prisma/engines": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.8.0.tgz", - "integrity": "sha512-Qhqm9WWLujNEC13AuZlUO14SQ15tNLe5puaz+tOk7UqINqJ3PtqMmuSuzomiw2diGVqZ+HYiSQzlR3+pPucVHA==", + "version": "4.15.0-integration-feat-client-esm.27", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-4.15.0-integration-feat-client-esm.27.tgz", + "integrity": "sha512-xzD1472StQ6GA3gaZpPVwj6qlkbs92mPMTSESWiDCtDvkyPvN/g1qgrdPxRt1uFuPM2NSj5Ngc2InyfwRUAQFA==", "dev": true, - "hasInstallScript": true, - "dependencies": { - "@prisma/debug": "5.8.0", - "@prisma/engines-version": "5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848", - "@prisma/fetch-engine": "5.8.0", - "@prisma/get-platform": "5.8.0" - } + "hasInstallScript": true }, "node_modules/@prisma/engines-version": { - "version": "5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848", - "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848.tgz", - "integrity": "sha512-cXcoVweYbnv8xRfkWq9oj8BECOdzHUazrSpYCa0ehp5TNz4l5Spa8jbq/VROCTzj3ZncH5D9Q2TmySYTOUeKlw==", + "version": "4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944.tgz", + "integrity": "sha512-sVOig4tjGxxlYaFcXgE71f/rtFhzyYrfyfNFUsxCIEJyVKU9rdOWIlIwQ2NQ7PntvGnn+x0XuFo4OC1jvPJKzg==", "dev": true }, - "node_modules/@prisma/fetch-engine": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.8.0.tgz", - "integrity": "sha512-1CAuE+JoYsPNggMEn6qk0zos06Uc9bYZBJ0VBPHD6R7REL05614koAbOCmn52IaYz3nobb7f25hqW6AY7rLkIw==", - "dev": true, - "dependencies": { - "@prisma/debug": "5.8.0", - "@prisma/engines-version": "5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848", - "@prisma/get-platform": "5.8.0" - } - }, - "node_modules/@prisma/get-platform": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.8.0.tgz", - "integrity": "sha512-Nk3rhTFZ1LYkFZJnpSvQcLPCaBWgJQfteHII6UEENOOkYlmP0k3FuswND54tzzEr4qs39wOdV9pbXKX9U2lv7A==", - "dev": true, - "dependencies": { - "@prisma/debug": "5.8.0" - } - }, "node_modules/@rollup/plugin-commonjs": { "version": "25.0.7", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-25.0.7.tgz", @@ -1143,24 +1114,24 @@ } }, "node_modules/@sveltejs/adapter-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-node/-/adapter-node-2.1.1.tgz", - "integrity": "sha512-ypAqdvjoyfOEgucDbY3P6DM6nLEQ7OwiWMfyITKF8svffe9FIx1ow6uELJd1BzbCrouYndkQfc5m6ihj0xxqTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-node/-/adapter-node-4.0.1.tgz", + "integrity": "sha512-IviiTtKCDp+0QoTmmMlGGZBA1EoUNsjecU6XGV9k62S3f01SNsVhpqi2e4nbI62BLGKh/YKKfFii+Vz/b9XIxg==", "dev": true, "dependencies": { "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", - "rollup": "^4.8.0" + "rollup": "^4.9.5" }, "peerDependencies": { - "@sveltejs/kit": "^2.0.0" + "@sveltejs/kit": "^2.4.0" } }, "node_modules/@sveltejs/kit": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.3.2.tgz", - "integrity": "sha512-AzGWV1TyUSkBuciy06E5NegXndIEgTthDtllv80qynEJFh8bZD62ZxLajiQLOsKGqRDilEQyshDARQxjIqiaqg==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", + "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", "dev": true, "hasInstallScript": true, "dependencies": { @@ -3789,19 +3760,20 @@ } }, "node_modules/prisma": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.8.0.tgz", - "integrity": "sha512-hDKoEqPt2qEUTH5yGO3l27CBnPtwvte0CGMKrpCr9+/A919JghfqJ3qgCGgMbOwdkXUOzdho0RH9tyUF3UhpMw==", + "version": "4.15.0-integration-feat-client-esm.27", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-4.15.0-integration-feat-client-esm.27.tgz", + "integrity": "sha512-hVuVm+KPUhPRbTE0Z86wnaD2P6wBu2byP6w7fQ8z5kM2/STrkGLz4xja8AlQQQ43bZObvmTy1GMP0eHTbi7G2Q==", "dev": true, "hasInstallScript": true, "dependencies": { - "@prisma/engines": "5.8.0" + "@prisma/engines": "4.15.0-integration-feat-client-esm.27" }, "bin": { - "prisma": "build/index.js" + "prisma": "build/index.js", + "prisma2": "build/index.js" }, "engines": { - "node": ">=16.13" + "node": ">=14.17" } }, "node_modules/proxy-from-env": { diff --git a/src/psaggregator/package.json b/src/psaggregator/package.json index 283e7d9..0f51d39 100644 --- a/src/psaggregator/package.json +++ b/src/psaggregator/package.json @@ -1,6 +1,6 @@ { "name": "psaggregator", - "version": "1.2.3", + "version": "1.3.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -16,12 +16,12 @@ "devDependencies": { "@fontsource/fira-mono": "^4.5.10", "@neoconfetti/svelte": "^1.0.0", - "@prisma/client": "^5.8.0", + "@prisma/client": "4.15.0-integration-feat-client-esm.27", "@skeletonlabs/skeleton": "^2.7.0", "@skeletonlabs/tw-plugin": "^0.3.1", "@sveltejs/adapter-auto": "^3.0.0", - "@sveltejs/adapter-node": "^2.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-node": "^4.0.1", + "@sveltejs/kit": "^2.4.3", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.7", "@types/eslint": "8.56.0", @@ -40,7 +40,7 @@ "prettier": "^3.1.1", "prettier-plugin-svelte": "^3.1.2", "prettier-plugin-tailwindcss": "^0.5.9", - "prisma": "^5.8.0", + "prisma": "4.15.0-integration-feat-client-esm.27", "svelte": "^4.2.7", "svelte-check": "^3.6.0", "tailwindcss": "^3", diff --git a/src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql b/src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql new file mode 100644 index 0000000..28c4663 --- /dev/null +++ b/src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql @@ -0,0 +1,18 @@ +-- AlterTable +ALTER TABLE `Information` ADD COLUMN `additionalInfo` VARCHAR(1024) NULL; + +-- CreateTable +CREATE TABLE `InformationResource` ( + `id` VARCHAR(191) NOT NULL, + `remoteId` VARCHAR(191) NULL, + `informationId` VARCHAR(191) NOT NULL, + `imageUri` VARCHAR(1024) NULL, + `videoUri` VARCHAR(1024) NULL, + `importedAt` DATETIME(3) NOT NULL, + `importedFrom` ENUM('Unknown', 'PietSmietDE', 'Instagram', 'Twitter', 'Threads', 'Reddit', 'YouTube', 'OpenAI', 'Custom') NOT NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- AddForeignKey +ALTER TABLE `InformationResource` ADD CONSTRAINT `InformationResource_informationId_fkey` FOREIGN KEY (`informationId`) REFERENCES `Information`(`id`) ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql b/src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql new file mode 100644 index 0000000..e8aa750 --- /dev/null +++ b/src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE `ContentPiece` MODIFY `title` LONGTEXT NOT NULL, + MODIFY `description` LONGTEXT NULL, + MODIFY `additionalInfo` LONGTEXT NULL; + +-- AlterTable +ALTER TABLE `Information` MODIFY `text` LONGTEXT NOT NULL, + MODIFY `additionalInfo` LONGTEXT NULL; + +-- AlterTable +ALTER TABLE `RedditPost` MODIFY `title` LONGTEXT NOT NULL, + MODIFY `description` LONGTEXT NULL; + +-- AlterTable +ALTER TABLE `ScheduledContentPiece` MODIFY `title` LONGTEXT NOT NULL, + MODIFY `description` LONGTEXT NULL, + MODIFY `additionalInfo` LONGTEXT NULL; diff --git a/src/psaggregator/src/config/schema.prisma b/src/psaggregator/src/config/schema.prisma index 014510a..442919e 100644 --- a/src/psaggregator/src/config/schema.prisma +++ b/src/psaggregator/src/config/schema.prisma @@ -28,9 +28,9 @@ enum ImportType { model ScheduledContentPiece { id String @id @default(uuid()) remoteId String? - title String @db.VarChar(1024) - description String? @db.VarChar(1024) - additionalInfo String? @db.VarChar(1024) + title String @db.LongText + description String? @db.LongText + additionalInfo String? @db.LongText startDate DateTime? imageUri String? @db.VarChar(1024) href String? @db.VarChar(1024) @@ -44,9 +44,9 @@ model ScheduledContentPiece { model ContentPiece { id String @id @default(uuid()) remoteId String - title String @db.VarChar(1024) - description String? @db.VarChar(1024) - additionalInfo String? @db.VarChar(1024) + title String @db.LongText + description String? @db.LongText + additionalInfo String? @db.LongText startDate DateTime? imageUri String? @db.VarChar(1024) href String? @db.VarChar(1024) @@ -58,15 +58,28 @@ model ContentPiece { } model Information { - id String @id @default(uuid()) - remoteId String? - text String @db.VarChar(1024) - imageUri String? @db.VarChar(1024) - href String? @db.VarChar(1024) - date DateTime? - analyzedAt DateTime? @default(now()) - importedAt DateTime - importedFrom ImportType + id String @id @default(uuid()) + remoteId String? + text String @db.LongText + additionalInfo String? @db.LongText + imageUri String? @db.VarChar(1024) + href String? @db.VarChar(1024) + date DateTime? + analyzedAt DateTime? @default(now()) + importedAt DateTime + importedFrom ImportType + InformationResource InformationResource[] +} + +model InformationResource { + id String @id @default(uuid()) + remoteId String? + information Information @relation(fields: [informationId], references: [id], onDelete: Cascade, onUpdate: Cascade) + informationId String + imageUri String? @db.VarChar(1024) + videoUri String? @db.VarChar(1024) + importedAt DateTime + importedFrom ImportType } model TwitchStatus { @@ -81,8 +94,8 @@ model TwitchStatus { model RedditPost { id String @id - title String @db.VarChar(1024) - description String? @db.VarChar(1024) + title String @db.LongText + description String? @db.LongText username String upvotes Int comments Int diff --git a/src/psaggregator/src/lib/components/InstagramPost.svelte b/src/psaggregator/src/lib/components/InstagramPost.svelte new file mode 100644 index 0000000..1c48a39 --- /dev/null +++ b/src/psaggregator/src/lib/components/InstagramPost.svelte @@ -0,0 +1,32 @@ + + + +
+ {titleCase(post.additionalInfo)} + {#if post.date} + {moment(post.date).fromNow()} + {/if} +
+
{post.text}
+ {#if post.InformationResource.filter((x) => x.imageUri).length > 1} +
+ {#each post.InformationResource.filter((x) => x.imageUri) as resource} +
+ {resource.remoteId} +
+ {/each} +
+ {:else if post.imageUri} + {"community + {/if} + diff --git a/src/psaggregator/src/lib/prisma.ts b/src/psaggregator/src/lib/prisma.ts index 8301a4c..921a1f3 100644 --- a/src/psaggregator/src/lib/prisma.ts +++ b/src/psaggregator/src/lib/prisma.ts @@ -1,6 +1,8 @@ -import { createRequire } from "module"; // https://github.com/prisma/prisma/issues/15614 -const require = createRequire(import.meta.url); -const { PrismaClient } = require("@prisma/client"); +import { createRequire as _createRequire } from 'module' +export const createCursedRequire: (path: string | URL) => (id: string) => TShape = _createRequire + +const require = createCursedRequire(import.meta.url ?? __filename) +const { PrismaClient } = require('@prisma/client'); const prisma = new PrismaClient(); diff --git a/src/psaggregator/src/routes/+page.server.ts b/src/psaggregator/src/routes/+page.server.ts index fdc50d8..682119e 100644 --- a/src/psaggregator/src/routes/+page.server.ts +++ b/src/psaggregator/src/routes/+page.server.ts @@ -62,7 +62,7 @@ export async function load() { orderBy: { sticky: "desc" }, - take: 5 + take: 10 }); const youtubeCommunityPosts = await prisma.information.findMany({ @@ -83,10 +83,32 @@ export async function load() { take: 3 }); + const instagramPosts = await prisma.information.findMany({ + where: { + importedFrom: { + equals: "Instagram" + }, + href: { + not: null + }, + date: { + not: null + } + }, + include: { + InformationResource: true, + }, + orderBy: { + date: "desc" + }, + take: 3 + }); + return { videos, upcomingStreams, youtubeCommunityPosts, + instagramPosts, today, twitchStatus, redditPosts diff --git a/src/psaggregator/src/routes/+page.svelte b/src/psaggregator/src/routes/+page.svelte index 00c5705..90d865e 100644 --- a/src/psaggregator/src/routes/+page.svelte +++ b/src/psaggregator/src/routes/+page.svelte @@ -4,8 +4,10 @@ import PSVideo from "$lib/components/PSVideo.svelte"; import TwitchStatus from "$lib/components/TwitchStatus.svelte"; import RedditPost from "$lib/components/RedditPost.svelte"; - import { List, DataError, VideoPlayer, LogoYoutube } from "carbon-icons-svelte"; + import { List, VideoPlayer, LogoYoutube, LogoInstagram } from "carbon-icons-svelte"; import YouTubeCommunityPost from "$lib/components/YouTubeCommunityPost.svelte"; + import MediaQuery from "$lib/utils/MediaQuery.svelte"; + import InstagramPost from "$lib/components/InstagramPost.svelte"; export let data: PageServerData; @@ -30,77 +32,94 @@ } -
-
-
-
- - Uploadplan + +
+
+
+
+ + Uploadplan +
+
+ {#each data.today as content} + + {:else} +
+ Bisher konnte heute kein Uploadplan importiert werden. +
+ {/each} +
-
- {#each data.today as content} - - {:else} -
- Bisher konnte heute kein Uploadplan importiert werden. + {#if data.twitchStatus} +
+
+ twitch + Twitch
- {/each} -
+ +
+ {/if}
- {#if data.twitchStatus} -
+
+
- twitch - Twitch + + YouTube +
+
+ {#each data.youtubeCommunityPosts as youtube} + + {/each}
- -
- {/if} -
-
-
-
- - YouTube
-
- {#each data.youtubeCommunityPosts as youtube} - - {/each} +
+
+ + Instagram +
+
+ {#each data.instagramPosts as instagram} + + {/each} +
-
-
-
- twitch - Anstehende Streams +
+
+ reddit + Reddit +
+
+ {#each data.redditPosts.slice(0, matches ? 10 : 5) as reddit} + + {/each} +
-
- {#each data.upcomingStreams as stream} - - {/each} +
+
+ twitch + Anstehende Streams +
+
+ {#each data.upcomingStreams as stream} + + {:else} +
+ Momentan sind keine geplanten Streams bekannt. +
+ {/each} +
- reddit - Reddit + + Neuste Videos
-
- {#each data.redditPosts as reddit} - +
+ {#each data.videos as video} + {/each}
-
-
- - Neuste Videos -
-
- {#each data.videos as video} - - {/each} -
-
-
+ diff --git a/src/psaggregator/src/routes/api/+page.svelte b/src/psaggregator/src/routes/api/+page.svelte index 9024cc4..7711337 100644 --- a/src/psaggregator/src/routes/api/+page.svelte +++ b/src/psaggregator/src/routes/api/+page.svelte @@ -13,7 +13,7 @@ const curlTwitch = `curl -X GET ${location.protocol}//${location.host}/api/twitch`; const curlThumbnails = `curl -X GET ${location.protocol}//${location.host}/api/thumbnails?skip=0`; const curlReddit = `curl -X GET ${location.protocol}//${location.host}/api/reddit`; - const curlInformation = `curl -X GET ${location.protocol}//${location.host}/api/information?skip=0&date=${moment().format("YYYY-MM-DD")}`; + const curlInformation = `curl -X GET ${location.protocol}//${location.host}/api/information?skip=0&date=${moment().format("YYYY-MM-DD")}&type=YouTube`; hljs.registerLanguage("bash", bash); storeHighlightJs.set(hljs); @@ -116,7 +116,9 @@
  • pietsmiet.de Uploadplan
  • YouTube Communityposts des Hauptkanals
    • +
  • Instagram Posts der ersten Reihe
+ Über den ?type Parameter kannst du einen Filter auf die Importquelle anwenden. Zukünftig sind hier noch Importe aus anderen Social Media geplant.
Ausblick diff --git a/src/psaggregator/src/routes/api/information/+server.ts b/src/psaggregator/src/routes/api/information/+server.ts index 6d89b06..878d78a 100644 --- a/src/psaggregator/src/routes/api/information/+server.ts +++ b/src/psaggregator/src/routes/api/information/+server.ts @@ -13,6 +13,8 @@ export async function GET({ url }) { } } + const type = url.searchParams.get("type") ?? undefined; + let date = null; if (url.searchParams.has("date")) { try { @@ -35,16 +37,26 @@ export async function GET({ url }) { date: { lt: upperBound, gt: lowerBound - } + }, + importedFrom: type, + }, + include: { + InformationResource: true, }, orderBy: { date: "desc" }, skip, - take: 20 + take: 20, }); } else { data = await prisma.information.findMany({ + where: { + importedFrom: type, + }, + include: { + InformationResource: true, + }, orderBy: { date: "desc" }, diff --git a/src/psaggregator/src/routes/news/+page.server.ts b/src/psaggregator/src/routes/news/+page.server.ts index 490a65d..02f1493 100644 --- a/src/psaggregator/src/routes/news/+page.server.ts +++ b/src/psaggregator/src/routes/news/+page.server.ts @@ -16,10 +16,32 @@ export async function load() { orderBy: { date: "desc" }, - take: 10 + take: 20 + }); + + const instagramPosts = await prisma.information.findMany({ + where: { + importedFrom: { + equals: "Instagram" + }, + href: { + not: null + }, + date: { + not: null + } + }, + include: { + InformationResource: true, + }, + orderBy: { + date: "desc" + }, + take: 20 }); return { - youtubeCommunityPosts + youtubeCommunityPosts, + instagramPosts }; } diff --git a/src/psaggregator/src/routes/news/+page.svelte b/src/psaggregator/src/routes/news/+page.svelte index 67386f5..542c816 100644 --- a/src/psaggregator/src/routes/news/+page.svelte +++ b/src/psaggregator/src/routes/news/+page.svelte @@ -1,10 +1,86 @@
@@ -18,23 +94,27 @@ {/each}
+ {#if loading[ImportType.YouTube]} +
+ +
+ {/if}
Instagram
-
-
- -
- Leider gibt es noch keinen Instagram-Import. - Dieses Projekt ist Open Source. - Beteilige dich gerne auf - GitHub - +
+ {#each data.instagramPosts as instagram} + + {/each}
+ {#if loading[ImportType.Instagram]} +
+ +
+ {/if}