From a8258661fa73d0f005e3c1051c08fff2e49dc49d Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 07:18:11 +0100 Subject: [PATCH 01/19] add instagram import script --- .env.example | 4 + docker-compose-test.yml | 3 + docker-compose.yml | 3 + src/dataimport/.env.example | 4 + src/dataimport/.gitignore | 2 +- src/dataimport/instagram.py | 167 ++++++++++++++++++ src/dataimport/requirements.txt | 3 + .../migration.sql | 18 ++ .../20240126060555_long_text/migration.sql | 17 ++ src/psaggregator/src/config/schema.prisma | 47 +++-- 10 files changed, 250 insertions(+), 18 deletions(-) create mode 100644 src/dataimport/instagram.py create mode 100644 src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql create mode 100644 src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql diff --git a/.env.example b/.env.example index 6d994cd..88766d5 100644 --- a/.env.example +++ b/.env.example @@ -13,3 +13,7 @@ LEGAL_URL= PUBLIC_KOFI_USERNAME=zaanposni OPENAI_API_KEY= + +INSTAGRAM_USERNAME = +INSTAGRAM_PASSWORD = +INSTAGRAM_2FA_SECRET = diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 05a33a0..a20828e 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -49,6 +49,9 @@ services: - OPENAI_API_KEY=${OPENAI_API_KEY} - SQLALCHEMY_SILENCE_UBER_WARNING=1 - YT_SERVER_BASE_URL=http://youtube-api + - INSTAGRAM_USERNAME=${INSTAGRAM_USERNAME} + - INSTAGRAM_PASSWORD=${INSTAGRAM_PASSWORD} + - INSTAGRAM_2FA_SECRET=${INSTAGRAM_2FA_SECRET} volumes: - shared-data:/app/cdn depends_on: diff --git a/docker-compose.yml b/docker-compose.yml index b22fabc..2aff4aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,6 +43,9 @@ services: - OPENAI_API_KEY=${OPENAI_API_KEY} - SQLALCHEMY_SILENCE_UBER_WARNING=1 - YT_SERVER_BASE_URL=http://youtube-api + - INSTAGRAM_USERNAME=${INSTAGRAM_USERNAME} + - INSTAGRAM_PASSWORD=${INSTAGRAM_PASSWORD} + - INSTAGRAM_2FA_SECRET=${INSTAGRAM_2FA_SECRET} volumes: - shared-data:/app/cdn depends_on: diff --git a/src/dataimport/.env.example b/src/dataimport/.env.example index be51dc4..73b19c9 100644 --- a/src/dataimport/.env.example +++ b/src/dataimport/.env.example @@ -7,3 +7,7 @@ TWITCH_CLIENT_ID= TWITCH_CLIENT_SECRET= OPENAI_API_KEY= + +INSTAGRAM_USERNAME = +INSTAGRAM_PASSWORD = +INSTAGRAM_2FA_SECRET = diff --git a/src/dataimport/.gitignore b/src/dataimport/.gitignore index 55b1e95..1b1d690 100644 --- a/src/dataimport/.gitignore +++ b/src/dataimport/.gitignore @@ -1,7 +1,7 @@ geckodriver.exe *.json -instagram.py openai_test.py .env videos.sql __pycache__/ +threads.py diff --git a/src/dataimport/instagram.py b/src/dataimport/instagram.py new file mode 100644 index 0000000..009233c --- /dev/null +++ b/src/dataimport/instagram.py @@ -0,0 +1,167 @@ +import asyncio +import os +from databases import Database +from uuid import uuid4 + +from rich.console import Console +from instagrapi import Client +from instagrapi.exceptions import LoginRequired +import pyotp + + +console = Console() + +USERNAME = os.getenv("INSTAGRAM_USERNAME") +PASSWORD = os.getenv("INSTAGRAM_PASSWORD") +KEY_2FA = os.getenv("INSTAGRAM_2FA_SECRET") + +if not USERNAME or not PASSWORD: + raise Exception("No Instagram username or password provided") + + +def login_user(): + """ + Attempts to login to Instagram using either the provided session information + or the provided username and password. + """ + + console.log("Attempting to login user...") + if not os.path.exists("session.json"): + console.log("No session file found, creating empty session file") + with open("session.json", "w") as f: + f.write("{}") + + cl = Client() + session = cl.load_settings("session.json") + + login_via_session = False + login_via_pw = False + + if session: + try: + cl.set_settings(session) + + code_2fa = "" + if KEY_2FA: + code_2fa = pyotp.TOTP(KEY_2FA.replace(" ", "")).now() + cl.login(USERNAME, PASSWORD, verification_code=code_2fa) + + # check if session is valid + try: + cl.get_timeline_feed() + except LoginRequired: + console.log( + "Session is invalid, need to login via username and password" + ) + + old_session = cl.get_settings() + + # use the same device uuids across logins + cl.set_settings({}) + cl.set_uuids(old_session["uuids"]) + + code_2fa = "" + if KEY_2FA: + code_2fa = pyotp.TOTP(KEY_2FA.replace(" ", "")).now() + cl.login(USERNAME, PASSWORD, verification_code=code_2fa) + login_via_session = True + except Exception as e: + console.log("Couldn't login user using session information: %s" % e) + + if not login_via_session: + try: + console.log( + "Attempting to login via username and password. username: %s" % USERNAME + ) + + code_2fa = "" + if KEY_2FA: + code_2fa = pyotp.TOTP(KEY_2FA.replace(" ", "")).now() + + if cl.login(USERNAME, PASSWORD, verification_code=code_2fa): + login_via_pw = True + except Exception as e: + console.log("Couldn't login user using username and password: %s" % e) + + if not login_via_pw and not login_via_session: + raise Exception("Couldn't login user with either password or session") + + cl.dump_settings("session.json") + return cl + + +cl = login_user() +console.log("Successfully logged in user") + +user_dict = { + "peter": 344058897, + "brammen": 1588473759, + "jay": 2030403724, + "sep": 1609561808, + "chris": 1057433625, +} + +console.log("Fetching last 50 media items for each user") + +INSERT_QUERY_INFORMATION = """ + INSERT INTO Information (id, remoteId, text, additionalInfo, imageUri, href, date, analyzedAt, importedAt, importedFrom) + VALUES (:id, :remoteId, :text, :additionalInfo, :imageUri, :href, :date, NULL, now(), 'Instagram')""" +INSERT_QUERY_RESOURCE = """ + INSERT INTO InformationResource (id, remoteId, informationId, imageUri, videoUri, importedAt, importedFrom) + VALUES (:id, :remoteId, :informationId, :imageUri, :videoUri, now(), 'Instagram')""" +SELECT_QUERY_INFORMATION = """ + SELECT id FROM Information WHERE remoteId = :remoteId AND importedFrom = 'Instagram'""" + + +async def instagram(): + console.log("Connecting to database...", style="bold green") + db = Database(url=os.getenv("DATABASE_URL")) + await db.connect() + for user, user_id in user_dict.items(): + console.log(f"Fetching last 50 media items for {user}") + last_media = cl.user_medias(user_id, amount=50) + console.log(f"Found {len(last_media)} media items for {user}") + for media in last_media: + remote_id = f"{user}_{str(media.id)}" + media_db_id = uuid4() + console.log(f"Processing media item {remote_id}") + if await db.fetch_one( + SELECT_QUERY_INFORMATION, {"remoteId": str(remote_id)} + ): + console.log( + f"Media item {remote_id} already in database", style="bold red" + ) + continue + console.log(f"Media item {remote_id} not in database, inserting") + thumbnail_url = media.thumbnail_url + if not thumbnail_url: + for resource in media.resources: + if resource.thumbnail_url: + thumbnail_url = resource.thumbnail_url + break + await db.execute( + INSERT_QUERY_INFORMATION, + { + "id": media_db_id, + "remoteId": remote_id, + "text": media.caption_text, + "additionalInfo": user, + "imageUri": thumbnail_url, + "href": f"https://www.instagram.com/p/{media.code}", + "date": media.taken_at.strftime("%Y-%m-%d %H:%M:%S"), + }, + ) + for resource in media.resources: + await db.execute( + INSERT_QUERY_RESOURCE, + { + "id": uuid4(), + "remoteId": str(resource.pk), + "informationId": media_db_id, + "imageUri": resource.thumbnail_url, + "videoUri": resource.video_url, + }, + ) + + +asyncio.run(instagram()) diff --git a/src/dataimport/requirements.txt b/src/dataimport/requirements.txt index bdbb56c..7e2acd9 100644 --- a/src/dataimport/requirements.txt +++ b/src/dataimport/requirements.txt @@ -9,3 +9,6 @@ praw==7.7.1 rich==12.4.4 requests openai==1.7.2 +pyotp==2.9.0 +instagrapi==2.0.1 +pydantic==1.10.13 diff --git a/src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql b/src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql new file mode 100644 index 0000000..28c4663 --- /dev/null +++ b/src/psaggregator/src/config/migrations/20240126054206_more_information_stuff/migration.sql @@ -0,0 +1,18 @@ +-- AlterTable +ALTER TABLE `Information` ADD COLUMN `additionalInfo` VARCHAR(1024) NULL; + +-- CreateTable +CREATE TABLE `InformationResource` ( + `id` VARCHAR(191) NOT NULL, + `remoteId` VARCHAR(191) NULL, + `informationId` VARCHAR(191) NOT NULL, + `imageUri` VARCHAR(1024) NULL, + `videoUri` VARCHAR(1024) NULL, + `importedAt` DATETIME(3) NOT NULL, + `importedFrom` ENUM('Unknown', 'PietSmietDE', 'Instagram', 'Twitter', 'Threads', 'Reddit', 'YouTube', 'OpenAI', 'Custom') NOT NULL, + + PRIMARY KEY (`id`) +) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +-- AddForeignKey +ALTER TABLE `InformationResource` ADD CONSTRAINT `InformationResource_informationId_fkey` FOREIGN KEY (`informationId`) REFERENCES `Information`(`id`) ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql b/src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql new file mode 100644 index 0000000..e8aa750 --- /dev/null +++ b/src/psaggregator/src/config/migrations/20240126060555_long_text/migration.sql @@ -0,0 +1,17 @@ +-- AlterTable +ALTER TABLE `ContentPiece` MODIFY `title` LONGTEXT NOT NULL, + MODIFY `description` LONGTEXT NULL, + MODIFY `additionalInfo` LONGTEXT NULL; + +-- AlterTable +ALTER TABLE `Information` MODIFY `text` LONGTEXT NOT NULL, + MODIFY `additionalInfo` LONGTEXT NULL; + +-- AlterTable +ALTER TABLE `RedditPost` MODIFY `title` LONGTEXT NOT NULL, + MODIFY `description` LONGTEXT NULL; + +-- AlterTable +ALTER TABLE `ScheduledContentPiece` MODIFY `title` LONGTEXT NOT NULL, + MODIFY `description` LONGTEXT NULL, + MODIFY `additionalInfo` LONGTEXT NULL; diff --git a/src/psaggregator/src/config/schema.prisma b/src/psaggregator/src/config/schema.prisma index 014510a..442919e 100644 --- a/src/psaggregator/src/config/schema.prisma +++ b/src/psaggregator/src/config/schema.prisma @@ -28,9 +28,9 @@ enum ImportType { model ScheduledContentPiece { id String @id @default(uuid()) remoteId String? - title String @db.VarChar(1024) - description String? @db.VarChar(1024) - additionalInfo String? @db.VarChar(1024) + title String @db.LongText + description String? @db.LongText + additionalInfo String? @db.LongText startDate DateTime? imageUri String? @db.VarChar(1024) href String? @db.VarChar(1024) @@ -44,9 +44,9 @@ model ScheduledContentPiece { model ContentPiece { id String @id @default(uuid()) remoteId String - title String @db.VarChar(1024) - description String? @db.VarChar(1024) - additionalInfo String? @db.VarChar(1024) + title String @db.LongText + description String? @db.LongText + additionalInfo String? @db.LongText startDate DateTime? imageUri String? @db.VarChar(1024) href String? @db.VarChar(1024) @@ -58,15 +58,28 @@ model ContentPiece { } model Information { - id String @id @default(uuid()) - remoteId String? - text String @db.VarChar(1024) - imageUri String? @db.VarChar(1024) - href String? @db.VarChar(1024) - date DateTime? - analyzedAt DateTime? @default(now()) - importedAt DateTime - importedFrom ImportType + id String @id @default(uuid()) + remoteId String? + text String @db.LongText + additionalInfo String? @db.LongText + imageUri String? @db.VarChar(1024) + href String? @db.VarChar(1024) + date DateTime? + analyzedAt DateTime? @default(now()) + importedAt DateTime + importedFrom ImportType + InformationResource InformationResource[] +} + +model InformationResource { + id String @id @default(uuid()) + remoteId String? + information Information @relation(fields: [informationId], references: [id], onDelete: Cascade, onUpdate: Cascade) + informationId String + imageUri String? @db.VarChar(1024) + videoUri String? @db.VarChar(1024) + importedAt DateTime + importedFrom ImportType } model TwitchStatus { @@ -81,8 +94,8 @@ model TwitchStatus { model RedditPost { id String @id - title String @db.VarChar(1024) - description String? @db.VarChar(1024) + title String @db.LongText + description String? @db.LongText username String upvotes Int comments Int From a20108268b2a21f5802f916b1b52a3bfd1a9c423 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 07:19:48 +0100 Subject: [PATCH 02/19] add instagram to cron --- src/dataimport/hello-cron | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dataimport/hello-cron b/src/dataimport/hello-cron index bd12615..52268d7 100644 --- a/src/dataimport/hello-cron +++ b/src/dataimport/hello-cron @@ -3,9 +3,10 @@ 35 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdevideoimporter.py >> /var/log/cron.log 2>&1 */15 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdevideoimporter.py >> /var/log/cron.log 2>&1 5 * * * * . /root/project_env.sh; /usr/local/bin/python /app/youtube.py >> /var/log/cron.log 2>&1 +5 * * * * . /root/project_env.sh; /usr/local/bin/python /app/instagram.py >> /var/log/cron.log 2>&1 */15 * * * * . /root/project_env.sh; /usr/local/bin/python /app/reddit.py >> /var/log/cron.log 2>&1 */1 * * * * . /root/project_env.sh; /usr/local/bin/python /app/twitch.py >> /var/log/cron.log 2>&1 -1 1 * * * . /root/project_env.sh; /usr/local/bin/python /app/informationopenaianalyze.py >> /var/log/cron.log 2>&1 +10 1 * * * . /root/project_env.sh; /usr/local/bin/python /app/informationopenaianalyze.py >> /var/log/cron.log 2>&1 2 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdeuploadplan.py >> /var/log/cron.log 2>&1 5 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdeuploadplan.py >> /var/log/cron.log 2>&1 30 * * * * . /root/project_env.sh; /usr/local/bin/python /app/pietsmietdeuploadplan.py >> /var/log/cron.log 2>&1 From be2f2b52616c50f3a7d6a6af4b58f66e8aa976af Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 07:20:00 +0100 Subject: [PATCH 03/19] respect importtyp in youtubescript --- src/dataimport/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dataimport/youtube.py b/src/dataimport/youtube.py index d433fbc..8e8e43e 100644 --- a/src/dataimport/youtube.py +++ b/src/dataimport/youtube.py @@ -40,7 +40,7 @@ async def youtube(): yt_data = requests.get(collection_url).json()["items"][0]["community"] for yt in yt_data: - query = f"SELECT * FROM Information WHERE remoteId='{yt['id']}'" + query = f"SELECT * FROM Information WHERE remoteId='{yt['id']}' AND importedFrom='YouTube'" result = await db.fetch_one(query=query) if result: console.log(f"{yt['id']} already in database", style="bold red") From 8f00a523db15b91fa76b4896a52bd5a78cf330aa Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 07:28:52 +0100 Subject: [PATCH 04/19] add insta ressources to api --- src/psaggregator/src/routes/api/+page.svelte | 4 +++- .../src/routes/api/information/+server.ts | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/psaggregator/src/routes/api/+page.svelte b/src/psaggregator/src/routes/api/+page.svelte index 9024cc4..7711337 100644 --- a/src/psaggregator/src/routes/api/+page.svelte +++ b/src/psaggregator/src/routes/api/+page.svelte @@ -13,7 +13,7 @@ const curlTwitch = `curl -X GET ${location.protocol}//${location.host}/api/twitch`; const curlThumbnails = `curl -X GET ${location.protocol}//${location.host}/api/thumbnails?skip=0`; const curlReddit = `curl -X GET ${location.protocol}//${location.host}/api/reddit`; - const curlInformation = `curl -X GET ${location.protocol}//${location.host}/api/information?skip=0&date=${moment().format("YYYY-MM-DD")}`; + const curlInformation = `curl -X GET ${location.protocol}//${location.host}/api/information?skip=0&date=${moment().format("YYYY-MM-DD")}&type=YouTube`; hljs.registerLanguage("bash", bash); storeHighlightJs.set(hljs); @@ -116,7 +116,9 @@
  • pietsmiet.de Uploadplan
  • YouTube Communityposts des Hauptkanals
    • +
  • Instagram Posts der ersten Reihe
+ Über den ?type Parameter kannst du einen Filter auf die Importquelle anwenden. Zukünftig sind hier noch Importe aus anderen Social Media geplant. Ausblick diff --git a/src/psaggregator/src/routes/api/information/+server.ts b/src/psaggregator/src/routes/api/information/+server.ts index 6d89b06..878d78a 100644 --- a/src/psaggregator/src/routes/api/information/+server.ts +++ b/src/psaggregator/src/routes/api/information/+server.ts @@ -13,6 +13,8 @@ export async function GET({ url }) { } } + const type = url.searchParams.get("type") ?? undefined; + let date = null; if (url.searchParams.has("date")) { try { @@ -35,16 +37,26 @@ export async function GET({ url }) { date: { lt: upperBound, gt: lowerBound - } + }, + importedFrom: type, + }, + include: { + InformationResource: true, }, orderBy: { date: "desc" }, skip, - take: 20 + take: 20, }); } else { data = await prisma.information.findMany({ + where: { + importedFrom: type, + }, + include: { + InformationResource: true, + }, orderBy: { date: "desc" }, From 40a2b41bec598d47c630ef82c28b8023454f89f2 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 08:17:43 +0100 Subject: [PATCH 05/19] save instagram posts in cdn --- docker-compose-test.yml | 3 +++ docker-compose.yml | 3 +++ src/dataimport/instagram.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index a20828e..51e8dc4 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -52,8 +52,10 @@ services: - INSTAGRAM_USERNAME=${INSTAGRAM_USERNAME} - INSTAGRAM_PASSWORD=${INSTAGRAM_PASSWORD} - INSTAGRAM_2FA_SECRET=${INSTAGRAM_2FA_SECRET} + - INSTAGRAM_CONFIG_PATH=/app/config/instagram.json volumes: - shared-data:/app/cdn + - config:/app/config depends_on: - db - frontend @@ -93,6 +95,7 @@ networks: volumes: mysql: + config: shared-data: driver: local driver_opts: diff --git a/docker-compose.yml b/docker-compose.yml index 2aff4aa..d4c00ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,8 +46,10 @@ services: - INSTAGRAM_USERNAME=${INSTAGRAM_USERNAME} - INSTAGRAM_PASSWORD=${INSTAGRAM_PASSWORD} - INSTAGRAM_2FA_SECRET=${INSTAGRAM_2FA_SECRET} + - INSTAGRAM_CONFIG_PATH=/app/config/instagram.json volumes: - shared-data:/app/cdn + - config:/app/config depends_on: - db - frontend @@ -85,6 +87,7 @@ networks: volumes: mysql: + config: shared-data: driver: local driver_opts: diff --git a/src/dataimport/instagram.py b/src/dataimport/instagram.py index 009233c..59d40af 100644 --- a/src/dataimport/instagram.py +++ b/src/dataimport/instagram.py @@ -1,5 +1,6 @@ import asyncio import os +import requests from databases import Database from uuid import uuid4 @@ -133,12 +134,25 @@ async def instagram(): ) continue console.log(f"Media item {remote_id} not in database, inserting") + thumbnail_url = media.thumbnail_url if not thumbnail_url: for resource in media.resources: if resource.thumbnail_url: thumbnail_url = resource.thumbnail_url break + + console.log(f"Downloading thumbnail for {remote_id}") + try: + thumbnail = requests.get(thumbnail_url).content + filename = f"instagram_{uuid4()}.jpg" + with open(f"/app/cdn/{filename}", "wb") as f: + f.write(thumbnail) + thumbnail_url = f"'/cdn/{filename}'" + except Exception as e: + console.log(f"Error downloading thumbnail: {e}", style="bold red") + continue + await db.execute( INSERT_QUERY_INFORMATION, { @@ -152,13 +166,28 @@ async def instagram(): }, ) for resource in media.resources: + thumbnail_url = resource.thumbnail_url + if thumbnail_url: + console.log(f"Downloading thumbnail for resource {resource.pk}") + try: + thumbnail = requests.get(thumbnail_url).content + filename = f"instagramr_{uuid4()}.jpg" + with open(f"/app/cdn/{filename}", "wb") as f: + f.write(thumbnail) + thumbnail_url = f"'/cdn/{filename}'" + except Exception as e: + console.log( + f"Error downloading thumbnail: {e}", style="bold red" + ) + continue + await db.execute( INSERT_QUERY_RESOURCE, { "id": uuid4(), "remoteId": str(resource.pk), "informationId": media_db_id, - "imageUri": resource.thumbnail_url, + "imageUri": thumbnail_url, "videoUri": resource.video_url, }, ) From abc6cd645dcb7962aa5abfb5446024ec3c1c8327 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 08:18:13 +0100 Subject: [PATCH 06/19] =?UTF-8?q?python=20dependency=20management=20?= =?UTF-8?q?=F0=9F=A4=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/dataimport/Dockerfile | 1 + src/dataimport/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/dataimport/Dockerfile b/src/dataimport/Dockerfile index 37d42d2..462019d 100644 --- a/src/dataimport/Dockerfile +++ b/src/dataimport/Dockerfile @@ -37,6 +37,7 @@ COPY . /app # Install any needed packages specified in requirements.txt RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --no-deps --force-reinstall pydantic==1.10.13 RUN touch /var/log/cron.log COPY hello-cron /etc/cron.d/hello-cron diff --git a/src/dataimport/requirements.txt b/src/dataimport/requirements.txt index 7e2acd9..41cb515 100644 --- a/src/dataimport/requirements.txt +++ b/src/dataimport/requirements.txt @@ -11,4 +11,4 @@ requests openai==1.7.2 pyotp==2.9.0 instagrapi==2.0.1 -pydantic==1.10.13 +pillow==10.2.0 From 8c4107f4a70128c842a4fa08a01ccce315115763 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 08:18:22 +0100 Subject: [PATCH 07/19] allow custom instagram config path --- src/dataimport/instagram.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/dataimport/instagram.py b/src/dataimport/instagram.py index 59d40af..5b4a06b 100644 --- a/src/dataimport/instagram.py +++ b/src/dataimport/instagram.py @@ -15,6 +15,10 @@ USERNAME = os.getenv("INSTAGRAM_USERNAME") PASSWORD = os.getenv("INSTAGRAM_PASSWORD") KEY_2FA = os.getenv("INSTAGRAM_2FA_SECRET") +CONFIG_PATH = os.getenv("INSTAGRAM_CONFIG_PATH") +if not CONFIG_PATH: + CONFIG_PATH = "session.json" +console.log(f"Using config path {CONFIG_PATH}") if not USERNAME or not PASSWORD: raise Exception("No Instagram username or password provided") @@ -27,13 +31,13 @@ def login_user(): """ console.log("Attempting to login user...") - if not os.path.exists("session.json"): + if not os.path.exists(CONFIG_PATH): console.log("No session file found, creating empty session file") - with open("session.json", "w") as f: + with open(CONFIG_PATH, "w") as f: f.write("{}") cl = Client() - session = cl.load_settings("session.json") + session = cl.load_settings(CONFIG_PATH) login_via_session = False login_via_pw = False @@ -87,7 +91,7 @@ def login_user(): if not login_via_pw and not login_via_session: raise Exception("Couldn't login user with either password or session") - cl.dump_settings("session.json") + cl.dump_settings(CONFIG_PATH) return cl @@ -148,7 +152,7 @@ async def instagram(): filename = f"instagram_{uuid4()}.jpg" with open(f"/app/cdn/{filename}", "wb") as f: f.write(thumbnail) - thumbnail_url = f"'/cdn/{filename}'" + thumbnail_url = f"/cdn/{filename}" except Exception as e: console.log(f"Error downloading thumbnail: {e}", style="bold red") continue @@ -174,7 +178,7 @@ async def instagram(): filename = f"instagramr_{uuid4()}.jpg" with open(f"/app/cdn/{filename}", "wb") as f: f.write(thumbnail) - thumbnail_url = f"'/cdn/{filename}'" + thumbnail_url = f"/cdn/{filename}" except Exception as e: console.log( f"Error downloading thumbnail: {e}", style="bold red" From b5bf84e9103dcc68dfa9b593f54459674a6d2782 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 08:22:13 +0100 Subject: [PATCH 08/19] add instagram posts to homepage --- .../src/lib/components/InstagramPost.svelte | 38 +++++ src/psaggregator/src/routes/+page.server.ts | 24 +++- src/psaggregator/src/routes/+page.svelte | 131 ++++++++++-------- 3 files changed, 134 insertions(+), 59 deletions(-) create mode 100644 src/psaggregator/src/lib/components/InstagramPost.svelte diff --git a/src/psaggregator/src/lib/components/InstagramPost.svelte b/src/psaggregator/src/lib/components/InstagramPost.svelte new file mode 100644 index 0000000..3968395 --- /dev/null +++ b/src/psaggregator/src/lib/components/InstagramPost.svelte @@ -0,0 +1,38 @@ + + + + + +
+ {titleCase(post.additionalInfo)} + {#if post.date} + {moment(post.date).fromNow()} + {/if} +
+
{post.text}
+ {#if post.InformationResource.filter((x) => x.imageUri).length > 1} +
+ {#each post.InformationResource.filter((x) => x.imageUri) as resource} +
+ {resource.remoteId} +
+ {/each} +
+ {:else if post.imageUri} + {"community + {/if} + diff --git a/src/psaggregator/src/routes/+page.server.ts b/src/psaggregator/src/routes/+page.server.ts index fdc50d8..682119e 100644 --- a/src/psaggregator/src/routes/+page.server.ts +++ b/src/psaggregator/src/routes/+page.server.ts @@ -62,7 +62,7 @@ export async function load() { orderBy: { sticky: "desc" }, - take: 5 + take: 10 }); const youtubeCommunityPosts = await prisma.information.findMany({ @@ -83,10 +83,32 @@ export async function load() { take: 3 }); + const instagramPosts = await prisma.information.findMany({ + where: { + importedFrom: { + equals: "Instagram" + }, + href: { + not: null + }, + date: { + not: null + } + }, + include: { + InformationResource: true, + }, + orderBy: { + date: "desc" + }, + take: 3 + }); + return { videos, upcomingStreams, youtubeCommunityPosts, + instagramPosts, today, twitchStatus, redditPosts diff --git a/src/psaggregator/src/routes/+page.svelte b/src/psaggregator/src/routes/+page.svelte index 00c5705..e86807b 100644 --- a/src/psaggregator/src/routes/+page.svelte +++ b/src/psaggregator/src/routes/+page.svelte @@ -4,8 +4,10 @@ import PSVideo from "$lib/components/PSVideo.svelte"; import TwitchStatus from "$lib/components/TwitchStatus.svelte"; import RedditPost from "$lib/components/RedditPost.svelte"; - import { List, DataError, VideoPlayer, LogoYoutube } from "carbon-icons-svelte"; + import { List, VideoPlayer, LogoYoutube, LogoInstagram } from "carbon-icons-svelte"; import YouTubeCommunityPost from "$lib/components/YouTubeCommunityPost.svelte"; + import MediaQuery from "$lib/utils/MediaQuery.svelte"; + import InstagramPost from "$lib/components/InstagramPost.svelte"; export let data: PageServerData; @@ -30,77 +32,90 @@ } -
-
-
-
- - Uploadplan + +
+
+
+
+ + Uploadplan +
+
+ {#each data.today as content} + + {:else} +
+ Bisher konnte heute kein Uploadplan importiert werden. +
+ {/each} +
-
- {#each data.today as content} - - {:else} -
- Bisher konnte heute kein Uploadplan importiert werden. + {#if data.twitchStatus} +
+
+ twitch + Twitch
- {/each} -
+ +
+ {/if}
- {#if data.twitchStatus} -
+
+
- twitch - Twitch + + YouTube +
+
+ {#each data.youtubeCommunityPosts as youtube} + + {/each}
- -
- {/if} -
-
-
-
- - YouTube
-
- {#each data.youtubeCommunityPosts as youtube} - - {/each} +
+
+ twitch + Anstehende Streams +
+
+ {#each data.upcomingStreams as stream} + + {/each} +
-
-
-
- twitch - Anstehende Streams +
+
+ + Instagram +
+
+ {#each data.instagramPosts as instagram} + + {/each} +
-
- {#each data.upcomingStreams as stream} - - {/each} +
+
+ reddit + Reddit +
+
+ {#each data.redditPosts.slice(0, matches ? 10 : 5) as reddit} + + {/each} +
- reddit - Reddit + + Neuste Videos
-
- {#each data.redditPosts as reddit} - +
+ {#each data.videos as video} + {/each}
-
-
- - Neuste Videos -
-
- {#each data.videos as video} - - {/each} -
-
-
+ From 61585298e24a8de87de87a84946a54989445c8ae Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 08:22:25 +0100 Subject: [PATCH 09/19] handle insta posts without image? --- src/dataimport/instagram.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/dataimport/instagram.py b/src/dataimport/instagram.py index 5b4a06b..15d2efe 100644 --- a/src/dataimport/instagram.py +++ b/src/dataimport/instagram.py @@ -146,6 +146,13 @@ async def instagram(): thumbnail_url = resource.thumbnail_url break + if not thumbnail_url: + console.log( + f"Media item {remote_id} has no thumbnail, skipping", + style="bold red", + ) + continue + console.log(f"Downloading thumbnail for {remote_id}") try: thumbnail = requests.get(thumbnail_url).content From 36f1d2bb96c736aea7f4af20a45112cee1d4cd5f Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 08:25:24 +0100 Subject: [PATCH 10/19] add instagram to news --- .../src/routes/news/+page.server.ts | 24 ++++++++++++++++++- src/psaggregator/src/routes/news/+page.svelte | 15 ++++-------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/src/psaggregator/src/routes/news/+page.server.ts b/src/psaggregator/src/routes/news/+page.server.ts index 490a65d..bdf3d43 100644 --- a/src/psaggregator/src/routes/news/+page.server.ts +++ b/src/psaggregator/src/routes/news/+page.server.ts @@ -19,7 +19,29 @@ export async function load() { take: 10 }); + const instagramPosts = await prisma.information.findMany({ + where: { + importedFrom: { + equals: "Instagram" + }, + href: { + not: null + }, + date: { + not: null + } + }, + include: { + InformationResource: true, + }, + orderBy: { + date: "desc" + }, + take: 10 + }); + return { - youtubeCommunityPosts + youtubeCommunityPosts, + instagramPosts }; } diff --git a/src/psaggregator/src/routes/news/+page.svelte b/src/psaggregator/src/routes/news/+page.svelte index 67386f5..725891b 100644 --- a/src/psaggregator/src/routes/news/+page.svelte +++ b/src/psaggregator/src/routes/news/+page.svelte @@ -1,4 +1,5 @@
@@ -19,6 +94,11 @@ {/each}
+ {#if loading[ImportType.YouTube]} +
+ +
+ {/if}
@@ -30,6 +110,11 @@ {/each}
+ {#if loading[ImportType.Instagram]} +
+ +
+ {/if}
From adf2c5d3d99966ff140e878ef8cd4dbfa985e939 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 13:47:53 +0100 Subject: [PATCH 12/19] upgrade adapter-node and sveltekit, downgrade prisma --- src/psaggregator/package-lock.json | 98 +++++++++++------------------- src/psaggregator/package.json | 10 +-- src/psaggregator/src/lib/prisma.ts | 8 ++- 3 files changed, 45 insertions(+), 71 deletions(-) diff --git a/src/psaggregator/package-lock.json b/src/psaggregator/package-lock.json index 46bd363..0a50b58 100644 --- a/src/psaggregator/package-lock.json +++ b/src/psaggregator/package-lock.json @@ -1,21 +1,21 @@ { "name": "psaggregator", - "version": "1.2.3", + "version": "1.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "psaggregator", - "version": "1.2.3", + "version": "1.3.0", "devDependencies": { "@fontsource/fira-mono": "^4.5.10", "@neoconfetti/svelte": "^1.0.0", - "@prisma/client": "^5.8.0", + "@prisma/client": "4.15.0-integration-feat-client-esm.27", "@skeletonlabs/skeleton": "^2.7.0", "@skeletonlabs/tw-plugin": "^0.3.1", "@sveltejs/adapter-auto": "^3.0.0", - "@sveltejs/adapter-node": "^2.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-node": "^4.0.1", + "@sveltejs/kit": "^2.4.3", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.7", "@types/eslint": "8.56.0", @@ -34,7 +34,7 @@ "prettier": "^3.1.1", "prettier-plugin-svelte": "^3.1.2", "prettier-plugin-tailwindcss": "^0.5.9", - "prisma": "^5.8.0", + "prisma": "4.15.0-integration-feat-client-esm.27", "svelte": "^4.2.7", "svelte-check": "^3.6.0", "tailwindcss": "^3", @@ -734,13 +734,16 @@ "dev": true }, "node_modules/@prisma/client": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/client/-/client-5.8.0.tgz", - "integrity": "sha512-QxO6C4MaA/ysTIbC+EcAH1aX/YkpymhXtO6zPdk+FvA7+59tNibIYpd+7koPdViLg2iKES4ojsxWNUGNJaEcbA==", + "version": "4.15.0-integration-feat-client-esm.27", + "resolved": "https://registry.npmjs.org/@prisma/client/-/client-4.15.0-integration-feat-client-esm.27.tgz", + "integrity": "sha512-d1+EXkJpNaLOFH4x2FIRj66rkdW8g8wAwDcw7ayPNzix+1MKW4IBWncZLZDf5by9gSwHJlCb5N2BTRJmUnE3qQ==", "dev": true, "hasInstallScript": true, + "dependencies": { + "@prisma/engines-version": "4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944" + }, "engines": { - "node": ">=16.13" + "node": ">=14.17" }, "peerDependencies": { "prisma": "*" @@ -751,51 +754,19 @@ } } }, - "node_modules/@prisma/debug": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/debug/-/debug-5.8.0.tgz", - "integrity": "sha512-ZqPpkvbovu/kQJ1bvy57NO4dw97fpQGcbQSCtsqlwSE1UNKJP75R3BKxdznk8ZPMY+GJdMRetWNv4oAvSbWn8Q==", - "dev": true - }, "node_modules/@prisma/engines": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-5.8.0.tgz", - "integrity": "sha512-Qhqm9WWLujNEC13AuZlUO14SQ15tNLe5puaz+tOk7UqINqJ3PtqMmuSuzomiw2diGVqZ+HYiSQzlR3+pPucVHA==", + "version": "4.15.0-integration-feat-client-esm.27", + "resolved": "https://registry.npmjs.org/@prisma/engines/-/engines-4.15.0-integration-feat-client-esm.27.tgz", + "integrity": "sha512-xzD1472StQ6GA3gaZpPVwj6qlkbs92mPMTSESWiDCtDvkyPvN/g1qgrdPxRt1uFuPM2NSj5Ngc2InyfwRUAQFA==", "dev": true, - "hasInstallScript": true, - "dependencies": { - "@prisma/debug": "5.8.0", - "@prisma/engines-version": "5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848", - "@prisma/fetch-engine": "5.8.0", - "@prisma/get-platform": "5.8.0" - } + "hasInstallScript": true }, "node_modules/@prisma/engines-version": { - "version": "5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848", - "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848.tgz", - "integrity": "sha512-cXcoVweYbnv8xRfkWq9oj8BECOdzHUazrSpYCa0ehp5TNz4l5Spa8jbq/VROCTzj3ZncH5D9Q2TmySYTOUeKlw==", + "version": "4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944", + "resolved": "https://registry.npmjs.org/@prisma/engines-version/-/engines-version-4.15.0-28.8fbc245156db7124f997f4cecdd8d1219e360944.tgz", + "integrity": "sha512-sVOig4tjGxxlYaFcXgE71f/rtFhzyYrfyfNFUsxCIEJyVKU9rdOWIlIwQ2NQ7PntvGnn+x0XuFo4OC1jvPJKzg==", "dev": true }, - "node_modules/@prisma/fetch-engine": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/fetch-engine/-/fetch-engine-5.8.0.tgz", - "integrity": "sha512-1CAuE+JoYsPNggMEn6qk0zos06Uc9bYZBJ0VBPHD6R7REL05614koAbOCmn52IaYz3nobb7f25hqW6AY7rLkIw==", - "dev": true, - "dependencies": { - "@prisma/debug": "5.8.0", - "@prisma/engines-version": "5.8.0-37.0a83d8541752d7582de2ebc1ece46519ce72a848", - "@prisma/get-platform": "5.8.0" - } - }, - "node_modules/@prisma/get-platform": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/@prisma/get-platform/-/get-platform-5.8.0.tgz", - "integrity": "sha512-Nk3rhTFZ1LYkFZJnpSvQcLPCaBWgJQfteHII6UEENOOkYlmP0k3FuswND54tzzEr4qs39wOdV9pbXKX9U2lv7A==", - "dev": true, - "dependencies": { - "@prisma/debug": "5.8.0" - } - }, "node_modules/@rollup/plugin-commonjs": { "version": "25.0.7", "resolved": "https://registry.npmjs.org/@rollup/plugin-commonjs/-/plugin-commonjs-25.0.7.tgz", @@ -1143,24 +1114,24 @@ } }, "node_modules/@sveltejs/adapter-node": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-node/-/adapter-node-2.1.1.tgz", - "integrity": "sha512-ypAqdvjoyfOEgucDbY3P6DM6nLEQ7OwiWMfyITKF8svffe9FIx1ow6uELJd1BzbCrouYndkQfc5m6ihj0xxqTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-node/-/adapter-node-4.0.1.tgz", + "integrity": "sha512-IviiTtKCDp+0QoTmmMlGGZBA1EoUNsjecU6XGV9k62S3f01SNsVhpqi2e4nbI62BLGKh/YKKfFii+Vz/b9XIxg==", "dev": true, "dependencies": { "@rollup/plugin-commonjs": "^25.0.7", "@rollup/plugin-json": "^6.1.0", "@rollup/plugin-node-resolve": "^15.2.3", - "rollup": "^4.8.0" + "rollup": "^4.9.5" }, "peerDependencies": { - "@sveltejs/kit": "^2.0.0" + "@sveltejs/kit": "^2.4.0" } }, "node_modules/@sveltejs/kit": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.3.2.tgz", - "integrity": "sha512-AzGWV1TyUSkBuciy06E5NegXndIEgTthDtllv80qynEJFh8bZD62ZxLajiQLOsKGqRDilEQyshDARQxjIqiaqg==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", + "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", "dev": true, "hasInstallScript": true, "dependencies": { @@ -3789,19 +3760,20 @@ } }, "node_modules/prisma": { - "version": "5.8.0", - "resolved": "https://registry.npmjs.org/prisma/-/prisma-5.8.0.tgz", - "integrity": "sha512-hDKoEqPt2qEUTH5yGO3l27CBnPtwvte0CGMKrpCr9+/A919JghfqJ3qgCGgMbOwdkXUOzdho0RH9tyUF3UhpMw==", + "version": "4.15.0-integration-feat-client-esm.27", + "resolved": "https://registry.npmjs.org/prisma/-/prisma-4.15.0-integration-feat-client-esm.27.tgz", + "integrity": "sha512-hVuVm+KPUhPRbTE0Z86wnaD2P6wBu2byP6w7fQ8z5kM2/STrkGLz4xja8AlQQQ43bZObvmTy1GMP0eHTbi7G2Q==", "dev": true, "hasInstallScript": true, "dependencies": { - "@prisma/engines": "5.8.0" + "@prisma/engines": "4.15.0-integration-feat-client-esm.27" }, "bin": { - "prisma": "build/index.js" + "prisma": "build/index.js", + "prisma2": "build/index.js" }, "engines": { - "node": ">=16.13" + "node": ">=14.17" } }, "node_modules/proxy-from-env": { diff --git a/src/psaggregator/package.json b/src/psaggregator/package.json index 283e7d9..0f51d39 100644 --- a/src/psaggregator/package.json +++ b/src/psaggregator/package.json @@ -1,6 +1,6 @@ { "name": "psaggregator", - "version": "1.2.3", + "version": "1.3.0", "scripts": { "dev": "vite dev", "build": "vite build", @@ -16,12 +16,12 @@ "devDependencies": { "@fontsource/fira-mono": "^4.5.10", "@neoconfetti/svelte": "^1.0.0", - "@prisma/client": "^5.8.0", + "@prisma/client": "4.15.0-integration-feat-client-esm.27", "@skeletonlabs/skeleton": "^2.7.0", "@skeletonlabs/tw-plugin": "^0.3.1", "@sveltejs/adapter-auto": "^3.0.0", - "@sveltejs/adapter-node": "^2.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-node": "^4.0.1", + "@sveltejs/kit": "^2.4.3", "@sveltejs/vite-plugin-svelte": "^3.0.0", "@tailwindcss/forms": "^0.5.7", "@types/eslint": "8.56.0", @@ -40,7 +40,7 @@ "prettier": "^3.1.1", "prettier-plugin-svelte": "^3.1.2", "prettier-plugin-tailwindcss": "^0.5.9", - "prisma": "^5.8.0", + "prisma": "4.15.0-integration-feat-client-esm.27", "svelte": "^4.2.7", "svelte-check": "^3.6.0", "tailwindcss": "^3", diff --git a/src/psaggregator/src/lib/prisma.ts b/src/psaggregator/src/lib/prisma.ts index 8301a4c..921a1f3 100644 --- a/src/psaggregator/src/lib/prisma.ts +++ b/src/psaggregator/src/lib/prisma.ts @@ -1,6 +1,8 @@ -import { createRequire } from "module"; // https://github.com/prisma/prisma/issues/15614 -const require = createRequire(import.meta.url); -const { PrismaClient } = require("@prisma/client"); +import { createRequire as _createRequire } from 'module' +export const createCursedRequire: (path: string | URL) => (id: string) => TShape = _createRequire + +const require = createCursedRequire(import.meta.url ?? __filename) +const { PrismaClient } = require('@prisma/client'); const prisma = new PrismaClient(); From a618144687d178b853164150befb90a35870d314 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 13:48:59 +0100 Subject: [PATCH 13/19] fix kofi env in .env.example --- .env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.example b/.env.example index 88766d5..0eff420 100644 --- a/.env.example +++ b/.env.example @@ -10,7 +10,7 @@ MYSQL_DATABASE=psaggregator MYSQL_ROOT_PASSWORD=psaggregator LEGAL_URL= -PUBLIC_KOFI_USERNAME=zaanposni +KOFI_USERNAME=zaanposni OPENAI_API_KEY= From a3ae7866dfd2de47ec79fa36c56fd4acdfc19115 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 14:19:39 +0100 Subject: [PATCH 14/19] change order of columns on homepage --- src/psaggregator/src/routes/+page.svelte | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/psaggregator/src/routes/+page.svelte b/src/psaggregator/src/routes/+page.svelte index e86807b..4428039 100644 --- a/src/psaggregator/src/routes/+page.svelte +++ b/src/psaggregator/src/routes/+page.svelte @@ -61,7 +61,7 @@ {/if}
-
+
YouTube @@ -72,18 +72,7 @@ {/each}
-
-
- twitch - Anstehende Streams -
-
- {#each data.upcomingStreams as stream} - - {/each} -
-
-
+
Instagram @@ -94,7 +83,7 @@ {/each}
-
+
reddit Reddit @@ -105,6 +94,17 @@ {/each}
+
+
+ twitch + Anstehende Streams +
+
+ {#each data.upcomingStreams as stream} + + {/each} +
+
From e52410248d48000da3fac0d1924f3666756ee361 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 14:20:36 +0100 Subject: [PATCH 15/19] empty streams --- src/psaggregator/src/routes/+page.svelte | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/psaggregator/src/routes/+page.svelte b/src/psaggregator/src/routes/+page.svelte index 4428039..90d865e 100644 --- a/src/psaggregator/src/routes/+page.svelte +++ b/src/psaggregator/src/routes/+page.svelte @@ -102,6 +102,10 @@
{#each data.upcomingStreams as stream} + {:else} +
+ Momentan sind keine geplanten Streams bekannt. +
{/each}
From d39f3fe155698d439bb7e73e5de6297f2eb5dbf6 Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 14:26:03 +0100 Subject: [PATCH 16/19] updated readme with newest features --- readme.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/readme.md b/readme.md index 2ccd87e..dc922aa 100644 --- a/readme.md +++ b/readme.md @@ -7,6 +7,15 @@ PietSmiet Aggregator is a selfhostable web application that aggregates all the videos, streams and additional content from PietSmiet and displays them in a nice overview. +⭐ **Dashboard** - Display all PietSmiet videos and streams in a nice overview. +⭐ **API** - Free and Public JSON HTTP API. +⭐ **PietSmietDE Import** - Import all videos and news from the PietSmiet website. +⭐ **YouTube Import** - Import all community posts from the PietSmiet YouTube channel. +⭐ **Reddit Import** - Import trending posts from r/pietsmiet. +⭐ **Instagram Import** - Import all posts. +⭐ **OpenAI Streamingplan Analysis** - Analyze the PietSmiet Streamingplan with OpenAI Vision and import scheduled streams into the database. +⭐ **Full Control** - Selfhostable. + ## ⚡️ Quick Start Self Deployment First, [download][docker_download_url] and install **Docker**. From 8de0dc8dcd4b268759248fe055a4610225c3c9ea Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 14:27:09 +0100 Subject: [PATCH 17/19] added missing newlines to readme --- readme.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/readme.md b/readme.md index dc922aa..c6f7146 100644 --- a/readme.md +++ b/readme.md @@ -7,13 +7,13 @@ PietSmiet Aggregator is a selfhostable web application that aggregates all the videos, streams and additional content from PietSmiet and displays them in a nice overview. -⭐ **Dashboard** - Display all PietSmiet videos and streams in a nice overview. -⭐ **API** - Free and Public JSON HTTP API. -⭐ **PietSmietDE Import** - Import all videos and news from the PietSmiet website. -⭐ **YouTube Import** - Import all community posts from the PietSmiet YouTube channel. -⭐ **Reddit Import** - Import trending posts from r/pietsmiet. -⭐ **Instagram Import** - Import all posts. -⭐ **OpenAI Streamingplan Analysis** - Analyze the PietSmiet Streamingplan with OpenAI Vision and import scheduled streams into the database. +⭐ **Dashboard** - Display all PietSmiet videos and streams in a nice overview.\ +⭐ **API** - Free and Public JSON HTTP API.\ +⭐ **PietSmietDE Import** - Import all videos and news from the PietSmiet website.\ +⭐ **YouTube Import** - Import all community posts from the PietSmiet YouTube channel.\ +⭐ **Reddit Import** - Import trending posts from r/pietsmiet.\ +⭐ **Instagram Import** - Import all posts.\ +⭐ **OpenAI Streamingplan Analysis** - Analyze the PietSmiet Streamingplan with OpenAI Vision and import scheduled streams into the database.\ ⭐ **Full Control** - Selfhostable. ## ⚡️ Quick Start Self Deployment From 2044c68955d1e4cd938612f294f6b7889996af0c Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 14:28:14 +0100 Subject: [PATCH 18/19] . --- readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index c6f7146..cfcca55 100644 --- a/readme.md +++ b/readme.md @@ -13,8 +13,8 @@ PietSmiet Aggregator is a selfhostable web application that aggregates all the v ⭐ **YouTube Import** - Import all community posts from the PietSmiet YouTube channel.\ ⭐ **Reddit Import** - Import trending posts from r/pietsmiet.\ ⭐ **Instagram Import** - Import all posts.\ -⭐ **OpenAI Streamingplan Analysis** - Analyze the PietSmiet Streamingplan with OpenAI Vision and import scheduled streams into the database.\ -⭐ **Full Control** - Selfhostable. +⭐ **OpenAI Streamingplan Analysis** - Analyze the Streamingplan with OpenAI Vision and import scheduled streams into the database.\ +⭐ **Full Control** - Selfhostablem, Open Source and Dockerized. ## ⚡️ Quick Start Self Deployment From e0e80486f3ded22ad1224e93c4fdf452cdd63b5e Mon Sep 17 00:00:00 2001 From: zaanposni Date: Fri, 26 Jan 2024 14:32:32 +0100 Subject: [PATCH 19/19] remove unneeded css class --- src/psaggregator/src/lib/components/InstagramPost.svelte | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/psaggregator/src/lib/components/InstagramPost.svelte b/src/psaggregator/src/lib/components/InstagramPost.svelte index 3968395..1c48a39 100644 --- a/src/psaggregator/src/lib/components/InstagramPost.svelte +++ b/src/psaggregator/src/lib/components/InstagramPost.svelte @@ -10,15 +10,9 @@ } - -
- {titleCase(post.additionalInfo)} + {titleCase(post.additionalInfo)} {#if post.date} {moment(post.date).fromNow()} {/if}