Ony hyrdate posts 7 days old

Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /usr/src/app/
 COPY requirements.txt ./
 RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy the rest of the application code
+# Copy the rest of the application code except .dockerignore files
 COPY . .
 
 EXPOSE 8000

docker-compose.yml

@@ -11,7 +11,7 @@ services:
 
   firehose:
     container_name: firehose
-    image: richardr1126/firehose
+    image: richardr1126/cosmere-firehose
     build:
       context: ./firehose
     env_file:

firehose/Dockerfile

@@ -5,12 +5,9 @@ FROM python:3.12.8-slim-bookworm
 VOLUME /var/data/
 WORKDIR /usr/src/app/
 
-# Copy package files and install dependencies
-COPY ../requirements.txt ./
-RUN pip install --no-cache-dir -r requirements.txt
-
 # Copy the rest of the application code
 COPY . .
+RUN pip install --no-cache-dir -r requirements.txt
 
 # Runs when the container is started
 CMD python start_stream.py

firehose/db_scheduler.py

@@ -69,8 +69,8 @@ def hydrate_posts_with_interactions(client: Client, batch_size: int = 25, schedu
     try:
         with db.connection_context():
             logger.info("Hydration Database connection opened.")
-            # get posts with uri and interactions
-            posts = Post.select(Post.uri, Post.interactions)
+            # get posts with uri and interactions from the last 7 days
+            posts = Post.select(Post.uri, Post.interactions).where(Post.indexed_at > (datetime.now() - timedelta(days=7)))
             uris = [post.uri for post in posts]
 
             if not uris:
@@ -134,7 +134,7 @@ def hydrate_posts_with_interactions(client: Client, batch_size: int = 25, schedu
                             posts_to_update.append(current_post)
 
                     # pause the loop for 3 seconds
-                    time.sleep(3)
+                    #time.sleep(3)
 
                 except exceptions.AtProtocolError as api_err:
                     if api_err.response:

requirements.txt

@@ -3,6 +3,4 @@ peewee
 Flask
 python-dotenv
 gunicorn
-honcho
-apscheduler
 psycopg2-binary