Add more Article columns

alembic/versions/2023_12_04_2010-1feba14fd658_add_more_article_fields.py

@@ -0,0 +1,38 @@
+"""add more article fields
+
+Revision ID: 1feba14fd658
+Revises: 70e5e82e94c0 # noqa: W291
+Create Date: 2023-12-04 20:10:13.657413
+
+"""
+import sqlalchemy as sa
+import sqlmodel.sql.sqltypes
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "1feba14fd658"
+down_revision = "70e5e82e94c0"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("article", sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True))
+    op.add_column("article", sa.Column("content", sa.Text(), nullable=False))
+    op.add_column("article", sa.Column("site_published_at", sa.DateTime(), nullable=False))
+    op.add_column("article", sa.Column("site_updated_at", sa.DateTime(), nullable=True))
+    op.drop_column("article", "published_at")
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column("article", sa.Column("published_at", postgresql.TIMESTAMP(), autoincrement=False, nullable=False))
+    op.drop_column("article", "site_updated_at")
+    op.drop_column("article", "site_published_at")
+    op.drop_column("article", "content")
+    op.drop_column("article", "description")
+    # ### end Alembic commands ###

article_rec_db/models/article.py

@@ -3,7 +3,7 @@
 from typing import Annotated
 from uuid import UUID
 
-from sqlmodel import Field, Relationship, String, UniqueConstraint
+from sqlmodel import Field, Relationship, String, Text, UniqueConstraint
 
 from article_rec_db.sites import SiteName
 
@@ -23,14 +23,31 @@ class Language(StrEnum):
 
 class Article(UpdateTracked, table=True):
     __table_args__ = (UniqueConstraint("site", "id_in_site", name="article_site_idinsite_unique"),)
-    __mapper_args__ = {"polymorphic_identity": "article"}
 
+    # Page ID refers to the page table
     page_id: Annotated[UUID, Field(primary_key=True, foreign_key="page.id")]
+
+    # Site name
     site: Annotated[SiteName, Field(sa_type=String)]
-    id_in_site: str  # ID of article in the partner site's internal system
+    # ID of article in the partner site's internal system
+    id_in_site: str
+
+    # Title/headline
     title: str
-    published_at: datetime
+    # Description/summary
+    description: str | None
+    # Full text of article; might also include (sanitized) HTML tags
+    content: Annotated[str, Field(sa_type=Text)]
+
+    # When the article was published on the partner site
+    site_published_at: datetime
+    # When the article was last updated on the partner site
+    site_updated_at: datetime | None
+
+    # Language of the article
     language: Language = Language.ENGLISH
+
+    # Whether the article is in-house content or not (e.g., republished from another source)
     is_in_house_content: bool = True
 
     # An article is always a page, but a page is not always an article

pyproject.toml

@@ -1,7 +1,7 @@
 [tool.poetry]
 description = "Database models and migrations for the Local News Lab's article recommendation system"
 name = "article-rec-db"
-version = "0.0.7"
+version = "0.0.8"
 authors = ["Duy Nguyen <hello.duyknguyen@gmail.com>"]
 license = "MIT"
 readme = "README.md"

tests/integration/models/conftest.py

@@ -3,7 +3,7 @@
 from enum import StrEnum
 
 import pytest
-from sqlalchemy.future.engine import Engine, create_engine
+from sqlalchemy.engine import Engine, create_engine
 from sqlmodel import text
 
 import alembic.command

tests/integration/models/test_article.py

@@ -22,7 +22,9 @@ def test_add_article_with_page(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article",
-        published_at=article_published_at,
+        description="Description",
+        content="<p>Content</p>",
+        site_published_at=article_published_at,
         language=Language.SPANISH,
         page=page,
     )
@@ -44,7 +46,10 @@ def test_add_article_with_page(refresh_tables, engine):
         assert article.site == DALLAS_FREE_PRESS.name
         assert article.id_in_site == "1234"
         assert article.title == "Example Article"
-        assert article.published_at == article_published_at
+        assert article.description == "Description"
+        assert article.content == "<p>Content</p>"
+        assert article.site_published_at == article_published_at
+        assert article.site_updated_at is None
         assert article.language == Language.SPANISH
         assert article.is_in_house_content is True
         assert article.page is page
@@ -59,7 +64,8 @@ def test_add_article_without_page(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="2345",
         title="Example Article",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
     )
 
     with Session(engine) as session:
@@ -90,14 +96,16 @@ def test_add_articles_duplicate_site_and_id_in_site(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site=id_in_site,
         title="Example Article",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page1,
     )
     article2 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site=id_in_site,
         title="Example Article 2",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page2,
     )
 
@@ -127,7 +135,8 @@ def test_update_article(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page,
     )
 
@@ -139,10 +148,14 @@ def test_update_article(refresh_tables, engine):
         assert article.db_updated_at is None
 
         article.title = "Example Article with Title Updated"
+        site_updated_at = datetime.utcnow()
+        article.site_updated_at = site_updated_at
         session.commit()
 
         # Upon update, db_updated_at should be set
         assert isinstance(article.db_updated_at, datetime)
+        assert article.title == "Example Article with Title Updated"
+        assert article.site_updated_at == site_updated_at
 
 
 def test_delete_article(refresh_tables, engine):
@@ -160,14 +173,16 @@ def test_delete_article(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article 1",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page1,
     )
     article2 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site="2345",
         title="Example Article 2",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page2,
     )
 

tests/integration/models/test_embedding.py

@@ -24,7 +24,8 @@ def test_add_embedding(refresh_tables, engine, rng):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page,
     )
     execution = Execution(
@@ -70,21 +71,24 @@ def test_select_embeddings_knn(refresh_tables, engine, rng):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page1,
     )
     article2 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site="2345",
         title="Example Article 2",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page2,
     )
     article3 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site="3456",
         title="Example Article 3",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page3,
     )
     execution = Execution(
@@ -152,14 +156,16 @@ def test_delete_embedding(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article 1",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page1,
     )
     article2 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site="2345",
         title="Example Article 2",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page2,
     )
 

tests/integration/models/test_execution.py

@@ -41,14 +41,16 @@ def test_delete_execution(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article 1",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page1,
     )
     article2 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site="2345",
         title="Example Article 2",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page2,
     )
 

tests/integration/models/test_page.py

@@ -85,14 +85,16 @@ def test_delete_page(refresh_tables, engine):
         site=DALLAS_FREE_PRESS.name,
         id_in_site="1234",
         title="Example Article 1",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page1,
     )
     article2 = Article(
         site=DALLAS_FREE_PRESS.name,
         id_in_site="2345",
         title="Example Article 2",
-        published_at=datetime.utcnow(),
+        content="<p>Content</p>",
+        site_published_at=datetime.utcnow(),
         page=page2,
     )