Skip to content

Commit

Permalink
black formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
alexcannan committed Jul 8, 2023
1 parent ee2adb1 commit 6de38fd
Show file tree
Hide file tree
Showing 5 changed files with 40 additions and 28 deletions.
3 changes: 0 additions & 3 deletions .github/workflows/test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,6 @@ jobs:
- name: Running ruff
run: ruff .

- name: Running black
run: black .

- name: Running mypy
run: make typecheck

Expand Down
5 changes: 4 additions & 1 deletion articlesa/logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,7 @@
from loguru import logger

logger.remove()
logger.add(sys.stderr, format='<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <5}</level> | <magenta>{process}</magenta> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>')
logger.add(
sys.stderr,
format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | <level>{level: <5}</level> | <magenta>{process}</magenta> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
)
8 changes: 7 additions & 1 deletion articlesa/serve/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,10 @@
import uvicorn


uvicorn.run("articlesa.serve:app", host="localhost", port=7654, reload=True, reload_dirs=[Path(__file__).parent])
uvicorn.run(
"articlesa.serve:app",
host="localhost",
port=7654,
reload=True,
reload_dirs=[Path(__file__).parent],
)
38 changes: 22 additions & 16 deletions articlesa/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,51 +10,55 @@
from yarl import URL



def clean_url(url: Union[str, URL]) -> str:
""" Parse article urls, remove query strings and fragments. """
"""Parse article urls, remove query strings and fragments."""
_parsed = urlparse(str(url))
_parsed = _parsed._replace(query='', fragment='')
_parsed = _parsed._replace(query="", fragment="")
return _parsed.geturl()


def relative_to_absolute_url(relative_url: str, base_url: str) -> str:
""" Given a relative url and a base url, return an absolute url. """
assert relative_url.startswith('/')
return urlparse(base_url)._replace(path=relative_url, query='', fragment='').geturl()
"""Given a relative url and a base url, return an absolute url."""
assert relative_url.startswith("/")
return (
urlparse(base_url)._replace(path=relative_url, query="", fragment="").geturl()
)


def url_to_hash(url: str) -> str:
""" Build hash unique to url. """
"""Build hash unique to url."""
return hashlib.md5(url.encode()).hexdigest() # noqa: S324


def read_blacklist() -> set:
""" Blacklist of netlocs to ignore. """
with Path('blacklist.txt').open('r') as f:
"""Blacklist of netlocs to ignore."""
with Path("blacklist.txt").open("r") as f:
blacklist = set()
for line in f:
if (sline := line.strip()):
if sline := line.strip():
blacklist.add(sline)
return blacklist


class PlaceholderArticle(BaseModel):
""" Objects for when a source is found but it's still processing. """
"""Objects for when a source is found but it's still processing."""

urlhash: str
depth: int
parent: Optional[str]


class ParseFailure(BaseModel):
""" Object returned from parse worker when parse failed. """
"""Object returned from parse worker when parse failed."""

message: str
status: Optional[int] = None
urlhash: Optional[str] = None


class ParsedArticle(BaseModel):
""" Object returned from parse worker, to be stored to & retrieved from redis. """
"""Object returned from parse worker, to be stored to & retrieved from redis."""

url: str
title: str
text: str
Expand All @@ -67,7 +71,8 @@ class ParsedArticle(BaseModel):


class StreamEvent(Enum):
""" SSE event types. """
"""SSE event types."""

STREAM_BEGIN = "stream_begin"
NODE_PROCESSING = "node_processing"
NODE_RENDER = "node_render"
Expand All @@ -91,13 +96,14 @@ class SSE(BaseModel):
reconnect. This must be an integer, specifying the reconnection time in
milliseconds. If a non-integer value is specified, the field is ignored.
"""

data: Optional[Json]
id: str
event: str # expects StreamEvent value
retry: int = 15000 # ms

@validator('event')
def event_must_be_valid(cls: 'SSE', v: str) -> str:
@validator("event")
def event_must_be_valid(cls: "SSE", v: str) -> str:
"""Validate that event is a valid StreamEvent."""
assert v in [e.value for e in StreamEvent]
return v
14 changes: 7 additions & 7 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
from setuptools import setup, find_namespace_packages

setup(
name='articlesa',
version='0',
python_requires='>=3.10',
author='Alex Cannan',
author_email='alexfcannan@gmail.com',
packages=find_namespace_packages(include=['articlesa.*']),
name="articlesa",
version="0",
python_requires=">=3.10",
author="Alex Cannan",
author_email="alexfcannan@gmail.com",
packages=find_namespace_packages(include=["articlesa.*"]),
long_description="article source aggregator generates source maps for online articles",
install_requires=Path("requirements.txt").read_text().split("\n")
install_requires=Path("requirements.txt").read_text().split("\n"),
)

0 comments on commit 6de38fd

Please sign in to comment.