FilterUnreasonablePrices renamed to PriceParser

lwitkowski · Aug 3, 2024 · 288a3d7 · 288a3d7
1 parent 75ee69a
commit 288a3d7
Show file tree

Hide file tree

Showing 4 changed files with 17 additions and 10 deletions.
diff --git a/backend/pipelines.py b/backend/pipelines.py
@@ -17,7 +17,7 @@ def process_item(self, item, _):
         return item
 
 
-class FilterUnreasonablePrices(object):
+class PriceParser(object):
 
     def process_item(self, item, _):
         price = Price.fromstring(item['raw_price'])

diff --git a/backend/settings.py b/backend/settings.py
@@ -15,7 +15,7 @@
 
 # scrapy pipeline components config, do not delete this
 ITEM_PIPELINES = {
-    'pipelines.FilterUnreasonablePrices': 100,
+    'pipelines.PriceParser': 100,
     'pipelines.FilterSearchAndCharterOffers': 200,
     'pipelines.DuplicateDetection': 300,
     'pipelines.StoragePipeline': 400,

diff --git a/backend/tests/test__testcontainers_setup.py b/backend/tests/test__testcontainers_setup.py
@@ -6,7 +6,7 @@
 from testcontainers.postgres import PostgresContainer
 
 db_migration_scripts_location = Path(__file__).parent.parent.parent / "db" / "migrations"
-postgres = PostgresContainer("postgres:16-alpine")
+postgres = PostgresContainer("postgres:16.3-alpine")
 postgres.with_volume_mapping(host=str(db_migration_scripts_location), container=f"/docker-entrypoint-initdb.d/")
 postgres.start()
 

diff --git a/backend/tests/test_pipelines.py b/backend/tests/test_pipelines.py
@@ -1,4 +1,5 @@
 import unittest
+from decimal import Decimal
 
 from price_parser import Price
 from scrapy.exceptions import DropItem
@@ -35,21 +36,27 @@ def test_existing_offer_is_duplicate(self):
         self.assertRaises(DropItem, self.detection.process_item, {"offer_url": "https://offers.com/1"}, None)
 
 @ddt
-class FilterUnreasonablePricesTest(unittest.TestCase):
+class PriceParserTest(unittest.TestCase):
     def setUp(self):
         self.sample_offer = buildOfferWithUrl("https://offers.com/1")
-        self.detection = pipelines.FilterUnreasonablePrices()
+        self.detection = pipelines.PriceParser()
 
     @data(
-        {"raw_price": "2,01 Euro €", "offer_url": "https://offers.com/1"},
-        {"raw_price": "1.234,00 Euro €", "offer_url": "https://offers.com/2"},
-        {"raw_price": "123.456,00 Euro €", "offer_url": "https://offers.com/3"},
+        ({"raw_price": "2,01 Euro €", "offer_url": "https://offers.com/1"}, 2.01),
+        ({"raw_price": "1.234,00 Euro €", "offer_url": "https://offers.com/2"}, 1_234.00),
+        ({"raw_price": "123.456,00 Euro €", "offer_url": "https://offers.com/3"}, 123_456.00),
     )
-    def test_should_allow_valid_prices(self, offer_with_valid_price):
+    def test_parse_valid_prices(self, testInput):
+        offer_with_valid_price = testInput[0]
+        expected_price = Decimal(testInput[1])
+        CENTS = Decimal(10) ** -2
         try:
             self.detection.process_item(offer_with_valid_price, None)
         except DropItem:
-            self.fail("FilterUnreasonablePrices unexpectedly dropped offer with valid price!")
+            self.fail("PriceParser unexpectedly dropped offer with valid price!")
+
+        self.assertEqual(offer_with_valid_price["price"].amount.quantize(CENTS), expected_price.quantize(CENTS))
+        self.assertEqual(offer_with_valid_price["price"].currency, "€")
 
     @data(
         {"raw_price": "", "offer_url": "https://offers.com/1"},