fix(scrapers): COALESCE-backfill listing_date/area_m2 in save_listings ON CONFLICT (Refs #821) #829
2 changed files with 60 additions and 1 deletions
|
|
@ -295,7 +295,9 @@ def save_listings(
|
|||
is_pro_seller = EXCLUDED.is_pro_seller,
|
||||
bargain_allowed = EXCLUDED.bargain_allowed,
|
||||
sale_type = EXCLUDED.sale_type,
|
||||
metro_stations = EXCLUDED.metro_stations
|
||||
metro_stations = EXCLUDED.metro_stations,
|
||||
listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
|
||||
area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2)
|
||||
RETURNING id, (xmax = 0) AS inserted
|
||||
"""
|
||||
),
|
||||
|
|
|
|||
|
|
@ -524,3 +524,60 @@ def test_dedup_hash_distinguishes_source_even_same_id():
|
|||
lot_avito = _base_lot(source="avito", source_id="500")
|
||||
lot_cian = _base_lot(source="cian", source_id="500")
|
||||
assert lot_avito.compute_dedup_hash() != lot_cian.compute_dedup_hash()
|
||||
|
||||
|
||||
# ── Tests 15-17: #821 COALESCE backfill listing_date / area_m2 ──────────────
|
||||
|
||||
|
||||
def test_save_listings_sql_contains_coalesce_listing_date_area_m2():
|
||||
"""#821: ON CONFLICT SET block содержит COALESCE для listing_date и area_m2."""
|
||||
db = _mock_db(inserted=False) # conflict path
|
||||
lot = _cian_lot()
|
||||
|
||||
save_listings(db, [lot])
|
||||
|
||||
sql_text = _get_listings_insert_sql(db)
|
||||
assert "COALESCE(EXCLUDED.listing_date, listings.listing_date)" in sql_text
|
||||
assert "COALESCE(EXCLUDED.area_m2, listings.area_m2)" in sql_text
|
||||
|
||||
|
||||
def test_save_listings_params_listing_date_area_m2_passed_through():
|
||||
"""#821: listing_date и area_m2 попадают в params INSERT (VALUES side)."""
|
||||
import datetime
|
||||
|
||||
db = _mock_db(inserted=True)
|
||||
lot = _base_lot(
|
||||
source_id="ld_test",
|
||||
area_m2=47.5,
|
||||
listing_date=datetime.date(2024, 3, 15),
|
||||
)
|
||||
|
||||
save_listings(db, [lot])
|
||||
|
||||
params = _get_execute_params(db)
|
||||
assert params["area_m2"] == 47.5
|
||||
assert params["listing_date"] == datetime.date(2024, 3, 15)
|
||||
|
||||
|
||||
def test_save_listings_null_area_and_date_pass_none_in_params():
|
||||
"""#821: re-scrape без area_m2/listing_date → params содержат None.
|
||||
|
||||
COALESCE в SQL сохранит существующее значение в БД — здесь проверяем
|
||||
только что params не ломают вызов (None передаётся без ошибки).
|
||||
"""
|
||||
db = _mock_db(inserted=False) # conflict/update path
|
||||
lot = _base_lot(
|
||||
source_id="null_test",
|
||||
area_m2=None,
|
||||
listing_date=None,
|
||||
)
|
||||
|
||||
inserted, updated = save_listings(db, [lot])
|
||||
|
||||
# Ожидаем updated (xmax != 0)
|
||||
assert inserted == 0
|
||||
assert updated == 1
|
||||
|
||||
params = _get_execute_params(db)
|
||||
assert params["area_m2"] is None
|
||||
assert params["listing_date"] is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue