fix(scrapers): COALESCE-backfill listing_date/area_m2 in save_listings ON CONFLICT (Refs #821) #829

Merged
bot-reviewer merged 1 commit from feat/821-coalesce-backfill into main 2026-05-30 19:58:07 +00:00
2 changed files with 60 additions and 1 deletions

View file

@ -295,7 +295,9 @@ def save_listings(
is_pro_seller = EXCLUDED.is_pro_seller,
bargain_allowed = EXCLUDED.bargain_allowed,
sale_type = EXCLUDED.sale_type,
metro_stations = EXCLUDED.metro_stations
metro_stations = EXCLUDED.metro_stations,
listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2)
RETURNING id, (xmax = 0) AS inserted
"""
),

View file

@ -524,3 +524,60 @@ def test_dedup_hash_distinguishes_source_even_same_id():
lot_avito = _base_lot(source="avito", source_id="500")
lot_cian = _base_lot(source="cian", source_id="500")
assert lot_avito.compute_dedup_hash() != lot_cian.compute_dedup_hash()
# ── Tests 15-17: #821 COALESCE backfill listing_date / area_m2 ──────────────
def test_save_listings_sql_contains_coalesce_listing_date_area_m2():
"""#821: ON CONFLICT SET block содержит COALESCE для listing_date и area_m2."""
db = _mock_db(inserted=False) # conflict path
lot = _cian_lot()
save_listings(db, [lot])
sql_text = _get_listings_insert_sql(db)
assert "COALESCE(EXCLUDED.listing_date, listings.listing_date)" in sql_text
assert "COALESCE(EXCLUDED.area_m2, listings.area_m2)" in sql_text
def test_save_listings_params_listing_date_area_m2_passed_through():
"""#821: listing_date и area_m2 попадают в params INSERT (VALUES side)."""
import datetime
db = _mock_db(inserted=True)
lot = _base_lot(
source_id="ld_test",
area_m2=47.5,
listing_date=datetime.date(2024, 3, 15),
)
save_listings(db, [lot])
params = _get_execute_params(db)
assert params["area_m2"] == 47.5
assert params["listing_date"] == datetime.date(2024, 3, 15)
def test_save_listings_null_area_and_date_pass_none_in_params():
"""#821: re-scrape без area_m2/listing_date → params содержат None.
COALESCE в SQL сохранит существующее значение в БД здесь проверяем
только что params не ломают вызов (None передаётся без ошибки).
"""
db = _mock_db(inserted=False) # conflict/update path
lot = _base_lot(
source_id="null_test",
area_m2=None,
listing_date=None,
)
inserted, updated = save_listings(db, [lot])
# Ожидаем updated (xmax != 0)
assert inserted == 0
assert updated == 1
params = _get_execute_params(db)
assert params["area_m2"] is None
assert params["listing_date"] is None