fix(scrapers): COALESCE-backfill listing_date/area_m2 in save_listings ON CONFLICT (Refs #821) #829
2 changed files with 60 additions and 1 deletions
|
|
@ -295,7 +295,9 @@ def save_listings(
|
||||||
is_pro_seller = EXCLUDED.is_pro_seller,
|
is_pro_seller = EXCLUDED.is_pro_seller,
|
||||||
bargain_allowed = EXCLUDED.bargain_allowed,
|
bargain_allowed = EXCLUDED.bargain_allowed,
|
||||||
sale_type = EXCLUDED.sale_type,
|
sale_type = EXCLUDED.sale_type,
|
||||||
metro_stations = EXCLUDED.metro_stations
|
metro_stations = EXCLUDED.metro_stations,
|
||||||
|
listing_date = COALESCE(EXCLUDED.listing_date, listings.listing_date),
|
||||||
|
area_m2 = COALESCE(EXCLUDED.area_m2, listings.area_m2)
|
||||||
RETURNING id, (xmax = 0) AS inserted
|
RETURNING id, (xmax = 0) AS inserted
|
||||||
"""
|
"""
|
||||||
),
|
),
|
||||||
|
|
|
||||||
|
|
@ -524,3 +524,60 @@ def test_dedup_hash_distinguishes_source_even_same_id():
|
||||||
lot_avito = _base_lot(source="avito", source_id="500")
|
lot_avito = _base_lot(source="avito", source_id="500")
|
||||||
lot_cian = _base_lot(source="cian", source_id="500")
|
lot_cian = _base_lot(source="cian", source_id="500")
|
||||||
assert lot_avito.compute_dedup_hash() != lot_cian.compute_dedup_hash()
|
assert lot_avito.compute_dedup_hash() != lot_cian.compute_dedup_hash()
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tests 15-17: #821 COALESCE backfill listing_date / area_m2 ──────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_sql_contains_coalesce_listing_date_area_m2():
|
||||||
|
"""#821: ON CONFLICT SET block содержит COALESCE для listing_date и area_m2."""
|
||||||
|
db = _mock_db(inserted=False) # conflict path
|
||||||
|
lot = _cian_lot()
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
sql_text = _get_listings_insert_sql(db)
|
||||||
|
assert "COALESCE(EXCLUDED.listing_date, listings.listing_date)" in sql_text
|
||||||
|
assert "COALESCE(EXCLUDED.area_m2, listings.area_m2)" in sql_text
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_params_listing_date_area_m2_passed_through():
|
||||||
|
"""#821: listing_date и area_m2 попадают в params INSERT (VALUES side)."""
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
db = _mock_db(inserted=True)
|
||||||
|
lot = _base_lot(
|
||||||
|
source_id="ld_test",
|
||||||
|
area_m2=47.5,
|
||||||
|
listing_date=datetime.date(2024, 3, 15),
|
||||||
|
)
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
params = _get_execute_params(db)
|
||||||
|
assert params["area_m2"] == 47.5
|
||||||
|
assert params["listing_date"] == datetime.date(2024, 3, 15)
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_null_area_and_date_pass_none_in_params():
|
||||||
|
"""#821: re-scrape без area_m2/listing_date → params содержат None.
|
||||||
|
|
||||||
|
COALESCE в SQL сохранит существующее значение в БД — здесь проверяем
|
||||||
|
только что params не ломают вызов (None передаётся без ошибки).
|
||||||
|
"""
|
||||||
|
db = _mock_db(inserted=False) # conflict/update path
|
||||||
|
lot = _base_lot(
|
||||||
|
source_id="null_test",
|
||||||
|
area_m2=None,
|
||||||
|
listing_date=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
inserted, updated = save_listings(db, [lot])
|
||||||
|
|
||||||
|
# Ожидаем updated (xmax != 0)
|
||||||
|
assert inserted == 0
|
||||||
|
assert updated == 1
|
||||||
|
|
||||||
|
params = _get_execute_params(db)
|
||||||
|
assert params["area_m2"] is None
|
||||||
|
assert params["listing_date"] is None
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue