fix(scrapers): write resolved house_id to listings.house_id_fk in realtime link hook #1727
2 changed files with 82 additions and 0 deletions
|
|
@ -521,6 +521,24 @@ def _link_listing_to_house(db: Session, listing_id: int, lot: ScrapedLot) -> Non
|
|||
source_url=lot.house_url or lot.source_url,
|
||||
)
|
||||
|
||||
# Mirror the resolved house into listings.house_id_fk so direct
|
||||
# `JOIN houses ON house_id_fk = id` keeps working on freshly scraped /
|
||||
# re-scraped active rows. Without this the FK is populated only by the
|
||||
# offline backfill (063_*.sql / backfill_listing_sources.py), leaving
|
||||
# new active listings unlinked (avito 0/444, cian ~7%). Mirrors the
|
||||
# backfill script's column/param style. IS DISTINCT FROM avoids a no-op
|
||||
# write when the linkage is already correct.
|
||||
if house_id is not None:
|
||||
db.execute(
|
||||
text(
|
||||
"UPDATE listings "
|
||||
" SET house_id_fk = CAST(:hid AS bigint) "
|
||||
" WHERE id = CAST(:lid AS bigint) "
|
||||
" AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)"
|
||||
),
|
||||
{"hid": house_id, "lid": listing_id},
|
||||
)
|
||||
|
||||
upsert_listing_source(
|
||||
db,
|
||||
listing_id=listing_id,
|
||||
|
|
|
|||
|
|
@ -596,6 +596,70 @@ def test_save_listings_null_area_and_date_pass_none_in_params():
|
|||
assert params["listing_date"] is None
|
||||
|
||||
|
||||
# ── Tests: house_id_fk realtime link write (avito/cian unlinked fix) ─────────
|
||||
|
||||
|
||||
def _find_house_fk_update(db: MagicMock):
|
||||
"""Return (sql, params) of the UPDATE listings SET house_id_fk call, or None.
|
||||
|
||||
match_or_create_house / upsert_listing_source are patched (no db.execute), so
|
||||
the only db.execute calls are the listings INSERT (call 0) and — when a house
|
||||
is resolved — the house_id_fk UPDATE added by the realtime link hook.
|
||||
"""
|
||||
for call in db.execute.call_args_list:
|
||||
sql = str(call.args[0])
|
||||
if "house_id_fk" in sql and "UPDATE listings" in sql:
|
||||
return sql, call.args[1]
|
||||
return None
|
||||
|
||||
|
||||
def test_save_listings_writes_house_id_fk_when_house_resolved(_patch_matching):
|
||||
"""House resolved (address/coords present) → UPDATE listings.house_id_fk runs
|
||||
with the resolved house_id and the saved listing id."""
|
||||
_patch_matching["house"].return_value = (909, 1.0, "geo")
|
||||
|
||||
db = _mock_db(inserted=True, listing_id=4242)
|
||||
lot = _cian_lot(source_id="c-fk-1") # has address + lat/lon → house resolved
|
||||
|
||||
save_listings(db, [lot])
|
||||
|
||||
found = _find_house_fk_update(db)
|
||||
assert found is not None, "UPDATE listings SET house_id_fk was not issued"
|
||||
_sql, params = found
|
||||
assert params["hid"] == 909
|
||||
assert params["lid"] == 4242
|
||||
|
||||
|
||||
def test_save_listings_no_house_fk_update_when_house_not_resolved(_patch_matching):
|
||||
"""No address and no coords → match_or_create_house skipped → no FK UPDATE."""
|
||||
db = _mock_db(inserted=True, listing_id=4243)
|
||||
# base lot has no address / lat / lon → house resolution skipped
|
||||
lot = _base_lot(source="avito", source_id="a-fk-no", address=None, lat=None, lon=None)
|
||||
|
||||
save_listings(db, [lot])
|
||||
|
||||
assert _patch_matching["house"].call_count == 0
|
||||
assert _find_house_fk_update(db) is None
|
||||
|
||||
|
||||
def test_save_listings_house_fk_update_guards_no_op_write(_patch_matching):
|
||||
"""The UPDATE is idempotency-guarded (IS DISTINCT FROM) and casts to bigint
|
||||
(psycopg v3 CAST, never ::bigint)."""
|
||||
_patch_matching["house"].return_value = (55, 1.0, "cadastr")
|
||||
|
||||
db = _mock_db(inserted=True, listing_id=1)
|
||||
lot = _cian_lot(source_id="c-fk-guard")
|
||||
|
||||
save_listings(db, [lot])
|
||||
|
||||
found = _find_house_fk_update(db)
|
||||
assert found is not None
|
||||
sql, _params = found
|
||||
assert "IS DISTINCT FROM" in sql
|
||||
assert "CAST(:hid AS bigint)" in sql
|
||||
assert "::bigint" not in sql
|
||||
|
||||
|
||||
# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ──
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue