fix(scrapers): write resolved house_id to listings.house_id_fk in realtime link hook #1727

Merged
lekss361 merged 1 commit from fix/house-id-fk-realtime-link into main 2026-06-17 20:16:28 +00:00
2 changed files with 82 additions and 0 deletions

View file

@ -521,6 +521,24 @@ def _link_listing_to_house(db: Session, listing_id: int, lot: ScrapedLot) -> Non
source_url=lot.house_url or lot.source_url,
)
# Mirror the resolved house into listings.house_id_fk so direct
# `JOIN houses ON house_id_fk = id` keeps working on freshly scraped /
# re-scraped active rows. Without this the FK is populated only by the
# offline backfill (063_*.sql / backfill_listing_sources.py), leaving
# new active listings unlinked (avito 0/444, cian ~7%). Mirrors the
# backfill script's column/param style. IS DISTINCT FROM avoids a no-op
# write when the linkage is already correct.
if house_id is not None:
db.execute(
text(
"UPDATE listings "
" SET house_id_fk = CAST(:hid AS bigint) "
" WHERE id = CAST(:lid AS bigint) "
" AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)"
),
{"hid": house_id, "lid": listing_id},
)
upsert_listing_source(
db,
listing_id=listing_id,

View file

@ -596,6 +596,70 @@ def test_save_listings_null_area_and_date_pass_none_in_params():
assert params["listing_date"] is None
# ── Tests: house_id_fk realtime link write (avito/cian unlinked fix) ─────────
def _find_house_fk_update(db: MagicMock):
"""Return (sql, params) of the UPDATE listings SET house_id_fk call, or None.
match_or_create_house / upsert_listing_source are patched (no db.execute), so
the only db.execute calls are the listings INSERT (call 0) and when a house
is resolved the house_id_fk UPDATE added by the realtime link hook.
"""
for call in db.execute.call_args_list:
sql = str(call.args[0])
if "house_id_fk" in sql and "UPDATE listings" in sql:
return sql, call.args[1]
return None
def test_save_listings_writes_house_id_fk_when_house_resolved(_patch_matching):
"""House resolved (address/coords present) → UPDATE listings.house_id_fk runs
with the resolved house_id and the saved listing id."""
_patch_matching["house"].return_value = (909, 1.0, "geo")
db = _mock_db(inserted=True, listing_id=4242)
lot = _cian_lot(source_id="c-fk-1") # has address + lat/lon → house resolved
save_listings(db, [lot])
found = _find_house_fk_update(db)
assert found is not None, "UPDATE listings SET house_id_fk was not issued"
_sql, params = found
assert params["hid"] == 909
assert params["lid"] == 4242
def test_save_listings_no_house_fk_update_when_house_not_resolved(_patch_matching):
"""No address and no coords → match_or_create_house skipped → no FK UPDATE."""
db = _mock_db(inserted=True, listing_id=4243)
# base lot has no address / lat / lon → house resolution skipped
lot = _base_lot(source="avito", source_id="a-fk-no", address=None, lat=None, lon=None)
save_listings(db, [lot])
assert _patch_matching["house"].call_count == 0
assert _find_house_fk_update(db) is None
def test_save_listings_house_fk_update_guards_no_op_write(_patch_matching):
"""The UPDATE is idempotency-guarded (IS DISTINCT FROM) and casts to bigint
(psycopg v3 CAST, never ::bigint)."""
_patch_matching["house"].return_value = (55, 1.0, "cadastr")
db = _mock_db(inserted=True, listing_id=1)
lot = _cian_lot(source_id="c-fk-guard")
save_listings(db, [lot])
found = _find_house_fk_update(db)
assert found is not None
sql, _params = found
assert "IS DISTINCT FROM" in sql
assert "CAST(:hid AS bigint)" in sql
assert "::bigint" not in sql
# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ──