fix(scrapers): write resolved house_id to listings.house_id_fk in realtime link hook
This commit is contained in:
parent
97fce43484
commit
e63b21eccb
2 changed files with 82 additions and 0 deletions
|
|
@ -521,6 +521,24 @@ def _link_listing_to_house(db: Session, listing_id: int, lot: ScrapedLot) -> Non
|
||||||
source_url=lot.house_url or lot.source_url,
|
source_url=lot.house_url or lot.source_url,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Mirror the resolved house into listings.house_id_fk so direct
|
||||||
|
# `JOIN houses ON house_id_fk = id` keeps working on freshly scraped /
|
||||||
|
# re-scraped active rows. Without this the FK is populated only by the
|
||||||
|
# offline backfill (063_*.sql / backfill_listing_sources.py), leaving
|
||||||
|
# new active listings unlinked (avito 0/444, cian ~7%). Mirrors the
|
||||||
|
# backfill script's column/param style. IS DISTINCT FROM avoids a no-op
|
||||||
|
# write when the linkage is already correct.
|
||||||
|
if house_id is not None:
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE listings "
|
||||||
|
" SET house_id_fk = CAST(:hid AS bigint) "
|
||||||
|
" WHERE id = CAST(:lid AS bigint) "
|
||||||
|
" AND house_id_fk IS DISTINCT FROM CAST(:hid AS bigint)"
|
||||||
|
),
|
||||||
|
{"hid": house_id, "lid": listing_id},
|
||||||
|
)
|
||||||
|
|
||||||
upsert_listing_source(
|
upsert_listing_source(
|
||||||
db,
|
db,
|
||||||
listing_id=listing_id,
|
listing_id=listing_id,
|
||||||
|
|
|
||||||
|
|
@ -596,6 +596,70 @@ def test_save_listings_null_area_and_date_pass_none_in_params():
|
||||||
assert params["listing_date"] is None
|
assert params["listing_date"] is None
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tests: house_id_fk realtime link write (avito/cian unlinked fix) ─────────
|
||||||
|
|
||||||
|
|
||||||
|
def _find_house_fk_update(db: MagicMock):
|
||||||
|
"""Return (sql, params) of the UPDATE listings SET house_id_fk call, or None.
|
||||||
|
|
||||||
|
match_or_create_house / upsert_listing_source are patched (no db.execute), so
|
||||||
|
the only db.execute calls are the listings INSERT (call 0) and — when a house
|
||||||
|
is resolved — the house_id_fk UPDATE added by the realtime link hook.
|
||||||
|
"""
|
||||||
|
for call in db.execute.call_args_list:
|
||||||
|
sql = str(call.args[0])
|
||||||
|
if "house_id_fk" in sql and "UPDATE listings" in sql:
|
||||||
|
return sql, call.args[1]
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_writes_house_id_fk_when_house_resolved(_patch_matching):
|
||||||
|
"""House resolved (address/coords present) → UPDATE listings.house_id_fk runs
|
||||||
|
with the resolved house_id and the saved listing id."""
|
||||||
|
_patch_matching["house"].return_value = (909, 1.0, "geo")
|
||||||
|
|
||||||
|
db = _mock_db(inserted=True, listing_id=4242)
|
||||||
|
lot = _cian_lot(source_id="c-fk-1") # has address + lat/lon → house resolved
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
found = _find_house_fk_update(db)
|
||||||
|
assert found is not None, "UPDATE listings SET house_id_fk was not issued"
|
||||||
|
_sql, params = found
|
||||||
|
assert params["hid"] == 909
|
||||||
|
assert params["lid"] == 4242
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_no_house_fk_update_when_house_not_resolved(_patch_matching):
|
||||||
|
"""No address and no coords → match_or_create_house skipped → no FK UPDATE."""
|
||||||
|
db = _mock_db(inserted=True, listing_id=4243)
|
||||||
|
# base lot has no address / lat / lon → house resolution skipped
|
||||||
|
lot = _base_lot(source="avito", source_id="a-fk-no", address=None, lat=None, lon=None)
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
assert _patch_matching["house"].call_count == 0
|
||||||
|
assert _find_house_fk_update(db) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_save_listings_house_fk_update_guards_no_op_write(_patch_matching):
|
||||||
|
"""The UPDATE is idempotency-guarded (IS DISTINCT FROM) and casts to bigint
|
||||||
|
(psycopg v3 CAST, never ::bigint)."""
|
||||||
|
_patch_matching["house"].return_value = (55, 1.0, "cadastr")
|
||||||
|
|
||||||
|
db = _mock_db(inserted=True, listing_id=1)
|
||||||
|
lot = _cian_lot(source_id="c-fk-guard")
|
||||||
|
|
||||||
|
save_listings(db, [lot])
|
||||||
|
|
||||||
|
found = _find_house_fk_update(db)
|
||||||
|
assert found is not None
|
||||||
|
sql, _params = found
|
||||||
|
assert "IS DISTINCT FROM" in sql
|
||||||
|
assert "CAST(:hid AS bigint)" in sql
|
||||||
|
assert "::bigint" not in sql
|
||||||
|
|
||||||
|
|
||||||
# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ──
|
# ── Tests 18-20: Yandex rich fields (predictedPrice, trend, description, agency) ──
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue