237 lines
8.9 KiB
Python
237 lines
8.9 KiB
Python
"""Listing cross-source matching — 3-tier algorithm.
|
|
|
|
Tier 0 (confidence 1.0): cadastral_number exact match on listings table.
|
|
Tier 1 (confidence 1.0): ext_source + ext_id already in listing_sources.
|
|
Tier 2 (confidence 0.85): description_minhash exact match within same house.
|
|
Tier 3 (confidence 0.75): composite match — house + floor + area ±2% + rooms.
|
|
New (sentinel 0): caller is responsible for INSERT + call _upsert_listing_source.
|
|
|
|
Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 4
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def match_or_create_listing(
|
|
db: Session,
|
|
ext_source: str,
|
|
ext_id: str,
|
|
house_id: int,
|
|
floor: int | None = None,
|
|
rooms_count: int | None = None,
|
|
area_m2: float | None = None,
|
|
price_rub: float | None = None,
|
|
*,
|
|
cadastral_number: str | None = None,
|
|
description_minhash: str | None = None,
|
|
source_url: str | None = None,
|
|
source_data: dict | None = None,
|
|
) -> tuple[int, float, str]:
|
|
"""Match existing listing or signal that caller should create a new one.
|
|
|
|
Returns:
|
|
(listing_id, confidence in [0.0, 1.0], method name)
|
|
|
|
Method values:
|
|
'cadastr_exact' — matched by cadastral number (confidence 1.0)
|
|
'source_exact' — already in listing_sources for this source+ext_id (confidence 1.0)
|
|
'minhash' — matched by description minhash within house (confidence 0.85)
|
|
'composite' — matched by house+floor+area±2%+rooms (confidence 0.75)
|
|
'new' — no match; caller must INSERT listing then call upsert_listing_source()
|
|
|
|
When method == 'new', listing_id is 0 (sentinel). Caller flow:
|
|
listing_id, conf, method = match_or_create_listing(db, ...)
|
|
if method == 'new':
|
|
listing_id = <INSERT canonical listing row>
|
|
upsert_listing_source(db, listing_id=listing_id, ...)
|
|
"""
|
|
# Tier 0: cadastral number exact match
|
|
if cadastral_number:
|
|
row = db.execute(
|
|
text(
|
|
'SELECT id FROM listings WHERE cadastral_number = :cad ORDER BY id ASC LIMIT 1'
|
|
),
|
|
{'cad': cadastral_number},
|
|
).mappings().first()
|
|
if row:
|
|
listing_id = int(row['id'])
|
|
_upsert_listing_source(
|
|
db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id,
|
|
method='cadastr_exact', confidence=1.0,
|
|
price_rub=price_rub, area_m2=area_m2, floor=floor,
|
|
rooms_count=rooms_count, source_url=source_url, source_data=source_data,
|
|
)
|
|
logger.info('listing match cadastr_exact id=%s cad=%s', listing_id, cadastral_number)
|
|
return (listing_id, 1.0, 'cadastr_exact')
|
|
|
|
# Tier 1: source+ext_id already registered
|
|
row = db.execute(
|
|
text(
|
|
'SELECT listing_id FROM listing_sources '
|
|
'WHERE ext_source = :s AND ext_id = :e LIMIT 1'
|
|
),
|
|
{'s': ext_source, 'e': str(ext_id)},
|
|
).mappings().first()
|
|
if row:
|
|
listing_id = int(row['listing_id'])
|
|
logger.info('listing match source_exact id=%s src=%s ext_id=%s', listing_id, ext_source,
|
|
ext_id)
|
|
return (listing_id, 1.0, 'source_exact')
|
|
|
|
# Tier 2: description minhash match within same house
|
|
# Stage 8 v1: exact match on pre-computed minhash string.
|
|
# Future (Stage 8.x): replace with datasketch MinHash LSH for approximate Jaccard.
|
|
if description_minhash:
|
|
row = db.execute(
|
|
text("""
|
|
SELECT id FROM listings
|
|
WHERE house_id = CAST(:hid AS bigint)
|
|
AND description_minhash = :hash
|
|
LIMIT 1
|
|
"""),
|
|
{'hid': house_id, 'hash': description_minhash},
|
|
).mappings().first()
|
|
if row:
|
|
listing_id = int(row['id'])
|
|
_upsert_listing_source(
|
|
db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id,
|
|
method='minhash', confidence=0.85,
|
|
price_rub=price_rub, area_m2=area_m2, floor=floor,
|
|
rooms_count=rooms_count, source_url=source_url, source_data=source_data,
|
|
)
|
|
logger.info('listing match minhash id=%s hash=%s', listing_id, description_minhash)
|
|
return (listing_id, 0.85, 'minhash')
|
|
|
|
# Tier 3: composite match — house + floor + area ±2% + rooms
|
|
if house_id and floor is not None and area_m2 is not None and rooms_count is not None:
|
|
row = db.execute(
|
|
text("""
|
|
SELECT id,
|
|
ABS(area_m2 - CAST(:area AS numeric)) AS diff
|
|
FROM listings
|
|
WHERE house_id = CAST(:hid AS bigint)
|
|
AND floor = :fl
|
|
AND rooms = :rc
|
|
AND area_m2 BETWEEN CAST(:area AS numeric) * 0.98
|
|
AND CAST(:area AS numeric) * 1.02
|
|
ORDER BY diff ASC
|
|
LIMIT 1
|
|
"""),
|
|
{'hid': house_id, 'fl': floor, 'rc': rooms_count, 'area': area_m2},
|
|
).mappings().first()
|
|
if row:
|
|
listing_id = int(row['id'])
|
|
_upsert_listing_source(
|
|
db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id,
|
|
method='composite', confidence=0.75,
|
|
price_rub=price_rub, area_m2=area_m2, floor=floor,
|
|
rooms_count=rooms_count, source_url=source_url, source_data=source_data,
|
|
)
|
|
logger.info(
|
|
'listing match composite id=%s house=%s floor=%s area=%.1f rooms=%s',
|
|
listing_id, house_id, floor, area_m2, rooms_count,
|
|
)
|
|
return (listing_id, 0.75, 'composite')
|
|
|
|
# No match — signal caller to create new listing
|
|
return (0, 1.0, 'new')
|
|
|
|
|
|
def upsert_listing_source(
|
|
db: Session,
|
|
*,
|
|
listing_id: int,
|
|
ext_source: str,
|
|
ext_id: str,
|
|
method: str,
|
|
confidence: float,
|
|
price_rub: float | None = None,
|
|
area_m2: float | None = None,
|
|
floor: int | None = None,
|
|
rooms_count: int | None = None,
|
|
source_url: str | None = None,
|
|
source_data: dict | None = None,
|
|
) -> None:
|
|
"""Public helper — register or refresh a listing_sources row.
|
|
|
|
Called by external code after creating a new canonical listing.
|
|
"""
|
|
_upsert_listing_source(
|
|
db,
|
|
listing_id=listing_id,
|
|
ext_source=ext_source,
|
|
ext_id=ext_id,
|
|
method=method,
|
|
confidence=confidence,
|
|
price_rub=price_rub,
|
|
area_m2=area_m2,
|
|
floor=floor,
|
|
rooms_count=rooms_count,
|
|
source_url=source_url,
|
|
source_data=source_data,
|
|
)
|
|
|
|
|
|
def _upsert_listing_source(
|
|
db: Session,
|
|
*,
|
|
listing_id: int,
|
|
ext_source: str,
|
|
ext_id: str,
|
|
method: str,
|
|
confidence: float,
|
|
price_rub: float | None,
|
|
area_m2: float | None,
|
|
floor: int | None,
|
|
rooms_count: int | None,
|
|
source_url: str | None,
|
|
source_data: dict | None,
|
|
) -> None:
|
|
"""Insert or refresh listing_sources row for this source+ext_id."""
|
|
raw = json.dumps(source_data) if source_data is not None else None
|
|
db.execute(
|
|
text("""
|
|
INSERT INTO listing_sources (
|
|
listing_id, ext_source, ext_id,
|
|
confidence, matched_method, matched_at, last_seen_at,
|
|
source_url, last_scraped_at,
|
|
price_rub, area_m2, floor, rooms_count, raw_payload
|
|
) VALUES (
|
|
CAST(:lid AS bigint), :s, :e,
|
|
CAST(:c AS real), :m, NOW(), NOW(),
|
|
:url, NOW(),
|
|
CAST(:p AS bigint), CAST(:a AS numeric), :fl, :rc,
|
|
CAST(:raw AS jsonb)
|
|
)
|
|
ON CONFLICT (ext_source, ext_id) DO UPDATE SET
|
|
confidence = GREATEST(EXCLUDED.confidence, listing_sources.confidence),
|
|
last_seen_at = NOW(),
|
|
last_scraped_at = NOW(),
|
|
price_rub = COALESCE(EXCLUDED.price_rub, listing_sources.price_rub),
|
|
area_m2 = COALESCE(EXCLUDED.area_m2, listing_sources.area_m2),
|
|
floor = COALESCE(EXCLUDED.floor, listing_sources.floor),
|
|
rooms_count = COALESCE(EXCLUDED.rooms_count, listing_sources.rooms_count),
|
|
raw_payload = COALESCE(EXCLUDED.raw_payload, listing_sources.raw_payload)
|
|
"""),
|
|
{
|
|
'lid': listing_id,
|
|
's': ext_source,
|
|
'e': str(ext_id),
|
|
'c': confidence,
|
|
'm': method,
|
|
'url': source_url,
|
|
# Migration 029 declares listing_sources.price_rub bigint — whole rubles only,
|
|
# kopecks truncated. Document expectation in caller.
|
|
'p': int(price_rub) if price_rub is not None else None,
|
|
'a': area_m2,
|
|
'fl': floor,
|
|
'rc': rooms_count,
|
|
'raw': raw,
|
|
},
|
|
)
|