"""Listing cross-source matching — 3-tier algorithm. Tier 0 (confidence 1.0): cadastral_number exact match on listings table. Tier 1 (confidence 1.0): ext_source + ext_id already in listing_sources. Tier 2 (confidence 0.85): description_minhash exact match within same house. Tier 3 (confidence 0.75): composite match — house + floor + area ±2% + rooms. New (sentinel 0): caller is responsible for INSERT + call _upsert_listing_source. Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 4 """ import json import logging from sqlalchemy import text from sqlalchemy.orm import Session logger = logging.getLogger(__name__) def match_or_create_listing( db: Session, ext_source: str, ext_id: str, house_id: int, floor: int | None = None, rooms_count: int | None = None, area_m2: float | None = None, price_rub: float | None = None, *, cadastral_number: str | None = None, description_minhash: str | None = None, source_url: str | None = None, source_data: dict | None = None, ) -> tuple[int, float, str]: """Match existing listing or signal that caller should create a new one. Returns: (listing_id, confidence in [0.0, 1.0], method name) Method values: 'cadastr_exact' — matched by cadastral number (confidence 1.0) 'source_exact' — already in listing_sources for this source+ext_id (confidence 1.0) 'minhash' — matched by description minhash within house (confidence 0.85) 'composite' — matched by house+floor+area±2%+rooms (confidence 0.75) 'new' — no match; caller must INSERT listing then call upsert_listing_source() When method == 'new', listing_id is 0 (sentinel). Caller flow: listing_id, conf, method = match_or_create_listing(db, ...) if method == 'new': listing_id = upsert_listing_source(db, listing_id=listing_id, ...) """ # Tier 0: cadastral number exact match if cadastral_number: row = db.execute( text( 'SELECT id FROM listings WHERE cadastral_number = :cad ORDER BY id ASC LIMIT 1' ), {'cad': cadastral_number}, ).mappings().first() if row: listing_id = int(row['id']) _upsert_listing_source( db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id, method='cadastr_exact', confidence=1.0, price_rub=price_rub, area_m2=area_m2, floor=floor, rooms_count=rooms_count, source_url=source_url, source_data=source_data, ) logger.info('listing match cadastr_exact id=%s cad=%s', listing_id, cadastral_number) return (listing_id, 1.0, 'cadastr_exact') # Tier 1: source+ext_id already registered row = db.execute( text( 'SELECT listing_id FROM listing_sources ' 'WHERE ext_source = :s AND ext_id = :e LIMIT 1' ), {'s': ext_source, 'e': str(ext_id)}, ).mappings().first() if row: listing_id = int(row['listing_id']) logger.info('listing match source_exact id=%s src=%s ext_id=%s', listing_id, ext_source, ext_id) return (listing_id, 1.0, 'source_exact') # Tier 2: description minhash match within same house # Stage 8 v1: exact match on pre-computed minhash string. # Future (Stage 8.x): replace with datasketch MinHash LSH for approximate Jaccard. if description_minhash: row = db.execute( text(""" SELECT id FROM listings WHERE house_id = CAST(:hid AS bigint) AND description_minhash = :hash LIMIT 1 """), {'hid': house_id, 'hash': description_minhash}, ).mappings().first() if row: listing_id = int(row['id']) _upsert_listing_source( db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id, method='minhash', confidence=0.85, price_rub=price_rub, area_m2=area_m2, floor=floor, rooms_count=rooms_count, source_url=source_url, source_data=source_data, ) logger.info('listing match minhash id=%s hash=%s', listing_id, description_minhash) return (listing_id, 0.85, 'minhash') # Tier 3: composite match — house + floor + area ±2% + rooms if house_id and floor is not None and area_m2 is not None and rooms_count is not None: row = db.execute( text(""" SELECT id, ABS(area_m2 - CAST(:area AS numeric)) AS diff FROM listings WHERE house_id = CAST(:hid AS bigint) AND floor = :fl AND rooms = :rc AND area_m2 BETWEEN CAST(:area AS numeric) * 0.98 AND CAST(:area AS numeric) * 1.02 ORDER BY diff ASC LIMIT 1 """), {'hid': house_id, 'fl': floor, 'rc': rooms_count, 'area': area_m2}, ).mappings().first() if row: listing_id = int(row['id']) _upsert_listing_source( db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id, method='composite', confidence=0.75, price_rub=price_rub, area_m2=area_m2, floor=floor, rooms_count=rooms_count, source_url=source_url, source_data=source_data, ) logger.info( 'listing match composite id=%s house=%s floor=%s area=%.1f rooms=%s', listing_id, house_id, floor, area_m2, rooms_count, ) return (listing_id, 0.75, 'composite') # No match — signal caller to create new listing return (0, 1.0, 'new') def upsert_listing_source( db: Session, *, listing_id: int, ext_source: str, ext_id: str, method: str, confidence: float, price_rub: float | None = None, area_m2: float | None = None, floor: int | None = None, rooms_count: int | None = None, source_url: str | None = None, source_data: dict | None = None, ) -> None: """Public helper — register or refresh a listing_sources row. Called by external code after creating a new canonical listing. """ _upsert_listing_source( db, listing_id=listing_id, ext_source=ext_source, ext_id=ext_id, method=method, confidence=confidence, price_rub=price_rub, area_m2=area_m2, floor=floor, rooms_count=rooms_count, source_url=source_url, source_data=source_data, ) def _upsert_listing_source( db: Session, *, listing_id: int, ext_source: str, ext_id: str, method: str, confidence: float, price_rub: float | None, area_m2: float | None, floor: int | None, rooms_count: int | None, source_url: str | None, source_data: dict | None, ) -> None: """Insert or refresh listing_sources row for this source+ext_id.""" raw = json.dumps(source_data) if source_data is not None else None db.execute( text(""" INSERT INTO listing_sources ( listing_id, ext_source, ext_id, confidence, matched_method, matched_at, last_seen_at, source_url, last_scraped_at, price_rub, area_m2, floor, rooms_count, raw_payload ) VALUES ( CAST(:lid AS bigint), :s, :e, CAST(:c AS real), :m, NOW(), NOW(), :url, NOW(), CAST(:p AS bigint), CAST(:a AS numeric), :fl, :rc, CAST(:raw AS jsonb) ) ON CONFLICT (ext_source, ext_id) DO UPDATE SET confidence = GREATEST(EXCLUDED.confidence, listing_sources.confidence), last_seen_at = NOW(), last_scraped_at = NOW(), price_rub = COALESCE(EXCLUDED.price_rub, listing_sources.price_rub), area_m2 = COALESCE(EXCLUDED.area_m2, listing_sources.area_m2), floor = COALESCE(EXCLUDED.floor, listing_sources.floor), rooms_count = COALESCE(EXCLUDED.rooms_count, listing_sources.rooms_count), raw_payload = COALESCE(EXCLUDED.raw_payload, listing_sources.raw_payload) """), { 'lid': listing_id, 's': ext_source, 'e': str(ext_id), 'c': confidence, 'm': method, 'url': source_url, # Migration 029 declares listing_sources.price_rub bigint — whole rubles only, # kopecks truncated. Document expectation in caller. 'p': int(price_rub) if price_rub is not None else None, 'a': area_m2, 'fl': floor, 'rc': rooms_count, 'raw': raw, }, )