gendesign/tradein-mvp/backend/app/services/matching/houses.py

"""House cross-source matching — tiered algorithm.

Tier 0   (confidence 1.0):  cadastral_number exact match on houses table.
Tier 0.5 (confidence 0.95): house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive.
Tier 1   (confidence 1.0):  ext_source + ext_id already in house_sources.
Tier 2   (confidence 0.9):  address_fingerprint match in house_address_aliases.
Tier 3   (confidence 0.7):  geo-proximity within 30 m (PostGIS ST_DWithin).
New      (confidence 1.0):  INSERT new canonical house.

Algorithm reference: decisions/Cross_Source_Matching_Strategy.md sec 3
"""

import logging

from sqlalchemy import text
from sqlalchemy.orm import Session

from app.services.matching.normalize import (
    EKB_CITY_TOKEN,
    address_fingerprint,
    has_city_token,
    has_house_number,
    house_number_token,
    normalize_address,
    resolve_city_token,
)

logger = logging.getLogger(__name__)

# Tier-2b oblast guard radius (metres). A bare street+number normalized_address carries
# NO city token yet is globally unique in house_address_aliases, so 'улица ленина 100'
# physically exists in ЕКБ, Н.Тагил, Каменск… When the incoming card HAS coords we accept
# the alias only if its house geom is within this radius. Cities in обл.66 are ≥ ~9-15 km
# apart, so 3 km cleanly separates them while absorbing the intra-city geocoder drift that
# Tier-2b exists to bridge (same building, different provider coords).
_TIER2B_GUARD_M = 3000


def match_or_create_house(
    db: Session,
    ext_source: str,
    ext_id: str,
    address: str | None = None,
    lat: float | None = None,
    lon: float | None = None,
    *,
    year_built: int | None = None,
    building_cadastral_number: str | None = None,
    cadastral_number: str | None = None,
    house_fias_id: str | None = None,
    source_url: str | None = None,
) -> tuple[int | None, float, str]:
    """Match existing house or create new canonical record.

    Concurrency-safe: serializes concurrent calls for the same address fingerprint
    via pg_advisory_xact_lock(42, hashtext(fp)). The lock is transaction-scoped,
    released on the caller's COMMIT/ROLLBACK. Without this, two scrapers calling
    for an unknown address could both miss Tier 0-3 and each INSERT a duplicate
    house row. Closes finding #1 from 2026-05-24 audit.

    Args:
        house_fias_id: ГАР OBJECTGUID (UUID) of the building, when known upstream
            (e.g. DaData /clean/address). Enables Tier 0.5 fias_exact — additive and
            optional, existing callers are unaffected.

    Returns:
        (house_id, confidence ∈ [0.0, 1.0], method ∈ {
            'cadastr_exact', 'fias_exact', 'source_exact', 'fingerprint',
            'geo_proximity', 'new', 'no_house_number'
        })
        house_id is None only for the 'no_house_number' terminal case below.

    Method values:
        'cadastr_exact'   — matched by cadastral number (confidence 1.0)
        'fias_exact'      — matched by house_fias_id (ГАР OBJECTGUID) (confidence 0.95)
        'source_exact'    — already in house_sources for this source+ext_id (confidence 1.0)
        'fingerprint'     — matched by address fingerprint (confidence 0.9)
        'geo_proximity'   — matched by geo within 30 m (confidence 0.7)
        'new'             — new house created (confidence 1.0)
        'no_house_number' — REFUSED: numberless address with no cadastral number →
            house_id=None, confidence 0.0. No house/house_source/alias is written.
            P1 philosophy: a bare-street building-key (no house number) mass-mis-buckets
            active listings into one "house". The alias/Tier-2b guards already refuse to
            MATCH such a key (house 131237 collapsed 8 numbers); this closes the last hole
            where the New-house INSERT would still MINT one. On prod the missing gate spawned
            3 384 numberless mega-buckets ('Москва' 664 listings, 'Екатеринбург (Avito)' 587,
            'р-н Чкаловский, мкр. Вторчермет' 480). A cadastral number is a precise building
            identity, so cad-carrying rows stay exempt (Tier 0 owns them).
    """
    cad = building_cadastral_number or cadastral_number

    # Compute fingerprint early so we can acquire the advisory lock before any tier reads.
    fp = address_fingerprint(address, lat, lon)

    # Serialize concurrent calls for the same address fingerprint to prevent
    # racing INSERTs into houses (finding #1 from 2026-05-24 audit).
    # pg_advisory_xact_lock takes (int4, int4) — namespace 42 = "tradein house matching".
    # Lock is released automatically on caller's COMMIT/ROLLBACK.
    db.execute(
        text("SELECT pg_advisory_xact_lock(42, hashtext(:fp))"),
        {"fp": fp},
    )

    # Normalize address once and detect whether it carries a house number.
    # `has_num` gates the bare-street guards (P1): a numberless normalized address
    # (e.g. 'екатеринбург улица мамина сибиряка' — common in Yandex SERP) is too
    # ambiguous to use as a building key, so it must NOT match or register a
    # normalized_address alias (mass mis-bucketing — house 131237 collapsed 8 numbers).
    norm_addr = normalize_address(address)
    has_num = has_house_number(norm_addr)

    # Tier 0: cadastral number exact match
    if cad:
        row = (
            db.execute(
                text("SELECT id FROM houses WHERE cadastral_number = :cad ORDER BY id ASC LIMIT 1"),
                {"cad": cad},
            )
            .mappings()
            .first()
        )
        if row:
            house_id = int(row["id"])
            _upsert_house_source(
                db,
                house_id=house_id,
                ext_source=ext_source,
                ext_id=ext_id,
                method="cadastr_exact",
                confidence=1.0,
            )
            # Register fingerprint/normalized_address alias so a later scrape of the same
            # house from a cadastr-less source (typical SERP) hits Tier 2a/2b instead of
            # falling through to a duplicate New INSERT when geo jitter exceeds 30 m.
            _insert_alias(db, house_id=house_id, address=address, fp=fp, source=ext_source)
            logger.info("house match cadastr_exact house_id=%s cad=%s", house_id, cad)
            return (house_id, 1.0, "cadastr_exact")

    # Tier 0.5: house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive.
    # Stable ORDER BY id so concurrent/duplicate rows resolve deterministically.
    if house_fias_id:
        row = (
            db.execute(
                text(
                    "SELECT id FROM houses "
                    "WHERE lower(house_fias_id) = lower(CAST(:fias AS text)) "
                    "ORDER BY id ASC LIMIT 1"
                ),
                {"fias": house_fias_id},
            )
            .mappings()
            .first()
        )
        if row:
            house_id = int(row["id"])
            _upsert_house_source(
                db,
                house_id=house_id,
                ext_source=ext_source,
                ext_id=ext_id,
                method="fias_exact",
                confidence=0.95,
            )
            _insert_alias(db, house_id=house_id, address=address, fp=fp, source=ext_source)
            logger.info("house match fias_exact house_id=%s fias=%s", house_id, house_fias_id)
            return (house_id, 0.95, "fias_exact")

    # Tier 1: source+ext_id already registered in house_sources
    row = (
        db.execute(
            text(
                "SELECT house_id FROM house_sources "
                "WHERE ext_source = :s AND ext_id = :e LIMIT 1"
            ),
            {"s": ext_source, "e": str(ext_id)},
        )
        .mappings()
        .first()
    )
    if row:
        house_id = int(row["house_id"])
        logger.info(
            "house match source_exact house_id=%s src=%s ext_id=%s", house_id, ext_source, ext_id
        )
        return (house_id, 1.0, "source_exact")

    # Tier 2: address fingerprint lookup.
    # Two sub-tiers to handle coord drift across scrapers:
    #   2a. exact fingerprint match (address + rounded coords)
    #   2b. normalized_address-only match — same street/number, different provider coords
    # Without 2b, two scrapers for the same house with slightly different lat/lon (beyond
    # the 4-decimal rounding tolerance) would produce distinct fingerprints, miss Tier 2a,
    # and each potentially create a duplicate house row.
    #
    # Tier-2a/2b oblast city guard (#2500 follow-up, closes the Tier-2a mis-bucket that the
    # #2500 Tier-2b geo-guard could not reach): a coord-less card whose address explicitly
    # resolves a NON-ЕКБ обл.66 city must NOT match a globally-unique alias (fingerprint OR
    # normalized_address) that almost certainly belongs to ЕКБ/another city — that would
    # corrupt ЕКБ house data. When it fires we skip both alias lookups and fall through to a
    # New house (Tier 3 is coord-gated, so coord-less cards skip it too).
    #
    # This PREVENTS ЕКБ CORRUPTION. Full oblast-internal dedup (deduping two listings of the
    # SAME oblast building) still needs city-keyed aliases — a separate follow-up, out of
    # scope, only relevant once the oblast sweep is enabled.
    #
    # EKB happy-path is byte-identical: the guard fires ONLY when the address names a non-ЕКБ
    # city AND no coords disambiguate. ЕКБ cards (resolved city = екатеринбург) and the
    # dominant bare/city-less Avito coord-less cards (resolved city None) run Tier-2a/2b
    # exactly as before. NB: a BARE oblast card (no city token in the address — today's Avito
    # SERP format) carries no signal here and is deliberately left on the unchanged path; that
    # residual needs sweep-context and is out of this fix's scope.
    _resolved_city = resolve_city_token(norm_addr) if (lat is None and lon is None) else None
    _skip_oblast_alias = _resolved_city is not None and _resolved_city != EKB_CITY_TOKEN
    if _skip_oblast_alias:
        logger.info(
            "house tier2a/2b skip: coord-less non-ЕКБ city %r na=%r src=%s",
            _resolved_city,
            norm_addr,
            ext_source,
        )

    row = None
    if not _skip_oblast_alias:
        row = (
            db.execute(
                text("SELECT house_id FROM house_address_aliases WHERE fingerprint = :fp LIMIT 1"),
                {"fp": fp},
            )
            .mappings()
            .first()
        )
    if row is None and norm_addr and has_num and not _skip_oblast_alias:
        # Tier 2b: same normalized address, possibly different coords fingerprint.
        # Gated on has_num (P1): never match a bare-street normalized_address — any
        # numberless listing would otherwise collapse into whichever house first
        # registered that street.
        #
        # Oblast guard (bug #2, обл.66 per-city rollout): normalized_address has NO city
        # token, so a bare street+number is globally unique in the alias table yet exists
        # physically in several cities. Accept the match only when the incoming listing is
        # confirmably the same place:
        #   • coords present → aliased house geom within _TIER2B_GUARD_M metres;
        #   • coords absent   → norm_addr carries a city token (self-disambiguating —
        #     'екатеринбург улица ленина 5' can't collide with a Н.Тагил address);
        #   • neither          → skip. A bare common-street with no geo signal is too
        #     ambiguous — matching would mis-bucket an oblast card into a same-named ЕКБ
        #     house. Conservative: fall through to a New house rather than mis-match.
        if lat is not None and lon is not None:
            row = (
                db.execute(
                    text(
                        "SELECT a.house_id FROM house_address_aliases a "
                        "JOIN houses h ON h.id = a.house_id "
                        "WHERE a.normalized_address = :na "
                        "  AND h.geom IS NOT NULL "
                        "  AND ST_DWithin(h.geom::geography, "
                        "                 ST_MakePoint(:lon, :lat)::geography, :thr) "
                        "LIMIT 1"
                    ),
                    {"na": norm_addr, "lon": lon, "lat": lat, "thr": _TIER2B_GUARD_M},
                )
                .mappings()
                .first()
            )
        elif has_city_token(norm_addr):
            row = (
                db.execute(
                    text(
                        "SELECT house_id FROM house_address_aliases "
                        "WHERE normalized_address = :na LIMIT 1"
                    ),
                    {"na": norm_addr},
                )
                .mappings()
                .first()
            )
        else:
            logger.info(
                "house tier2b skip: bare common-street, no coords/city na=%r src=%s",
                norm_addr,
                ext_source,
            )
    if row:
        house_id = int(row["house_id"])
        _upsert_house_source(
            db,
            house_id=house_id,
            ext_source=ext_source,
            ext_id=ext_id,
            method="fingerprint",
            confidence=0.9,
        )
        # Ensure this fingerprint is also registered so future calls hit Tier 2a directly.
        _insert_alias(db, house_id=house_id, address=address, fp=fp, source=ext_source)
        logger.info("house match fingerprint house_id=%s fp=%s", house_id, fp)
        return (house_id, 0.9, "fingerprint")

    # Tier 3: geo-proximity — within 30 m (PostGIS geography cast on both sides)
    if lat is not None and lon is not None:
        row = (
            db.execute(
                text("""
                SELECT id,
                       ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS dist,
                       COALESCE(short_address, full_address, address) AS h_addr
                FROM houses
                WHERE geom IS NOT NULL
                  AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, 30)
                ORDER BY dist ASC
                LIMIT 1
            """),
                {"lat": lat, "lon": lon},
            )
            .mappings()
            .first()
        )
        if row:
            # P2: address-consistency guard. Coarse street-centroid geocodes can place
            # DIFFERENT house numbers within 30 m of one bad point (52/64/126 all matched
            # one centroid → collapsed into house 131237). Reject the geo candidate when
            # both sides carry a house number and the numbers differ; accept only when the
            # tokens agree OR at least one side has no number (geo is the only signal then).
            cand_token = house_number_token(normalize_address(row.get("h_addr")))
            lst_token = house_number_token(norm_addr)
            if cand_token is not None and lst_token is not None and cand_token != lst_token:
                logger.info(
                    "house geo reject: listing %s != house %s (dist=%.0f)",
                    lst_token,
                    cand_token,
                    float(row["dist"]),
                )
                # Fall through to the New-house INSERT below — do NOT match this house.
            else:
                house_id = int(row["id"])
                _upsert_house_source(
                    db,
                    house_id=house_id,
                    ext_source=ext_source,
                    ext_id=ext_id,
                    method="geo_proximity",
                    confidence=0.7,
                )
                # P3: do NOT register an alias for a geo match. A 0.7-confidence
                # proximity hit must not be cemented as a building-key alias — that is
                # what compounded the mis-bucketing (each loose match spawned a new alias).
                logger.info(
                    "house match geo_proximity house_id=%s dist=%.1f src=%s",
                    house_id,
                    float(row["dist"]),
                    ext_source,
                )
                return (house_id, 0.7, "geo_proximity")

    # P1 (extended to INSERT-new): never CREATE a house from a numberless address that
    # carries no cadastral number. `has_num` is False for a bare-street normalized_address
    # (e.g. 'екатеринбург улица мамина сибиряка') AND for address=None. Such a building-key
    # is too ambiguous to mint a canonical row — it becomes a mega-bucket that every later
    # numberless listing geo-collapses into. Tier 0/aliases already refuse to MATCH it; here
    # we also refuse to CREATE it. A cadastral number (`cad`) is a precise identity → exempt.
    if not has_num and not cad:
        logger.info("house create skipped: numberless address %r src=%s", address, ext_source)
        return (None, 0.0, "no_house_number")

    # New house — INSERT canonical record.
    # geom column is auto-populated by houses_set_geom_trg BEFORE INSERT trigger from lat/lon.
    # Do NOT include geom in the INSERT column list — trigger handles it.
    url = source_url or f"matching://{ext_source}/{ext_id}"
    row = (
        db.execute(
            text("""
            INSERT INTO houses (source, ext_house_id, url, address, lat, lon, year_built,
                                cadastral_number)
            VALUES (
                :src, :eid, :url,
                :addr,
                CAST(:lat AS double precision),
                CAST(:lon AS double precision),
                CAST(:yb AS integer),
                :cad
            )
            ON CONFLICT (source, ext_house_id) DO UPDATE SET
                address = COALESCE(EXCLUDED.address, houses.address)
            RETURNING id
        """),
            {
                "src": ext_source,
                "eid": str(ext_id),
                "url": url,
                "addr": address,
                "lat": lat,
                "lon": lon,
                "yb": year_built,
                "cad": cad,
            },
        )
        .mappings()
        .one()
    )
    house_id = int(row["id"])

    _upsert_house_source(
        db,
        house_id=house_id,
        ext_source=ext_source,
        ext_id=ext_id,
        method="new",
        confidence=1.0,
    )
    _insert_alias(db, house_id=house_id, address=address, fp=fp, source=ext_source)

    logger.info("house new house_id=%s addr=%r src=%s", house_id, address, ext_source)
    return (house_id, 1.0, "new")


def match_house_readonly(
    db: Session,
    *,
    address: str | None = None,
    lat: float | None = None,
    lon: float | None = None,
    cadastral_number: str | None = None,
    house_fias_id: str | None = None,
) -> tuple[int, float, str] | None:
    """Match a target address to an EXISTING canonical house WITHOUT creating one.

    Read-only counterpart of match_or_create_house() for the estimate-target flow:
    we must NOT pollute `houses` with user-queried addresses, so this never INSERTs,
    never writes house_sources / aliases, and takes no advisory lock (no write race
    to guard).

    Tiers (same lookups as match_or_create_house, minus source_exact which needs an
    ext_id):
      0.  cadastr_exact  — houses.cadastral_number = :cad         (confidence 1.0)
      0.5 fias_exact     — houses.house_fias_id = :fias (ci)      (confidence 0.95)
      1.  fingerprint    — house_address_aliases.fingerprint      (confidence 0.9)
      2.  geo_proximity  — within 50 m of an existing house        (confidence 0.7)

    Args:
        house_fias_id: ГАР OBJECTGUID (UUID) of the target building when known
            (e.g. DaData /clean/address or a client-supplied target_fias_id).
            Additive/optional — omitting it preserves prior behaviour exactly.

    Returns (house_id, confidence, method) or None if nothing confident matches.
    Uses 50 m for geo (vs 30 m in match_or_create_house) to absorb geocoder jitter
    on the user-entered target address.
    """
    # Tier 0: cadastral number exact match
    if cadastral_number:
        row = (
            db.execute(
                text("SELECT id FROM houses WHERE cadastral_number = :cad ORDER BY id ASC LIMIT 1"),
                {"cad": cadastral_number},
            )
            .mappings()
            .first()
        )
        if row:
            house_id = int(row["id"])
            logger.info(
                "house readonly match cadastr_exact house_id=%s cad=%s", house_id, cadastral_number
            )
            return (house_id, 1.0, "cadastr_exact")

    # Tier 0.5: house_fias_id (ГАР OBJECTGUID) exact match, case-insensitive.
    if house_fias_id:
        row = (
            db.execute(
                text(
                    "SELECT id FROM houses "
                    "WHERE lower(house_fias_id) = lower(CAST(:fias AS text)) "
                    "ORDER BY id ASC LIMIT 1"
                ),
                {"fias": house_fias_id},
            )
            .mappings()
            .first()
        )
        if row:
            house_id = int(row["id"])
            logger.info(
                "house readonly match fias_exact house_id=%s fias=%s", house_id, house_fias_id
            )
            return (house_id, 0.95, "fias_exact")

    # Tier 1: address fingerprint lookup
    fp = address_fingerprint(address, lat, lon)
    row = (
        db.execute(
            text("SELECT house_id FROM house_address_aliases WHERE fingerprint = :fp LIMIT 1"),
            {"fp": fp},
        )
        .mappings()
        .first()
    )
    if row:
        house_id = int(row["house_id"])
        logger.info("house readonly match fingerprint house_id=%s fp=%s", house_id, fp)
        return (house_id, 0.9, "fingerprint")

    # Tier 2: geo-proximity — within 50 m
    if lat is not None and lon is not None:
        row = (
            db.execute(
                text("""
                SELECT id,
                       ST_Distance(geom::geography, ST_MakePoint(:lon, :lat)::geography) AS dist,
                       COALESCE(short_address, full_address, address) AS h_addr
                FROM houses
                WHERE geom IS NOT NULL
                  AND ST_DWithin(geom::geography, ST_MakePoint(:lon, :lat)::geography, 50)
                ORDER BY dist ASC
                LIMIT 1
            """),
                {"lat": lat, "lon": lon},
            )
            .mappings()
            .first()
        )
        if row:
            # P2: same house-number consistency guard as match_or_create_house. Reject a
            # geo candidate whose house number differs from the target's; accept when the
            # numbers agree or at least one side has no number.
            cand_token = house_number_token(normalize_address(row.get("h_addr")))
            lst_token = house_number_token(normalize_address(address))
            if cand_token is not None and lst_token is not None and cand_token != lst_token:
                logger.info(
                    "house readonly geo reject: target %s != house %s (dist=%.0f)",
                    lst_token,
                    cand_token,
                    float(row["dist"]),
                )
            else:
                house_id = int(row["id"])
                logger.info(
                    "house readonly match geo_proximity house_id=%s dist=%.1f",
                    house_id,
                    float(row["dist"]),
                )
                return (house_id, 0.7, "geo_proximity")

    return None


def _upsert_house_source(
    db: Session,
    *,
    house_id: int,
    ext_source: str,
    ext_id: str,
    method: str,
    confidence: float,
) -> None:
    """Insert or refresh house_sources row for this source+ext_id."""
    db.execute(
        text("""
            INSERT INTO house_sources (
                house_id, ext_source, ext_id, confidence, matched_method, matched_at, last_seen_at
            ) VALUES (
                CAST(:hid AS bigint), :s, :e,
                CAST(:c AS real), :m, NOW(), NOW()
            )
            ON CONFLICT (ext_source, ext_id) DO UPDATE SET
                confidence    = GREATEST(EXCLUDED.confidence, house_sources.confidence),
                -- Keep provenance consistent with the winning confidence: only adopt the
                -- new method/timestamp when the incoming match is more confident than the
                -- stored one (e.g. geo_proximity/0.7 later upgraded to cadastr_exact/1.0).
                matched_method = CASE
                    WHEN EXCLUDED.confidence > house_sources.confidence
                    THEN EXCLUDED.matched_method
                    ELSE house_sources.matched_method
                END,
                matched_at    = CASE
                    WHEN EXCLUDED.confidence > house_sources.confidence
                    THEN EXCLUDED.matched_at
                    ELSE house_sources.matched_at
                END,
                last_seen_at  = NOW()
        """),
        {"hid": house_id, "s": ext_source, "e": str(ext_id), "c": confidence, "m": method},
    )


def _insert_alias(
    db: Session,
    *,
    house_id: int,
    address: str | None,
    fp: str,
    source: str,
) -> None:
    """Register normalized_address alias for this house (idempotent on normalized_address).

    ON CONFLICT DO UPDATE keeps the fingerprint in sync with the most recent call so that
    two scrapers providing the same address with slightly different coords (beyond the
    4-decimal rounding boundary) do NOT spawn duplicate alias rows — they converge to one
    row with the latest fingerprint, which is then found by Tier 2a on the next scrape.
    house_id is not updated on conflict: the first writer wins canonical ownership.

    Oblast guard (bug #2): the fingerprint/source are refreshed ONLY when the conflicting
    row belongs to the SAME house. A DIFFERENT house colliding on a bare common-street
    normalized_address (an oblast card that fell through Tier-2b's guard to a New INSERT)
    must NOT rewrite the owner's fingerprint — doing so would downgrade a safe coord-bearing
    alias to a coord-less one and let the next no-coord card mis-bucket via Tier 2a.

    P1: a bare-street normalized_address (no house number) is NOT registered as an
    alias — it is too ambiguous to serve as a building key. Any later numberless
    listing would otherwise hit that alias via Tier 2b and be mass-dumped into the
    wrong house (house 131237 collapsed 8 distinct numbers via a bare-street alias).
    """
    na = normalize_address(address)
    if not has_house_number(na):
        return
    db.execute(
        text("""
            INSERT INTO house_address_aliases (house_id, normalized_address, fingerprint, source)
            VALUES (CAST(:hid AS bigint), :na, :fp, :src)
            ON CONFLICT (normalized_address) DO UPDATE SET
                fingerprint = CASE
                    WHEN house_address_aliases.house_id = EXCLUDED.house_id
                    THEN EXCLUDED.fingerprint ELSE house_address_aliases.fingerprint END,
                source = CASE
                    WHEN house_address_aliases.house_id = EXCLUDED.house_id
                    THEN EXCLUDED.source ELSE house_address_aliases.source END
        """),
        {
            "hid": house_id,
            "na": na,
            "fp": fp,
            "src": source,
        },
    )