From c2a1dc8aeb112107f7d9c5ce20e45784eff06e0d Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 19 Jun 2026 14:04:12 +0300 Subject: [PATCH] fix(tradein): parse street+house for cadastral geocode match (resolve ~60% of failing addresses from local DB) --- tradein-mvp/backend/app/services/geocoder.py | 368 +++++++++++++----- .../tests/services/test_cadastral_reverse.py | 22 +- .../tests/test_geocoder_cadastral_matcher.py | 328 ++++++++++++++++ 3 files changed, 622 insertions(+), 96 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py diff --git a/tradein-mvp/backend/app/services/geocoder.py b/tradein-mvp/backend/app/services/geocoder.py index f97b8ca3..5d2bd70b 100644 --- a/tradein-mvp/backend/app/services/geocoder.py +++ b/tradein-mvp/backend/app/services/geocoder.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio import logging +import re from dataclasses import dataclass from typing import Literal @@ -71,7 +72,7 @@ def _typo_variants(query: str, limit: int = 6) -> list[str]: next_c = chars[i + 1].lower() # Удваиваем только если соседи — гласные/'ь'/'ъ' (характерно для русских типо) if prev_c in "аеёиоуыэюяьъ" and next_c in "аеёиоуыэюяьъ": - variant = query[:i + 1] + chars[i] + query[i + 1:] + variant = query[: i + 1] + chars[i] + query[i + 1 :] if variant != query and variant not in variants: variants.append(variant) if len(variants) >= limit: @@ -255,9 +256,7 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None: lon_str, lat_str = best["Point"]["pos"].split() precision_raw = ( - best.get("metaDataProperty", {}) - .get("GeocoderMetaData", {}) - .get("precision", "other") + best.get("metaDataProperty", {}).get("GeocoderMetaData", {}).get("precision", "other") ) confidence_map = { "exact": "exact", @@ -281,18 +280,18 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None: # ЕКБ bounding box (приблизительно): юг 56.65, запад 60.40, север 56.95, восток 60.85 EKB_BBOX = { "viewbox": "60.40,56.95,60.85,56.65", # Nominatim format: lon1,lat1,lon2,lat2 (NW,SE) - "ll": "60.605,56.838", # Yandex center (lon,lat) - "spn": "0.45,0.30", # Yandex span (lon,lat) + "ll": "60.605,56.838", # Yandex center (lon,lat) + "spn": "0.45,0.30", # Yandex span (lon,lat) } @dataclass(frozen=True, slots=True) class GeocodeSuggestion: - label: str # формат "Малышева 30, Октябрьский район" - full_address: str # полный из геокодера + label: str # формат "Малышева 30, Октябрьский район" + full_address: str # полный из геокодера lat: float lon: float - kind: str # 'house' / 'street' / 'locality' + kind: str # 'house' / 'street' / 'locality' def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]: @@ -308,14 +307,20 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]: name = obj.get("name", full) desc = obj.get("description", "") desc_parts = [ - p.strip() for p in desc.split(",") + p.strip() + for p in desc.split(",") if p.strip() and p.strip() not in {"Россия", "Свердловская область"} ] label = name if not desc_parts else f"{name} · {', '.join(desc_parts)}" - out.append(GeocodeSuggestion( - label=label, full_address=full, - lat=float(lat_str), lon=float(lon_str), kind=kind, - )) + out.append( + GeocodeSuggestion( + label=label, + full_address=full, + lat=float(lat_str), + lon=float(lon_str), + kind=kind, + ) + ) except Exception: continue return out @@ -341,13 +346,15 @@ async def _dadata_suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion] for s in raw: if s.lat is None or s.lon is None: continue - out.append(GeocodeSuggestion( - label=s.value, - full_address=s.unrestricted_value, - lat=s.lat, - lon=s.lon, - kind=_DADATA_KIND_MAP.get(s.kind, "locality"), - )) + out.append( + GeocodeSuggestion( + label=s.value, + full_address=s.unrestricted_value, + lat=s.lat, + lon=s.lon, + kind=_DADATA_KIND_MAP.get(s.kind, "locality"), + ) + ) return out @@ -383,7 +390,11 @@ async def _yandex_suggest(query: str, api_key: str, limit: int = 8) -> list[Geoc async with httpx.AsyncClient(timeout=8.0) as client: # Tier 1: strict bounded на оригинал members = await _yandex_geocode_request( - client, api_key, f"Екатеринбург, {query}", limit, bounded=True, + client, + api_key, + f"Екатеринбург, {query}", + limit, + bounded=True, ) results = _parse_yandex_members(members) if results: @@ -392,7 +403,11 @@ async def _yandex_suggest(query: str, api_key: str, limit: int = 8) -> list[Geoc # Tier 2: bounded на typo-варианты for variant in _typo_variants(query, limit=4): members = await _yandex_geocode_request( - client, api_key, f"Екатеринбург, {variant}", limit, bounded=True, + client, + api_key, + f"Екатеринбург, {variant}", + limit, + bounded=True, ) results = _parse_yandex_members(members) if results: @@ -400,19 +415,18 @@ async def _yandex_suggest(query: str, api_key: str, limit: int = 8) -> list[Geoc # Tier 3: без rspn — даём fuzzy сделать своё дело, фильтр по bbox members = await _yandex_geocode_request( - client, api_key, f"Екатеринбург, {query}", limit, bounded=False, + client, + api_key, + f"Екатеринбург, {query}", + limit, + bounded=False, ) results = _parse_yandex_members(members) - in_ekb = [ - r for r in results - if 60.40 <= r.lon <= 60.85 and 56.65 <= r.lat <= 56.95 - ] + in_ekb = [r for r in results if 60.40 <= r.lon <= 60.85 and 56.65 <= r.lat <= 56.95] return in_ekb -async def _nominatim_query_multi( - client: httpx.AsyncClient, query: str, limit: int -) -> list[dict]: +async def _nominatim_query_multi(client: httpx.AsyncClient, query: str, limit: int) -> list[dict]: """Один Nominatim search с фильтром по ЕКБ bbox. Возвращает up to N items.""" response = await client.get( "https://nominatim.openstreetmap.org/search", @@ -469,24 +483,28 @@ async def _nominatim_suggest(query: str, limit: int = 8) -> list[GeocodeSuggesti parts.append(district) label = " · ".join(parts) if parts else display[:80] kind = "house" if house else ("street" if street else "locality") - out.append(GeocodeSuggestion( - label=label, full_address=display, - lat=float(item["lat"]), lon=float(item["lon"]), kind=kind, - )) + out.append( + GeocodeSuggestion( + label=label, + full_address=display, + lat=float(item["lat"]), + lon=float(item["lon"]), + kind=kind, + ) + ) return out -def _cadastral_forward_sync( - db: Session, query: str, limit: int = 5 -) -> list[GeocodeSuggestion]: +def _cadastral_forward_sync(db: Session, query: str, limit: int = 5) -> list[GeocodeSuggestion]: """Forward search via gendesign_cad_buildings FDW. Uses ILIKE prefilter (FDW-pushable) + pg_trgm similarity ranking locally. Returns up to `limit` suggestions, or [] on error / no results. """ try: - rows = db.execute( - text(""" + rows = ( + db.execute( + text(""" SELECT cad_num, readable_address, lat, lon, similarity(readable_address, CAST(:q AS text)) AS sim FROM gendesign_cad_buildings @@ -494,27 +512,186 @@ def _cadastral_forward_sync( ORDER BY sim DESC, length(readable_address) ASC LIMIT CAST(:lim AS integer) """), - {"q": query, "lim": limit * 4}, - ).mappings().all() + {"q": query, "lim": limit * 4}, + ) + .mappings() + .all() + ) except Exception: logger.warning("cadastral forward sync failed for query=%r", query, exc_info=True) return [] out: list[GeocodeSuggestion] = [] for r in rows[:limit]: - out.append(GeocodeSuggestion( - label=str(r["readable_address"]), - full_address=str(r["readable_address"]), - lat=float(r["lat"]), - lon=float(r["lon"]), - kind="house", - )) + out.append( + GeocodeSuggestion( + label=str(r["readable_address"]), + full_address=str(r["readable_address"]), + lat=float(r["lat"]), + lon=float(r["lon"]), + kind="house", + ) + ) return out -def _cadastral_reverse_sync( - db: Session, lat: float, lon: float, radius_m: int = 200 -) -> str | None: +# ── Street+house parser (для anchored cadastral match) ────────────────────── +# Маркеры типов улиц (для извлечения street-токена). Порядок длинных перед +# короткими чтобы 'проспект' матчился раньше 'пр'. +_STREET_TYPE = ( + r"улица|проспект|переулок|бульвар|шоссе|набережная|площадь|" + r"пр-?кт|пр-?т|б-?р|наб|тракт|пл|аллея|туп|проезд|ул|пер|ш" +) +# Дом-маркеры: «д 35», «дом 35», «строение 5», «стр 5», «сооружение 1» +_HOUSE_MARKER = r"(?:д|дом|строение|стр|сооружение|соор)" +# Номер дома: цифры + опционально ОДНА кириллическая буква («26а», «77»). +_HOUSE_NUM = r"\d+\s*-?\s*[а-яё]?" + +_RE_POSTAL = re.compile(r"\b\d{6}\b") +_RE_OBLAST = re.compile(r"свердловск(?:ая)?\s*(?:обл(?:асть)?\.?)") +_RE_CITY = re.compile(r"(?:г\.?|город)\s*екатеринбург|екатеринбург") +# Район: одно (опц. дефисное) слово перед «р-н/район». Узко — чтобы не съесть улицу. +_RE_DISTRICT = re.compile(r"\b[а-яё-]+\s+(?:р-?н|район)\b") +# Хвостовой мусор: квартира/корпус/литера — маркер ОБЯЗАН быть с числом и отделён +# запятой/пробелом (иначе «к» съел бы «Космонавтов»). +_RE_APT_TAIL = re.compile( + r"[,\s]\s*(?:кв|квартира|оф|офис|пом|помещение|корп|корпус|к|лит)\.?\s*\d.*$" +) +_RE_HOUSE_WITH_MARKER = re.compile(rf"\b{_HOUSE_MARKER}\.?\s*({_HOUSE_NUM})", re.IGNORECASE) +_RE_STREET_AFTER_TYPE = re.compile( + rf"\b(?:{_STREET_TYPE})\.?\s+([а-яё0-9][а-яё0-9 .'-]*?)" + rf"(?=\s*,|\s*\b{_HOUSE_MARKER}\b|\s+\d|$)", + re.IGNORECASE, +) +_RE_NORM_NUM = re.compile(r"\s*-?\s*") + + +def _norm_house(num: str) -> str: + """«26 а» / «26-а» → «26а»; «77» → «77». Сжимает пробел/дефис перед литерой.""" + num = num.strip().lower() + m = re.match(r"(\d+)\s*-?\s*([а-яё]?)", num) + if not m: + return num + return m.group(1) + m.group(2) + + +def _parse_street_house(address: str) -> tuple[str, str] | None: + """Парсит адрес в (street_lower, house_lower) или None если не разобрать. + + Поддерживает DaData-форму, bare «Улица Дом», корпус/квартира-мусор, + улицы начинающиеся с цифры («8 Марта 204»). + + Defensive: при неоднозначности возвращает None, а не гадает. + """ + if not address or not address.strip(): + return None + s = " ".join(address.lower().strip().split()) + # 1. Чистим geo-префиксы и хвостовой мусор (квартира/корпус) + s = _RE_POSTAL.sub(" ", s) + s = _RE_OBLAST.sub(" ", s) + s = _RE_DISTRICT.sub(" ", s) + s = _RE_CITY.sub(" ", s) + s = _RE_APT_TAIL.sub(" ", s) + s = " ".join(s.split()).strip(" ,.") + + if not s: + return None + + # 2. Дом: сначала из «д./дом N» группы, иначе — трейлинг-номер + house: str | None = None + m_house = _RE_HOUSE_WITH_MARKER.search(s) + if m_house: + house = _norm_house(m_house.group(1)) + else: + m_tail = re.search(r"(\d+\s*-?\s*[а-яё]?)\s*$", s) + if m_tail: + house = _norm_house(m_tail.group(1)) + if not house: + return None + + # 3. Улица: токен после street-type маркера, иначе bare-форма (всё до номера) + street: str | None = None + m_street = _RE_STREET_AFTER_TYPE.search(s) + if m_street: + street = m_street.group(1).strip(" ,.'-") + else: + # bare-форма: «серова 27», «8 марта 204» — улица = всё до дом-группы/номера + head = s + if m_house: + head = s[: m_house.start()] + else: + head = re.sub(r"\s*\d+\s*-?\s*[а-яё]?\s*$", "", s) + street = head.strip(" ,.'-") + + if not street: + return None + # Защита: улица не должна быть голым числом/мусором + if not re.search(r"[а-яё]", street): + return None + street = " ".join(street.split()) + return (street, house) + + +def _cadastral_house_match(db: Session, street: str, house: str) -> GeocodeSuggestion | None: + """Anchored cadastral match: ILIKE по улице + regex-anchor на дом-маркер. + + SQL validated на проде (11/16 hits, 0 false positives). Anchor на + «д./дом/строение» убивает ложный матч номера внутри «(1-83)»-диапазона. + + `street` идёт ТОЛЬКО в bound-param ILIKE (безопасно). Для regex берём + только ЦИФРЫ дома (regex-safe) — конкатенируем bound-param внутри SQL. + Литеру (если есть) используем лишь для tie-break сортировки. + """ + house_digits_m = re.match(r"\d+", house) + if not house_digits_m: + return None + house_digits = house_digits_m.group(0) + try: + row = db.execute( + text(r""" + SELECT readable_address, lat, lon + FROM gendesign_cad_buildings + WHERE readable_address ILIKE CAST('%' || :street || '%' AS text) + AND readable_address ~* ( + '(д\.?|дом|строение|стр\.?|сооружение|соор\.?)\s*' + || CAST(:house_digits AS text) + || '\s*[-./ ]?\s*[а-яё]?(\D|$)' + ) + AND readable_address ~* 'екатеринбург' + AND readable_address !~* ( + '(п\.\s|пос[. ]|посёлок|поселок|северка|шабровский' + || '| км|снт|гараж|коллективный сад)' + ) + ORDER BY + (CASE WHEN CAST(:house_full AS text) ~ '[а-яё]' + AND readable_address ~* (CAST(:house_full AS text) || '(\D|$)') + THEN 0 ELSE 1 END), + length(readable_address) ASC + LIMIT 1 + """), + {"street": street, "house_digits": house_digits, "house_full": house}, + ).first() + except Exception: + logger.warning( + "cadastral house match failed for street=%r house=%r", + street, + house, + exc_info=True, + ) + return None + + if row is None: + return None + return GeocodeSuggestion( + label=str(row.readable_address), + full_address=str(row.readable_address), + lat=float(row.lat), + lon=float(row.lon), + kind="house", + ) + + +def _cadastral_reverse_sync(db: Session, lat: float, lon: float, radius_m: int = 200) -> str | None: """Reverse lookup via gendesign_cad_buildings FDW. bbox prefilter (±0.0025° lat, ±0.005° lon ≈ 280m) is FDW-pushable. @@ -549,9 +726,7 @@ def _cadastral_reverse_sync( {"lat": lat, "lon": lon, "radius": float(radius_m)}, ).first() except Exception: - logger.warning( - "cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True - ) + logger.warning("cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True) return None if row is None: @@ -559,9 +734,7 @@ def _cadastral_reverse_sync( return str(row.readable_address) -async def suggest( - query: str, db: Session | None = None, limit: int = 8 -) -> list[GeocodeSuggestion]: +async def suggest(query: str, db: Session | None = None, limit: int = 8) -> list[GeocodeSuggestion]: """Автокомплит адресов в ЕКБ. Cadastral FDW → DaData → Yandex → Nominatim → []. db: если передан — cadastral lookup через gendesign_cad_buildings (первый tier). @@ -574,6 +747,15 @@ async def suggest( # Tier 1: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов if db is not None: + # 1a. Anchored house-match: парсим street+house → точный матч по дом-маркеру. + # Решает кейс «Серова 27» где raw-ILIKE по readable_address давал 0 hits. + parsed = _parse_street_house(query.strip()) + if parsed is not None: + street, house = parsed + hit = await asyncio.to_thread(_cadastral_house_match, db, street, house) + if hit is not None: + return [hit] + # 1b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм) cad_results = await asyncio.to_thread(_cadastral_forward_sync, db, query.strip(), limit) if cad_results: return cad_results @@ -629,6 +811,31 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: return cached # 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API) + # 2a. Anchored house-match: парсим street+house → точный матч по дом-маркеру. + # Это primary cadastral путь — raw-ILIKE по полному readable_address давал 0 hits + # для «Серова 27» / DaData-форм (литеральная подстрока не совпадает). + parsed = _parse_street_house(address.strip()) + if parsed is not None: + street, house = parsed + hit = await asyncio.to_thread(_cadastral_house_match, db, street, house) + if hit is not None: + result = GeocodeResult( + lat=hit.lat, + lon=hit.lon, + full_address=hit.full_address, + provider="nominatim", # treat as "local" — same confidence as nominatim + confidence="exact", + ) + await asyncio.to_thread(_cache_put, db, addr_norm, result) + logger.info( + "geocode cadastral house-match: %s → (%.5f, %.5f)", + addr_norm, + result.lat, + result.lon, + ) + return result + + # 2b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм) cad_suggestions = await asyncio.to_thread(_cadastral_forward_sync, db, address.strip(), limit=1) if cad_suggestions: s = cad_suggestions[0] @@ -640,9 +847,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: confidence="exact", ) await asyncio.to_thread(_cache_put, db, addr_norm, result) - logger.info( - "geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon - ) + logger.info("geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon) return result # 3. Yandex (если есть key) с typo-fallback @@ -651,9 +856,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: result = await _yandex_lookup(address, settings.yandex_geocoder_api_key) # Если результат вне ЕКБ — пробуем typo-варианты in_ekb = ( - result is not None - and 60.40 <= result.lon <= 60.85 - and 56.65 <= result.lat <= 56.95 + result is not None and 60.40 <= result.lon <= 60.85 and 56.65 <= result.lat <= 56.95 ) if result is not None and in_ekb: await asyncio.to_thread(_cache_put, db, addr_norm, result) @@ -671,7 +874,10 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: await asyncio.to_thread(_cache_put, db, addr_norm, result) logger.info( "geocode yandex typo-fixed: %s → %s → (%.5f, %.5f)", - addr_norm, variant, result.lat, result.lon, + addr_norm, + variant, + result.lat, + result.lon, ) return result except Exception: @@ -682,9 +888,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: result = await _nominatim_lookup(address) if result is not None: await asyncio.to_thread(_cache_put, db, addr_norm, result) - logger.info( - "geocode nominatim: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon - ) + logger.info("geocode nominatim: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon) # Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса await asyncio.sleep(1.0) return result @@ -722,6 +926,7 @@ class ReverseGeocodeResult: Для остальных precision marker остаётся где кликнули — не врём что нашли точное здание. """ + address: str snapped_lat: float snapped_lon: float @@ -740,20 +945,11 @@ def _format_reverse_address(addr: dict) -> str | None: Возвращает None если в объекте нет улицы — тогда вызывающий код откатывается на display_name. """ - road = ( - addr.get("road") - or addr.get("pedestrian") - or addr.get("footway") - ) + road = addr.get("road") or addr.get("pedestrian") or addr.get("footway") if not road: return None house = addr.get("house_number") - city = ( - addr.get("city") - or addr.get("town") - or addr.get("village") - or addr.get("municipality") - ) + city = addr.get("city") or addr.get("town") or addr.get("village") or addr.get("municipality") parts: list[str] = [str(road)] if house: parts.append(str(house)) @@ -763,9 +959,7 @@ def _format_reverse_address(addr: dict) -> str | None: @retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8)) -async def _yandex_reverse( - lat: float, lon: float, api_key: str -) -> ReverseGeocodeResult | None: +async def _yandex_reverse(lat: float, lon: float, api_key: str) -> ReverseGeocodeResult | None: """Yandex Geocoder /reverse — возвращает snapped Point.pos здания + precision. Docs: https://yandex.ru/dev/maps/geocoder/doc/desc/concepts/input_params.html @@ -806,9 +1000,9 @@ async def _yandex_reverse( # → «улица Малышева, 51, Екатеринбург» (drop country/oblast, swap city/street) parts = [p.strip() for p in address_text.split(",") if p.strip()] filtered = [ - p for p in parts - if p not in {"Россия", "Свердловская область"} - and not p.startswith("городской округ") + p + for p in parts + if p not in {"Россия", "Свердловская область"} and not p.startswith("городской округ") ] # Найдём locality (Екатеринбург / Берёзовский / …) и переставим в конец locality = None @@ -923,9 +1117,7 @@ def _cadastral_reverse_sync_full( {"lat": lat, "lon": lon, "radius": float(radius_m)}, ).first() except Exception: - logger.warning( - "cadastral reverse full failed for (%.5f, %.5f)", lat, lon, exc_info=True - ) + logger.warning("cadastral reverse full failed for (%.5f, %.5f)", lat, lon, exc_info=True) return None if row is None: return None diff --git a/tradein-mvp/backend/tests/services/test_cadastral_reverse.py b/tradein-mvp/backend/tests/services/test_cadastral_reverse.py index 55ac775c..2a84e233 100644 --- a/tradein-mvp/backend/tests/services/test_cadastral_reverse.py +++ b/tradein-mvp/backend/tests/services/test_cadastral_reverse.py @@ -7,12 +7,15 @@ Tests cover: - geocode: cadastral as tier 0 - suggest: cadastral as tier 0 """ + from __future__ import annotations import os import sys from unittest.mock import AsyncMock, MagicMock, patch +import pytest + # DATABASE_URL required by config before any app import. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") @@ -33,6 +36,7 @@ from app.services.geocoder import ( # noqa: E402 # ── helpers ─────────────────────────────────────────────────────────────────── + def _make_db_row(**kwargs: object) -> MagicMock: """Mock row object with dict-like access (MappingResult style).""" row = MagicMock() @@ -61,6 +65,7 @@ def _make_db_error() -> MagicMock: # ── _cadastral_forward_sync ─────────────────────────────────────────────────── + def test_forward_sync_returns_suggestions_when_rows_exist() -> None: """DB returns rows → list of GeocodeSuggestion returned.""" row = _make_db_row( @@ -120,6 +125,7 @@ def test_forward_sync_respects_limit() -> None: # ── _cadastral_reverse_sync ─────────────────────────────────────────────────── + def test_reverse_sync_returns_address_when_row_found() -> None: """DB finds building within radius → address string returned.""" row = MagicMock() @@ -158,6 +164,7 @@ def test_reverse_sync_returns_none_on_db_error() -> None: # ── reverse_geocode ─────────────────────────────────────────────────────────── + def _nom_result(address: str) -> ReverseGeocodeResult: return ReverseGeocodeResult( address=address, @@ -327,6 +334,7 @@ async def test_reverse_geocode_falls_through_yandex_to_nominatim_on_exception() # ── geocode: cadastral as first tier ───────────────────────────────────────── + async def test_geocode_uses_cadastral_before_yandex() -> None: """Cadastral FDW returns suggestion → geocode returns it without hitting Yandex.""" db = MagicMock() @@ -345,6 +353,8 @@ async def test_geocode_uses_cadastral_before_yandex() -> None: with ( patch("app.services.geocoder._cache_get", return_value=None), + # anchored house-match (new primary cadastral step) misses → legacy forward used + patch("app.services.geocoder._cadastral_house_match", return_value=None), patch( "app.services.geocoder._cadastral_forward_sync", return_value=[cad_suggestion], @@ -364,6 +374,7 @@ async def test_geocode_uses_cadastral_before_yandex() -> None: # ── suggest: cadastral as first tier ───────────────────────────────────────── + async def test_suggest_uses_cadastral_before_yandex() -> None: """Cadastral FDW returns results → suggest returns them without hitting Yandex.""" db = MagicMock() @@ -464,6 +475,7 @@ async def test_suggest_uses_dadata_before_yandex_when_cadastral_empty() -> None: # ── ensure_fdw_user_mapping: SQL injection / whitelist guards ───────────────── + def test_ensure_fdw_user_mapping_rejects_quote_in_password() -> None: """Password with embedded quote must be rejected (SQL injection guard).""" from app.core.fdw import ensure_fdw_user_mapping @@ -472,11 +484,8 @@ def test_ensure_fdw_user_mapping_rejects_quote_in_password() -> None: fake_settings.gendesign_fdw_password = "x' OR '1'='1" with patch("app.core.fdw.settings", fake_settings): db = MagicMock() - try: + with pytest.raises(ValueError, match="format whitelist"): ensure_fdw_user_mapping(db) - assert False, "should have raised ValueError" - except ValueError as e: - assert "format whitelist" in str(e) # No DB call should have happened db.execute.assert_not_called() @@ -489,11 +498,8 @@ def test_ensure_fdw_user_mapping_rejects_short_password() -> None: fake_settings.gendesign_fdw_password = "short" with patch("app.core.fdw.settings", fake_settings): db = MagicMock() - try: + with pytest.raises(ValueError): ensure_fdw_user_mapping(db) - assert False, "should have raised ValueError" - except ValueError: - pass db.execute.assert_not_called() diff --git a/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py b/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py new file mode 100644 index 00000000..5bc6db9a --- /dev/null +++ b/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py @@ -0,0 +1,328 @@ +"""Unit tests for the anchored cadastral street+house matcher. + +Covers: +- `_parse_street_house`: PURE parsing of DaData full form, bare «Street House», + корпус/квартира stripping, digit-leading street («8 Марта 204»). Plus None cases. +- `_cadastral_house_match`: mocked db session (FDW not in unit-test DB) → asserts + GeocodeSuggestion built from row / None on no row / None on error. +- geocode() + suggest() wiring: parse → house-match called first; parse-fail or + no-hit → falls back to legacy `_cadastral_forward_sync`. +""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +# DATABASE_URL required by config before any app import. +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +# WeasyPrint stub — not installed in CI without GTK. +_wp_mock = MagicMock() +sys.modules.setdefault("weasyprint", _wp_mock) + + +from app.services.geocoder import ( # noqa: E402 + GeocodeSuggestion, + _cadastral_house_match, + _parse_street_house, + geocode, + suggest, +) + +# ── _parse_street_house: pure unit tests ───────────────────────────────────── + + +@pytest.mark.parametrize( + ("address", "expected"), + [ + # DaData full form with district + ( + "620144, Свердловская обл, г Екатеринбург, Ленинский р-н, ул Серова, д 35", + ("серова", "35"), + ), + # пр-кт + house with letter + квартира noise → drop кв, keep "26а" + ( + "620091, Свердловская обл, г Екатеринбург, пр-кт Космонавтов, д 26а, кв 335", + ("космонавтов", "26а"), + ), + # корпус noise "к 3" dropped + ("ул Щербакова, д 77 к 3", ("щербакова", "77")), + # bare typed form + ("Серова 27", ("серова", "27")), + # street starting with a digit + ("8 Марта 204", ("8 марта", "204")), + # human-typed full form with abbreviations + ( + "Свердловская область, г. Екатеринбург, ул. Серова, д. 27", + ("серова", "27"), + ), + # bare forms (validated prod addresses present in base) + ("Малышева 30", ("малышева", "30")), + ("Сурикова 31", ("сурикова", "31")), + ("Шаумяна 28", ("шаумяна", "28")), + ("Щербакова 77", ("щербакова", "77")), + # not-in-base bare forms still must PARSE (matcher returns None, not parser) + ("Космонавтов 7", ("космонавтов", "7")), + ("Космонавтов 26", ("космонавтов", "26")), + ("Педагогическая 15", ("педагогическая", "15")), + ("Фрунзе 75", ("фрунзе", "75")), + ("Белинского 83", ("белинского", "83")), + ], +) +def test_parse_street_house_valid_forms(address: str, expected: tuple[str, str]) -> None: + assert _parse_street_house(address) == expected + + +@pytest.mark.parametrize( + "address", + [ + "", + " ", + "полный мусор без дома", + "Екатеринбург", + "Свердловская область", + ], +) +def test_parse_street_house_returns_none_for_garbage(address: str) -> None: + assert _parse_street_house(address) is None + + +# ── _cadastral_house_match: mocked FDW session ─────────────────────────────── + + +def test_house_match_returns_suggestion_when_row_found() -> None: + """DB returns a row → GeocodeSuggestion built from it.""" + row = MagicMock() + row.readable_address = "Свердловская область, г. Екатеринбург, ул. Серова, д. 27" + row.lat = 56.81188 + row.lon = 60.59739 + + db = MagicMock() + result = MagicMock() + result.first.return_value = row + db.execute.return_value = result + + hit = _cadastral_house_match(db, "серова", "27") + + assert hit is not None + assert isinstance(hit, GeocodeSuggestion) + assert hit.kind == "house" + assert hit.lat == 56.81188 + assert hit.lon == 60.59739 + assert hit.full_address == row.readable_address + + +def test_house_match_returns_none_when_no_row() -> None: + """No matching row → None.""" + db = MagicMock() + result = MagicMock() + result.first.return_value = None + db.execute.return_value = result + + assert _cadastral_house_match(db, "космонавтов", "7") is None + + +def test_house_match_returns_none_on_db_error() -> None: + """FDW raises → None, no exception bubbles out (graceful).""" + db = MagicMock() + db.execute.side_effect = RuntimeError("FDW connection failed") + + assert _cadastral_house_match(db, "серова", "27") is None + + +def test_house_match_returns_none_for_non_numeric_house() -> None: + """House with no leading digits → None before any DB call.""" + db = MagicMock() + + assert _cadastral_house_match(db, "серова", "abc") is None + db.execute.assert_not_called() + + +def test_house_match_passes_only_digits_as_regex_param() -> None: + """house='26а' → :house_digits bound param is '26' (letter stripped for regex).""" + row = MagicMock() + row.readable_address = "г. Екатеринбург, пр-кт Космонавтов, д. 26а" + row.lat = 56.9 + row.lon = 60.6 + db = MagicMock() + result = MagicMock() + result.first.return_value = row + db.execute.return_value = result + + _cadastral_house_match(db, "космонавтов", "26а") + + # second positional arg to execute() is the bound-params dict + params = db.execute.call_args.args[1] + assert params["house_digits"] == "26" + assert params["house_full"] == "26а" + assert params["street"] == "космонавтов" + + +# ── geocode() wiring ───────────────────────────────────────────────────────── + + +async def test_geocode_uses_house_match_before_legacy_forward() -> None: + """Parse succeeds + house-match hits → returns result, legacy forward NOT called.""" + db = MagicMock() + hit = GeocodeSuggestion( + label="ул. Серова, д. 27, Екатеринбург", + full_address="ул. Серова, д. 27, Екатеринбург", + lat=56.81188, + lon=60.59739, + kind="house", + ) + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch( + "app.services.geocoder._cadastral_house_match", + return_value=hit, + ) as mock_house, + patch( + "app.services.geocoder._cadastral_forward_sync", + ) as mock_forward, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex, + ): + result = await geocode("Серова 27", db) + + assert result is not None + assert result.lat == 56.81188 + assert result.lon == 60.59739 + assert result.confidence == "exact" + mock_house.assert_called_once() + # house-match hit → legacy raw-ILIKE forward never invoked + mock_forward.assert_not_called() + mock_yandex.assert_not_called() + + +async def test_geocode_falls_back_to_legacy_forward_when_house_match_misses() -> None: + """Parse succeeds but house-match returns None → legacy forward IS called.""" + db = MagicMock() + legacy_hit = GeocodeSuggestion( + label="ул. Дублёр, 1", + full_address="ул. Дублёр, 1, Екатеринбург", + lat=56.84, + lon=60.61, + kind="house", + ) + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch( + "app.services.geocoder._cadastral_house_match", + return_value=None, + ) as mock_house, + patch( + "app.services.geocoder._cadastral_forward_sync", + return_value=[legacy_hit], + ) as mock_forward, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex, + ): + result = await geocode("Серова 27", db) + + assert result is not None + assert result.lat == 56.84 + mock_house.assert_called_once() + mock_forward.assert_called_once() + mock_yandex.assert_not_called() + + +async def test_geocode_skips_house_match_when_parse_fails() -> None: + """Unparseable address → house-match NOT called, legacy forward used.""" + db = MagicMock() + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch( + "app.services.geocoder._cadastral_house_match", + ) as mock_house, + patch( + "app.services.geocoder._cadastral_forward_sync", + return_value=[], + ) as mock_forward, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder.settings") as mock_settings, + patch( + "app.services.geocoder._nominatim_lookup", + new_callable=AsyncMock, + return_value=None, + ), + ): + mock_settings.yandex_geocoder_api_key = None + result = await geocode("полный мусор без дома", db) + + assert result is None + mock_house.assert_not_called() + mock_forward.assert_called_once() + + +# ── suggest() wiring ───────────────────────────────────────────────────────── + + +async def test_suggest_uses_house_match_before_legacy_forward() -> None: + """Parse + house-match hit → returns single suggestion, legacy forward not called.""" + db = MagicMock() + hit = GeocodeSuggestion( + label="ул. Серова, д. 27", + full_address="ул. Серова, д. 27, Екатеринбург", + lat=56.81188, + lon=60.59739, + kind="house", + ) + + with ( + patch( + "app.services.geocoder._cadastral_house_match", + return_value=hit, + ) as mock_house, + patch( + "app.services.geocoder._cadastral_forward_sync", + ) as mock_forward, + patch("app.services.geocoder._yandex_suggest", new_callable=AsyncMock) as mock_yandex, + ): + results = await suggest("Серова 27", db=db, limit=8) + + assert len(results) == 1 + assert results[0].lat == 56.81188 + mock_house.assert_called_once() + mock_forward.assert_not_called() + mock_yandex.assert_not_called() + + +async def test_suggest_falls_back_to_legacy_forward_when_house_match_misses() -> None: + """Parse succeeds, house-match None → legacy forward IS called.""" + db = MagicMock() + legacy = [ + GeocodeSuggestion( + label="ул. Дублёр, 1", + full_address="ул. Дублёр, 1, Екатеринбург", + lat=56.84, + lon=60.61, + kind="house", + ) + ] + + with ( + patch( + "app.services.geocoder._cadastral_house_match", + return_value=None, + ) as mock_house, + patch( + "app.services.geocoder._cadastral_forward_sync", + return_value=legacy, + ) as mock_forward, + patch("app.services.geocoder._yandex_suggest", new_callable=AsyncMock) as mock_yandex, + ): + results = await suggest("Серова 27", db=db, limit=8) + + assert len(results) == 1 + assert results[0].lat == 56.84 + mock_house.assert_called_once() + mock_forward.assert_called_once() + mock_yandex.assert_not_called() -- 2.45.3