fix(tradein): чинит кадастр-матчер геокодера — резолв адресов из своей базы (~60% упавших) #1780
3 changed files with 622 additions and 96 deletions
|
|
@ -15,6 +15,7 @@ from __future__ import annotations
|
|||
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from typing import Literal
|
||||
|
||||
|
|
@ -71,7 +72,7 @@ def _typo_variants(query: str, limit: int = 6) -> list[str]:
|
|||
next_c = chars[i + 1].lower()
|
||||
# Удваиваем только если соседи — гласные/'ь'/'ъ' (характерно для русских типо)
|
||||
if prev_c in "аеёиоуыэюяьъ" and next_c in "аеёиоуыэюяьъ":
|
||||
variant = query[:i + 1] + chars[i] + query[i + 1:]
|
||||
variant = query[: i + 1] + chars[i] + query[i + 1 :]
|
||||
if variant != query and variant not in variants:
|
||||
variants.append(variant)
|
||||
if len(variants) >= limit:
|
||||
|
|
@ -255,9 +256,7 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None:
|
|||
|
||||
lon_str, lat_str = best["Point"]["pos"].split()
|
||||
precision_raw = (
|
||||
best.get("metaDataProperty", {})
|
||||
.get("GeocoderMetaData", {})
|
||||
.get("precision", "other")
|
||||
best.get("metaDataProperty", {}).get("GeocoderMetaData", {}).get("precision", "other")
|
||||
)
|
||||
confidence_map = {
|
||||
"exact": "exact",
|
||||
|
|
@ -281,18 +280,18 @@ async def _yandex_lookup(address: str, api_key: str) -> GeocodeResult | None:
|
|||
# ЕКБ bounding box (приблизительно): юг 56.65, запад 60.40, север 56.95, восток 60.85
|
||||
EKB_BBOX = {
|
||||
"viewbox": "60.40,56.95,60.85,56.65", # Nominatim format: lon1,lat1,lon2,lat2 (NW,SE)
|
||||
"ll": "60.605,56.838", # Yandex center (lon,lat)
|
||||
"spn": "0.45,0.30", # Yandex span (lon,lat)
|
||||
"ll": "60.605,56.838", # Yandex center (lon,lat)
|
||||
"spn": "0.45,0.30", # Yandex span (lon,lat)
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class GeocodeSuggestion:
|
||||
label: str # формат "Малышева 30, Октябрьский район"
|
||||
full_address: str # полный из геокодера
|
||||
label: str # формат "Малышева 30, Октябрьский район"
|
||||
full_address: str # полный из геокодера
|
||||
lat: float
|
||||
lon: float
|
||||
kind: str # 'house' / 'street' / 'locality'
|
||||
kind: str # 'house' / 'street' / 'locality'
|
||||
|
||||
|
||||
def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
|
||||
|
|
@ -308,14 +307,20 @@ def _parse_yandex_members(members: list[dict]) -> list[GeocodeSuggestion]:
|
|||
name = obj.get("name", full)
|
||||
desc = obj.get("description", "")
|
||||
desc_parts = [
|
||||
p.strip() for p in desc.split(",")
|
||||
p.strip()
|
||||
for p in desc.split(",")
|
||||
if p.strip() and p.strip() not in {"Россия", "Свердловская область"}
|
||||
]
|
||||
label = name if not desc_parts else f"{name} · {', '.join(desc_parts)}"
|
||||
out.append(GeocodeSuggestion(
|
||||
label=label, full_address=full,
|
||||
lat=float(lat_str), lon=float(lon_str), kind=kind,
|
||||
))
|
||||
out.append(
|
||||
GeocodeSuggestion(
|
||||
label=label,
|
||||
full_address=full,
|
||||
lat=float(lat_str),
|
||||
lon=float(lon_str),
|
||||
kind=kind,
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
continue
|
||||
return out
|
||||
|
|
@ -341,13 +346,15 @@ async def _dadata_suggest(query: str, limit: int = 8) -> list[GeocodeSuggestion]
|
|||
for s in raw:
|
||||
if s.lat is None or s.lon is None:
|
||||
continue
|
||||
out.append(GeocodeSuggestion(
|
||||
label=s.value,
|
||||
full_address=s.unrestricted_value,
|
||||
lat=s.lat,
|
||||
lon=s.lon,
|
||||
kind=_DADATA_KIND_MAP.get(s.kind, "locality"),
|
||||
))
|
||||
out.append(
|
||||
GeocodeSuggestion(
|
||||
label=s.value,
|
||||
full_address=s.unrestricted_value,
|
||||
lat=s.lat,
|
||||
lon=s.lon,
|
||||
kind=_DADATA_KIND_MAP.get(s.kind, "locality"),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
|
|
@ -383,7 +390,11 @@ async def _yandex_suggest(query: str, api_key: str, limit: int = 8) -> list[Geoc
|
|||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
# Tier 1: strict bounded на оригинал
|
||||
members = await _yandex_geocode_request(
|
||||
client, api_key, f"Екатеринбург, {query}", limit, bounded=True,
|
||||
client,
|
||||
api_key,
|
||||
f"Екатеринбург, {query}",
|
||||
limit,
|
||||
bounded=True,
|
||||
)
|
||||
results = _parse_yandex_members(members)
|
||||
if results:
|
||||
|
|
@ -392,7 +403,11 @@ async def _yandex_suggest(query: str, api_key: str, limit: int = 8) -> list[Geoc
|
|||
# Tier 2: bounded на typo-варианты
|
||||
for variant in _typo_variants(query, limit=4):
|
||||
members = await _yandex_geocode_request(
|
||||
client, api_key, f"Екатеринбург, {variant}", limit, bounded=True,
|
||||
client,
|
||||
api_key,
|
||||
f"Екатеринбург, {variant}",
|
||||
limit,
|
||||
bounded=True,
|
||||
)
|
||||
results = _parse_yandex_members(members)
|
||||
if results:
|
||||
|
|
@ -400,19 +415,18 @@ async def _yandex_suggest(query: str, api_key: str, limit: int = 8) -> list[Geoc
|
|||
|
||||
# Tier 3: без rspn — даём fuzzy сделать своё дело, фильтр по bbox
|
||||
members = await _yandex_geocode_request(
|
||||
client, api_key, f"Екатеринбург, {query}", limit, bounded=False,
|
||||
client,
|
||||
api_key,
|
||||
f"Екатеринбург, {query}",
|
||||
limit,
|
||||
bounded=False,
|
||||
)
|
||||
results = _parse_yandex_members(members)
|
||||
in_ekb = [
|
||||
r for r in results
|
||||
if 60.40 <= r.lon <= 60.85 and 56.65 <= r.lat <= 56.95
|
||||
]
|
||||
in_ekb = [r for r in results if 60.40 <= r.lon <= 60.85 and 56.65 <= r.lat <= 56.95]
|
||||
return in_ekb
|
||||
|
||||
|
||||
async def _nominatim_query_multi(
|
||||
client: httpx.AsyncClient, query: str, limit: int
|
||||
) -> list[dict]:
|
||||
async def _nominatim_query_multi(client: httpx.AsyncClient, query: str, limit: int) -> list[dict]:
|
||||
"""Один Nominatim search с фильтром по ЕКБ bbox. Возвращает up to N items."""
|
||||
response = await client.get(
|
||||
"https://nominatim.openstreetmap.org/search",
|
||||
|
|
@ -469,24 +483,28 @@ async def _nominatim_suggest(query: str, limit: int = 8) -> list[GeocodeSuggesti
|
|||
parts.append(district)
|
||||
label = " · ".join(parts) if parts else display[:80]
|
||||
kind = "house" if house else ("street" if street else "locality")
|
||||
out.append(GeocodeSuggestion(
|
||||
label=label, full_address=display,
|
||||
lat=float(item["lat"]), lon=float(item["lon"]), kind=kind,
|
||||
))
|
||||
out.append(
|
||||
GeocodeSuggestion(
|
||||
label=label,
|
||||
full_address=display,
|
||||
lat=float(item["lat"]),
|
||||
lon=float(item["lon"]),
|
||||
kind=kind,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _cadastral_forward_sync(
|
||||
db: Session, query: str, limit: int = 5
|
||||
) -> list[GeocodeSuggestion]:
|
||||
def _cadastral_forward_sync(db: Session, query: str, limit: int = 5) -> list[GeocodeSuggestion]:
|
||||
"""Forward search via gendesign_cad_buildings FDW.
|
||||
|
||||
Uses ILIKE prefilter (FDW-pushable) + pg_trgm similarity ranking locally.
|
||||
Returns up to `limit` suggestions, or [] on error / no results.
|
||||
"""
|
||||
try:
|
||||
rows = db.execute(
|
||||
text("""
|
||||
rows = (
|
||||
db.execute(
|
||||
text("""
|
||||
SELECT cad_num, readable_address, lat, lon,
|
||||
similarity(readable_address, CAST(:q AS text)) AS sim
|
||||
FROM gendesign_cad_buildings
|
||||
|
|
@ -494,27 +512,186 @@ def _cadastral_forward_sync(
|
|||
ORDER BY sim DESC, length(readable_address) ASC
|
||||
LIMIT CAST(:lim AS integer)
|
||||
"""),
|
||||
{"q": query, "lim": limit * 4},
|
||||
).mappings().all()
|
||||
{"q": query, "lim": limit * 4},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("cadastral forward sync failed for query=%r", query, exc_info=True)
|
||||
return []
|
||||
|
||||
out: list[GeocodeSuggestion] = []
|
||||
for r in rows[:limit]:
|
||||
out.append(GeocodeSuggestion(
|
||||
label=str(r["readable_address"]),
|
||||
full_address=str(r["readable_address"]),
|
||||
lat=float(r["lat"]),
|
||||
lon=float(r["lon"]),
|
||||
kind="house",
|
||||
))
|
||||
out.append(
|
||||
GeocodeSuggestion(
|
||||
label=str(r["readable_address"]),
|
||||
full_address=str(r["readable_address"]),
|
||||
lat=float(r["lat"]),
|
||||
lon=float(r["lon"]),
|
||||
kind="house",
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _cadastral_reverse_sync(
|
||||
db: Session, lat: float, lon: float, radius_m: int = 200
|
||||
) -> str | None:
|
||||
# ── Street+house parser (для anchored cadastral match) ──────────────────────
|
||||
# Маркеры типов улиц (для извлечения street-токена). Порядок длинных перед
|
||||
# короткими чтобы 'проспект' матчился раньше 'пр'.
|
||||
_STREET_TYPE = (
|
||||
r"улица|проспект|переулок|бульвар|шоссе|набережная|площадь|"
|
||||
r"пр-?кт|пр-?т|б-?р|наб|тракт|пл|аллея|туп|проезд|ул|пер|ш"
|
||||
)
|
||||
# Дом-маркеры: «д 35», «дом 35», «строение 5», «стр 5», «сооружение 1»
|
||||
_HOUSE_MARKER = r"(?:д|дом|строение|стр|сооружение|соор)"
|
||||
# Номер дома: цифры + опционально ОДНА кириллическая буква («26а», «77»).
|
||||
_HOUSE_NUM = r"\d+\s*-?\s*[а-яё]?"
|
||||
|
||||
_RE_POSTAL = re.compile(r"\b\d{6}\b")
|
||||
_RE_OBLAST = re.compile(r"свердловск(?:ая)?\s*(?:обл(?:асть)?\.?)")
|
||||
_RE_CITY = re.compile(r"(?:г\.?|город)\s*екатеринбург|екатеринбург")
|
||||
# Район: одно (опц. дефисное) слово перед «р-н/район». Узко — чтобы не съесть улицу.
|
||||
_RE_DISTRICT = re.compile(r"\b[а-яё-]+\s+(?:р-?н|район)\b")
|
||||
# Хвостовой мусор: квартира/корпус/литера — маркер ОБЯЗАН быть с числом и отделён
|
||||
# запятой/пробелом (иначе «к» съел бы «Космонавтов»).
|
||||
_RE_APT_TAIL = re.compile(
|
||||
r"[,\s]\s*(?:кв|квартира|оф|офис|пом|помещение|корп|корпус|к|лит)\.?\s*\d.*$"
|
||||
)
|
||||
_RE_HOUSE_WITH_MARKER = re.compile(rf"\b{_HOUSE_MARKER}\.?\s*({_HOUSE_NUM})", re.IGNORECASE)
|
||||
_RE_STREET_AFTER_TYPE = re.compile(
|
||||
rf"\b(?:{_STREET_TYPE})\.?\s+([а-яё0-9][а-яё0-9 .'-]*?)"
|
||||
rf"(?=\s*,|\s*\b{_HOUSE_MARKER}\b|\s+\d|$)",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_RE_NORM_NUM = re.compile(r"\s*-?\s*")
|
||||
|
||||
|
||||
def _norm_house(num: str) -> str:
|
||||
"""«26 а» / «26-а» → «26а»; «77» → «77». Сжимает пробел/дефис перед литерой."""
|
||||
num = num.strip().lower()
|
||||
m = re.match(r"(\d+)\s*-?\s*([а-яё]?)", num)
|
||||
if not m:
|
||||
return num
|
||||
return m.group(1) + m.group(2)
|
||||
|
||||
|
||||
def _parse_street_house(address: str) -> tuple[str, str] | None:
|
||||
"""Парсит адрес в (street_lower, house_lower) или None если не разобрать.
|
||||
|
||||
Поддерживает DaData-форму, bare «Улица Дом», корпус/квартира-мусор,
|
||||
улицы начинающиеся с цифры («8 Марта 204»).
|
||||
|
||||
Defensive: при неоднозначности возвращает None, а не гадает.
|
||||
"""
|
||||
if not address or not address.strip():
|
||||
return None
|
||||
s = " ".join(address.lower().strip().split())
|
||||
# 1. Чистим geo-префиксы и хвостовой мусор (квартира/корпус)
|
||||
s = _RE_POSTAL.sub(" ", s)
|
||||
s = _RE_OBLAST.sub(" ", s)
|
||||
s = _RE_DISTRICT.sub(" ", s)
|
||||
s = _RE_CITY.sub(" ", s)
|
||||
s = _RE_APT_TAIL.sub(" ", s)
|
||||
s = " ".join(s.split()).strip(" ,.")
|
||||
|
||||
if not s:
|
||||
return None
|
||||
|
||||
# 2. Дом: сначала из «д./дом N» группы, иначе — трейлинг-номер
|
||||
house: str | None = None
|
||||
m_house = _RE_HOUSE_WITH_MARKER.search(s)
|
||||
if m_house:
|
||||
house = _norm_house(m_house.group(1))
|
||||
else:
|
||||
m_tail = re.search(r"(\d+\s*-?\s*[а-яё]?)\s*$", s)
|
||||
if m_tail:
|
||||
house = _norm_house(m_tail.group(1))
|
||||
if not house:
|
||||
return None
|
||||
|
||||
# 3. Улица: токен после street-type маркера, иначе bare-форма (всё до номера)
|
||||
street: str | None = None
|
||||
m_street = _RE_STREET_AFTER_TYPE.search(s)
|
||||
if m_street:
|
||||
street = m_street.group(1).strip(" ,.'-")
|
||||
else:
|
||||
# bare-форма: «серова 27», «8 марта 204» — улица = всё до дом-группы/номера
|
||||
head = s
|
||||
if m_house:
|
||||
head = s[: m_house.start()]
|
||||
else:
|
||||
head = re.sub(r"\s*\d+\s*-?\s*[а-яё]?\s*$", "", s)
|
||||
street = head.strip(" ,.'-")
|
||||
|
||||
if not street:
|
||||
return None
|
||||
# Защита: улица не должна быть голым числом/мусором
|
||||
if not re.search(r"[а-яё]", street):
|
||||
return None
|
||||
street = " ".join(street.split())
|
||||
return (street, house)
|
||||
|
||||
|
||||
def _cadastral_house_match(db: Session, street: str, house: str) -> GeocodeSuggestion | None:
|
||||
"""Anchored cadastral match: ILIKE по улице + regex-anchor на дом-маркер.
|
||||
|
||||
SQL validated на проде (11/16 hits, 0 false positives). Anchor на
|
||||
«д./дом/строение» убивает ложный матч номера внутри «(1-83)»-диапазона.
|
||||
|
||||
`street` идёт ТОЛЬКО в bound-param ILIKE (безопасно). Для regex берём
|
||||
только ЦИФРЫ дома (regex-safe) — конкатенируем bound-param внутри SQL.
|
||||
Литеру (если есть) используем лишь для tie-break сортировки.
|
||||
"""
|
||||
house_digits_m = re.match(r"\d+", house)
|
||||
if not house_digits_m:
|
||||
return None
|
||||
house_digits = house_digits_m.group(0)
|
||||
try:
|
||||
row = db.execute(
|
||||
text(r"""
|
||||
SELECT readable_address, lat, lon
|
||||
FROM gendesign_cad_buildings
|
||||
WHERE readable_address ILIKE CAST('%' || :street || '%' AS text)
|
||||
AND readable_address ~* (
|
||||
'(д\.?|дом|строение|стр\.?|сооружение|соор\.?)\s*'
|
||||
|| CAST(:house_digits AS text)
|
||||
|| '\s*[-./ ]?\s*[а-яё]?(\D|$)'
|
||||
)
|
||||
AND readable_address ~* 'екатеринбург'
|
||||
AND readable_address !~* (
|
||||
'(п\.\s|пос[. ]|посёлок|поселок|северка|шабровский'
|
||||
|| '| км|снт|гараж|коллективный сад)'
|
||||
)
|
||||
ORDER BY
|
||||
(CASE WHEN CAST(:house_full AS text) ~ '[а-яё]'
|
||||
AND readable_address ~* (CAST(:house_full AS text) || '(\D|$)')
|
||||
THEN 0 ELSE 1 END),
|
||||
length(readable_address) ASC
|
||||
LIMIT 1
|
||||
"""),
|
||||
{"street": street, "house_digits": house_digits, "house_full": house},
|
||||
).first()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"cadastral house match failed for street=%r house=%r",
|
||||
street,
|
||||
house,
|
||||
exc_info=True,
|
||||
)
|
||||
return None
|
||||
|
||||
if row is None:
|
||||
return None
|
||||
return GeocodeSuggestion(
|
||||
label=str(row.readable_address),
|
||||
full_address=str(row.readable_address),
|
||||
lat=float(row.lat),
|
||||
lon=float(row.lon),
|
||||
kind="house",
|
||||
)
|
||||
|
||||
|
||||
def _cadastral_reverse_sync(db: Session, lat: float, lon: float, radius_m: int = 200) -> str | None:
|
||||
"""Reverse lookup via gendesign_cad_buildings FDW.
|
||||
|
||||
bbox prefilter (±0.0025° lat, ±0.005° lon ≈ 280m) is FDW-pushable.
|
||||
|
|
@ -549,9 +726,7 @@ def _cadastral_reverse_sync(
|
|||
{"lat": lat, "lon": lon, "radius": float(radius_m)},
|
||||
).first()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True
|
||||
)
|
||||
logger.warning("cadastral reverse sync failed for (%.5f, %.5f)", lat, lon, exc_info=True)
|
||||
return None
|
||||
|
||||
if row is None:
|
||||
|
|
@ -559,9 +734,7 @@ def _cadastral_reverse_sync(
|
|||
return str(row.readable_address)
|
||||
|
||||
|
||||
async def suggest(
|
||||
query: str, db: Session | None = None, limit: int = 8
|
||||
) -> list[GeocodeSuggestion]:
|
||||
async def suggest(query: str, db: Session | None = None, limit: int = 8) -> list[GeocodeSuggestion]:
|
||||
"""Автокомплит адресов в ЕКБ. Cadastral FDW → DaData → Yandex → Nominatim → [].
|
||||
|
||||
db: если передан — cadastral lookup через gendesign_cad_buildings (первый tier).
|
||||
|
|
@ -574,6 +747,15 @@ async def suggest(
|
|||
|
||||
# Tier 1: cadastral FDW (если db доступна) — самый быстрый, без внешних запросов
|
||||
if db is not None:
|
||||
# 1a. Anchored house-match: парсим street+house → точный матч по дом-маркеру.
|
||||
# Решает кейс «Серова 27» где raw-ILIKE по readable_address давал 0 hits.
|
||||
parsed = _parse_street_house(query.strip())
|
||||
if parsed is not None:
|
||||
street, house = parsed
|
||||
hit = await asyncio.to_thread(_cadastral_house_match, db, street, house)
|
||||
if hit is not None:
|
||||
return [hit]
|
||||
# 1b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм)
|
||||
cad_results = await asyncio.to_thread(_cadastral_forward_sync, db, query.strip(), limit)
|
||||
if cad_results:
|
||||
return cad_results
|
||||
|
|
@ -629,6 +811,31 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
return cached
|
||||
|
||||
# 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
|
||||
# 2a. Anchored house-match: парсим street+house → точный матч по дом-маркеру.
|
||||
# Это primary cadastral путь — raw-ILIKE по полному readable_address давал 0 hits
|
||||
# для «Серова 27» / DaData-форм (литеральная подстрока не совпадает).
|
||||
parsed = _parse_street_house(address.strip())
|
||||
if parsed is not None:
|
||||
street, house = parsed
|
||||
hit = await asyncio.to_thread(_cadastral_house_match, db, street, house)
|
||||
if hit is not None:
|
||||
result = GeocodeResult(
|
||||
lat=hit.lat,
|
||||
lon=hit.lon,
|
||||
full_address=hit.full_address,
|
||||
provider="nominatim", # treat as "local" — same confidence as nominatim
|
||||
confidence="exact",
|
||||
)
|
||||
await asyncio.to_thread(_cache_put, db, addr_norm, result)
|
||||
logger.info(
|
||||
"geocode cadastral house-match: %s → (%.5f, %.5f)",
|
||||
addr_norm,
|
||||
result.lat,
|
||||
result.lon,
|
||||
)
|
||||
return result
|
||||
|
||||
# 2b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм)
|
||||
cad_suggestions = await asyncio.to_thread(_cadastral_forward_sync, db, address.strip(), limit=1)
|
||||
if cad_suggestions:
|
||||
s = cad_suggestions[0]
|
||||
|
|
@ -640,9 +847,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
confidence="exact",
|
||||
)
|
||||
await asyncio.to_thread(_cache_put, db, addr_norm, result)
|
||||
logger.info(
|
||||
"geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon
|
||||
)
|
||||
logger.info("geocode cadastral fdw: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon)
|
||||
return result
|
||||
|
||||
# 3. Yandex (если есть key) с typo-fallback
|
||||
|
|
@ -651,9 +856,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
result = await _yandex_lookup(address, settings.yandex_geocoder_api_key)
|
||||
# Если результат вне ЕКБ — пробуем typo-варианты
|
||||
in_ekb = (
|
||||
result is not None
|
||||
and 60.40 <= result.lon <= 60.85
|
||||
and 56.65 <= result.lat <= 56.95
|
||||
result is not None and 60.40 <= result.lon <= 60.85 and 56.65 <= result.lat <= 56.95
|
||||
)
|
||||
if result is not None and in_ekb:
|
||||
await asyncio.to_thread(_cache_put, db, addr_norm, result)
|
||||
|
|
@ -671,7 +874,10 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
await asyncio.to_thread(_cache_put, db, addr_norm, result)
|
||||
logger.info(
|
||||
"geocode yandex typo-fixed: %s → %s → (%.5f, %.5f)",
|
||||
addr_norm, variant, result.lat, result.lon,
|
||||
addr_norm,
|
||||
variant,
|
||||
result.lat,
|
||||
result.lon,
|
||||
)
|
||||
return result
|
||||
except Exception:
|
||||
|
|
@ -682,9 +888,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
|
|||
result = await _nominatim_lookup(address)
|
||||
if result is not None:
|
||||
await asyncio.to_thread(_cache_put, db, addr_norm, result)
|
||||
logger.info(
|
||||
"geocode nominatim: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon
|
||||
)
|
||||
logger.info("geocode nominatim: %s → (%.5f, %.5f)", addr_norm, result.lat, result.lon)
|
||||
# Nominatim rate-limit policy: 1 req/sec — спим после успешного запроса
|
||||
await asyncio.sleep(1.0)
|
||||
return result
|
||||
|
|
@ -722,6 +926,7 @@ class ReverseGeocodeResult:
|
|||
Для остальных precision marker остаётся где кликнули — не врём что нашли
|
||||
точное здание.
|
||||
"""
|
||||
|
||||
address: str
|
||||
snapped_lat: float
|
||||
snapped_lon: float
|
||||
|
|
@ -740,20 +945,11 @@ def _format_reverse_address(addr: dict) -> str | None:
|
|||
Возвращает None если в объекте нет улицы — тогда вызывающий код
|
||||
откатывается на display_name.
|
||||
"""
|
||||
road = (
|
||||
addr.get("road")
|
||||
or addr.get("pedestrian")
|
||||
or addr.get("footway")
|
||||
)
|
||||
road = addr.get("road") or addr.get("pedestrian") or addr.get("footway")
|
||||
if not road:
|
||||
return None
|
||||
house = addr.get("house_number")
|
||||
city = (
|
||||
addr.get("city")
|
||||
or addr.get("town")
|
||||
or addr.get("village")
|
||||
or addr.get("municipality")
|
||||
)
|
||||
city = addr.get("city") or addr.get("town") or addr.get("village") or addr.get("municipality")
|
||||
parts: list[str] = [str(road)]
|
||||
if house:
|
||||
parts.append(str(house))
|
||||
|
|
@ -763,9 +959,7 @@ def _format_reverse_address(addr: dict) -> str | None:
|
|||
|
||||
|
||||
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=1, max=8))
|
||||
async def _yandex_reverse(
|
||||
lat: float, lon: float, api_key: str
|
||||
) -> ReverseGeocodeResult | None:
|
||||
async def _yandex_reverse(lat: float, lon: float, api_key: str) -> ReverseGeocodeResult | None:
|
||||
"""Yandex Geocoder /reverse — возвращает snapped Point.pos здания + precision.
|
||||
|
||||
Docs: https://yandex.ru/dev/maps/geocoder/doc/desc/concepts/input_params.html
|
||||
|
|
@ -806,9 +1000,9 @@ async def _yandex_reverse(
|
|||
# → «улица Малышева, 51, Екатеринбург» (drop country/oblast, swap city/street)
|
||||
parts = [p.strip() for p in address_text.split(",") if p.strip()]
|
||||
filtered = [
|
||||
p for p in parts
|
||||
if p not in {"Россия", "Свердловская область"}
|
||||
and not p.startswith("городской округ")
|
||||
p
|
||||
for p in parts
|
||||
if p not in {"Россия", "Свердловская область"} and not p.startswith("городской округ")
|
||||
]
|
||||
# Найдём locality (Екатеринбург / Берёзовский / …) и переставим в конец
|
||||
locality = None
|
||||
|
|
@ -923,9 +1117,7 @@ def _cadastral_reverse_sync_full(
|
|||
{"lat": lat, "lon": lon, "radius": float(radius_m)},
|
||||
).first()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"cadastral reverse full failed for (%.5f, %.5f)", lat, lon, exc_info=True
|
||||
)
|
||||
logger.warning("cadastral reverse full failed for (%.5f, %.5f)", lat, lon, exc_info=True)
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -7,12 +7,15 @@ Tests cover:
|
|||
- geocode: cadastral as tier 0
|
||||
- suggest: cadastral as tier 0
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# DATABASE_URL required by config before any app import.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
|
|
@ -33,6 +36,7 @@ from app.services.geocoder import ( # noqa: E402
|
|||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _make_db_row(**kwargs: object) -> MagicMock:
|
||||
"""Mock row object with dict-like access (MappingResult style)."""
|
||||
row = MagicMock()
|
||||
|
|
@ -61,6 +65,7 @@ def _make_db_error() -> MagicMock:
|
|||
|
||||
# ── _cadastral_forward_sync ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_forward_sync_returns_suggestions_when_rows_exist() -> None:
|
||||
"""DB returns rows → list of GeocodeSuggestion returned."""
|
||||
row = _make_db_row(
|
||||
|
|
@ -120,6 +125,7 @@ def test_forward_sync_respects_limit() -> None:
|
|||
|
||||
# ── _cadastral_reverse_sync ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_reverse_sync_returns_address_when_row_found() -> None:
|
||||
"""DB finds building within radius → address string returned."""
|
||||
row = MagicMock()
|
||||
|
|
@ -158,6 +164,7 @@ def test_reverse_sync_returns_none_on_db_error() -> None:
|
|||
|
||||
# ── reverse_geocode ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _nom_result(address: str) -> ReverseGeocodeResult:
|
||||
return ReverseGeocodeResult(
|
||||
address=address,
|
||||
|
|
@ -327,6 +334,7 @@ async def test_reverse_geocode_falls_through_yandex_to_nominatim_on_exception()
|
|||
|
||||
# ── geocode: cadastral as first tier ─────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_geocode_uses_cadastral_before_yandex() -> None:
|
||||
"""Cadastral FDW returns suggestion → geocode returns it without hitting Yandex."""
|
||||
db = MagicMock()
|
||||
|
|
@ -345,6 +353,8 @@ async def test_geocode_uses_cadastral_before_yandex() -> None:
|
|||
|
||||
with (
|
||||
patch("app.services.geocoder._cache_get", return_value=None),
|
||||
# anchored house-match (new primary cadastral step) misses → legacy forward used
|
||||
patch("app.services.geocoder._cadastral_house_match", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
return_value=[cad_suggestion],
|
||||
|
|
@ -364,6 +374,7 @@ async def test_geocode_uses_cadastral_before_yandex() -> None:
|
|||
|
||||
# ── suggest: cadastral as first tier ─────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_suggest_uses_cadastral_before_yandex() -> None:
|
||||
"""Cadastral FDW returns results → suggest returns them without hitting Yandex."""
|
||||
db = MagicMock()
|
||||
|
|
@ -464,6 +475,7 @@ async def test_suggest_uses_dadata_before_yandex_when_cadastral_empty() -> None:
|
|||
|
||||
# ── ensure_fdw_user_mapping: SQL injection / whitelist guards ─────────────────
|
||||
|
||||
|
||||
def test_ensure_fdw_user_mapping_rejects_quote_in_password() -> None:
|
||||
"""Password with embedded quote must be rejected (SQL injection guard)."""
|
||||
from app.core.fdw import ensure_fdw_user_mapping
|
||||
|
|
@ -472,11 +484,8 @@ def test_ensure_fdw_user_mapping_rejects_quote_in_password() -> None:
|
|||
fake_settings.gendesign_fdw_password = "x' OR '1'='1"
|
||||
with patch("app.core.fdw.settings", fake_settings):
|
||||
db = MagicMock()
|
||||
try:
|
||||
with pytest.raises(ValueError, match="format whitelist"):
|
||||
ensure_fdw_user_mapping(db)
|
||||
assert False, "should have raised ValueError"
|
||||
except ValueError as e:
|
||||
assert "format whitelist" in str(e)
|
||||
# No DB call should have happened
|
||||
db.execute.assert_not_called()
|
||||
|
||||
|
|
@ -489,11 +498,8 @@ def test_ensure_fdw_user_mapping_rejects_short_password() -> None:
|
|||
fake_settings.gendesign_fdw_password = "short"
|
||||
with patch("app.core.fdw.settings", fake_settings):
|
||||
db = MagicMock()
|
||||
try:
|
||||
with pytest.raises(ValueError):
|
||||
ensure_fdw_user_mapping(db)
|
||||
assert False, "should have raised ValueError"
|
||||
except ValueError:
|
||||
pass
|
||||
db.execute.assert_not_called()
|
||||
|
||||
|
||||
|
|
|
|||
328
tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py
Normal file
328
tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py
Normal file
|
|
@ -0,0 +1,328 @@
|
|||
"""Unit tests for the anchored cadastral street+house matcher.
|
||||
|
||||
Covers:
|
||||
- `_parse_street_house`: PURE parsing of DaData full form, bare «Street House»,
|
||||
корпус/квартира stripping, digit-leading street («8 Марта 204»). Plus None cases.
|
||||
- `_cadastral_house_match`: mocked db session (FDW not in unit-test DB) → asserts
|
||||
GeocodeSuggestion built from row / None on no row / None on error.
|
||||
- geocode() + suggest() wiring: parse → house-match called first; parse-fail or
|
||||
no-hit → falls back to legacy `_cadastral_forward_sync`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
# DATABASE_URL required by config before any app import.
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
# WeasyPrint stub — not installed in CI without GTK.
|
||||
_wp_mock = MagicMock()
|
||||
sys.modules.setdefault("weasyprint", _wp_mock)
|
||||
|
||||
|
||||
from app.services.geocoder import ( # noqa: E402
|
||||
GeocodeSuggestion,
|
||||
_cadastral_house_match,
|
||||
_parse_street_house,
|
||||
geocode,
|
||||
suggest,
|
||||
)
|
||||
|
||||
# ── _parse_street_house: pure unit tests ─────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("address", "expected"),
|
||||
[
|
||||
# DaData full form with district
|
||||
(
|
||||
"620144, Свердловская обл, г Екатеринбург, Ленинский р-н, ул Серова, д 35",
|
||||
("серова", "35"),
|
||||
),
|
||||
# пр-кт + house with letter + квартира noise → drop кв, keep "26а"
|
||||
(
|
||||
"620091, Свердловская обл, г Екатеринбург, пр-кт Космонавтов, д 26а, кв 335",
|
||||
("космонавтов", "26а"),
|
||||
),
|
||||
# корпус noise "к 3" dropped
|
||||
("ул Щербакова, д 77 к 3", ("щербакова", "77")),
|
||||
# bare typed form
|
||||
("Серова 27", ("серова", "27")),
|
||||
# street starting with a digit
|
||||
("8 Марта 204", ("8 марта", "204")),
|
||||
# human-typed full form with abbreviations
|
||||
(
|
||||
"Свердловская область, г. Екатеринбург, ул. Серова, д. 27",
|
||||
("серова", "27"),
|
||||
),
|
||||
# bare forms (validated prod addresses present in base)
|
||||
("Малышева 30", ("малышева", "30")),
|
||||
("Сурикова 31", ("сурикова", "31")),
|
||||
("Шаумяна 28", ("шаумяна", "28")),
|
||||
("Щербакова 77", ("щербакова", "77")),
|
||||
# not-in-base bare forms still must PARSE (matcher returns None, not parser)
|
||||
("Космонавтов 7", ("космонавтов", "7")),
|
||||
("Космонавтов 26", ("космонавтов", "26")),
|
||||
("Педагогическая 15", ("педагогическая", "15")),
|
||||
("Фрунзе 75", ("фрунзе", "75")),
|
||||
("Белинского 83", ("белинского", "83")),
|
||||
],
|
||||
)
|
||||
def test_parse_street_house_valid_forms(address: str, expected: tuple[str, str]) -> None:
|
||||
assert _parse_street_house(address) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"address",
|
||||
[
|
||||
"",
|
||||
" ",
|
||||
"полный мусор без дома",
|
||||
"Екатеринбург",
|
||||
"Свердловская область",
|
||||
],
|
||||
)
|
||||
def test_parse_street_house_returns_none_for_garbage(address: str) -> None:
|
||||
assert _parse_street_house(address) is None
|
||||
|
||||
|
||||
# ── _cadastral_house_match: mocked FDW session ───────────────────────────────
|
||||
|
||||
|
||||
def test_house_match_returns_suggestion_when_row_found() -> None:
|
||||
"""DB returns a row → GeocodeSuggestion built from it."""
|
||||
row = MagicMock()
|
||||
row.readable_address = "Свердловская область, г. Екатеринбург, ул. Серова, д. 27"
|
||||
row.lat = 56.81188
|
||||
row.lon = 60.59739
|
||||
|
||||
db = MagicMock()
|
||||
result = MagicMock()
|
||||
result.first.return_value = row
|
||||
db.execute.return_value = result
|
||||
|
||||
hit = _cadastral_house_match(db, "серова", "27")
|
||||
|
||||
assert hit is not None
|
||||
assert isinstance(hit, GeocodeSuggestion)
|
||||
assert hit.kind == "house"
|
||||
assert hit.lat == 56.81188
|
||||
assert hit.lon == 60.59739
|
||||
assert hit.full_address == row.readable_address
|
||||
|
||||
|
||||
def test_house_match_returns_none_when_no_row() -> None:
|
||||
"""No matching row → None."""
|
||||
db = MagicMock()
|
||||
result = MagicMock()
|
||||
result.first.return_value = None
|
||||
db.execute.return_value = result
|
||||
|
||||
assert _cadastral_house_match(db, "космонавтов", "7") is None
|
||||
|
||||
|
||||
def test_house_match_returns_none_on_db_error() -> None:
|
||||
"""FDW raises → None, no exception bubbles out (graceful)."""
|
||||
db = MagicMock()
|
||||
db.execute.side_effect = RuntimeError("FDW connection failed")
|
||||
|
||||
assert _cadastral_house_match(db, "серова", "27") is None
|
||||
|
||||
|
||||
def test_house_match_returns_none_for_non_numeric_house() -> None:
|
||||
"""House with no leading digits → None before any DB call."""
|
||||
db = MagicMock()
|
||||
|
||||
assert _cadastral_house_match(db, "серова", "abc") is None
|
||||
db.execute.assert_not_called()
|
||||
|
||||
|
||||
def test_house_match_passes_only_digits_as_regex_param() -> None:
|
||||
"""house='26а' → :house_digits bound param is '26' (letter stripped for regex)."""
|
||||
row = MagicMock()
|
||||
row.readable_address = "г. Екатеринбург, пр-кт Космонавтов, д. 26а"
|
||||
row.lat = 56.9
|
||||
row.lon = 60.6
|
||||
db = MagicMock()
|
||||
result = MagicMock()
|
||||
result.first.return_value = row
|
||||
db.execute.return_value = result
|
||||
|
||||
_cadastral_house_match(db, "космонавтов", "26а")
|
||||
|
||||
# second positional arg to execute() is the bound-params dict
|
||||
params = db.execute.call_args.args[1]
|
||||
assert params["house_digits"] == "26"
|
||||
assert params["house_full"] == "26а"
|
||||
assert params["street"] == "космонавтов"
|
||||
|
||||
|
||||
# ── geocode() wiring ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_geocode_uses_house_match_before_legacy_forward() -> None:
|
||||
"""Parse succeeds + house-match hits → returns result, legacy forward NOT called."""
|
||||
db = MagicMock()
|
||||
hit = GeocodeSuggestion(
|
||||
label="ул. Серова, д. 27, Екатеринбург",
|
||||
full_address="ул. Серова, д. 27, Екатеринбург",
|
||||
lat=56.81188,
|
||||
lon=60.59739,
|
||||
kind="house",
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.services.geocoder._cache_get", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_house_match",
|
||||
return_value=hit,
|
||||
) as mock_house,
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
) as mock_forward,
|
||||
patch("app.services.geocoder._cache_put"),
|
||||
patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
|
||||
):
|
||||
result = await geocode("Серова 27", db)
|
||||
|
||||
assert result is not None
|
||||
assert result.lat == 56.81188
|
||||
assert result.lon == 60.59739
|
||||
assert result.confidence == "exact"
|
||||
mock_house.assert_called_once()
|
||||
# house-match hit → legacy raw-ILIKE forward never invoked
|
||||
mock_forward.assert_not_called()
|
||||
mock_yandex.assert_not_called()
|
||||
|
||||
|
||||
async def test_geocode_falls_back_to_legacy_forward_when_house_match_misses() -> None:
|
||||
"""Parse succeeds but house-match returns None → legacy forward IS called."""
|
||||
db = MagicMock()
|
||||
legacy_hit = GeocodeSuggestion(
|
||||
label="ул. Дублёр, 1",
|
||||
full_address="ул. Дублёр, 1, Екатеринбург",
|
||||
lat=56.84,
|
||||
lon=60.61,
|
||||
kind="house",
|
||||
)
|
||||
|
||||
with (
|
||||
patch("app.services.geocoder._cache_get", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_house_match",
|
||||
return_value=None,
|
||||
) as mock_house,
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
return_value=[legacy_hit],
|
||||
) as mock_forward,
|
||||
patch("app.services.geocoder._cache_put"),
|
||||
patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
|
||||
):
|
||||
result = await geocode("Серова 27", db)
|
||||
|
||||
assert result is not None
|
||||
assert result.lat == 56.84
|
||||
mock_house.assert_called_once()
|
||||
mock_forward.assert_called_once()
|
||||
mock_yandex.assert_not_called()
|
||||
|
||||
|
||||
async def test_geocode_skips_house_match_when_parse_fails() -> None:
|
||||
"""Unparseable address → house-match NOT called, legacy forward used."""
|
||||
db = MagicMock()
|
||||
|
||||
with (
|
||||
patch("app.services.geocoder._cache_get", return_value=None),
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_house_match",
|
||||
) as mock_house,
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
return_value=[],
|
||||
) as mock_forward,
|
||||
patch("app.services.geocoder._cache_put"),
|
||||
patch("app.services.geocoder.settings") as mock_settings,
|
||||
patch(
|
||||
"app.services.geocoder._nominatim_lookup",
|
||||
new_callable=AsyncMock,
|
||||
return_value=None,
|
||||
),
|
||||
):
|
||||
mock_settings.yandex_geocoder_api_key = None
|
||||
result = await geocode("полный мусор без дома", db)
|
||||
|
||||
assert result is None
|
||||
mock_house.assert_not_called()
|
||||
mock_forward.assert_called_once()
|
||||
|
||||
|
||||
# ── suggest() wiring ─────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def test_suggest_uses_house_match_before_legacy_forward() -> None:
|
||||
"""Parse + house-match hit → returns single suggestion, legacy forward not called."""
|
||||
db = MagicMock()
|
||||
hit = GeocodeSuggestion(
|
||||
label="ул. Серова, д. 27",
|
||||
full_address="ул. Серова, д. 27, Екатеринбург",
|
||||
lat=56.81188,
|
||||
lon=60.59739,
|
||||
kind="house",
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_house_match",
|
||||
return_value=hit,
|
||||
) as mock_house,
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
) as mock_forward,
|
||||
patch("app.services.geocoder._yandex_suggest", new_callable=AsyncMock) as mock_yandex,
|
||||
):
|
||||
results = await suggest("Серова 27", db=db, limit=8)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].lat == 56.81188
|
||||
mock_house.assert_called_once()
|
||||
mock_forward.assert_not_called()
|
||||
mock_yandex.assert_not_called()
|
||||
|
||||
|
||||
async def test_suggest_falls_back_to_legacy_forward_when_house_match_misses() -> None:
|
||||
"""Parse succeeds, house-match None → legacy forward IS called."""
|
||||
db = MagicMock()
|
||||
legacy = [
|
||||
GeocodeSuggestion(
|
||||
label="ул. Дублёр, 1",
|
||||
full_address="ул. Дублёр, 1, Екатеринбург",
|
||||
lat=56.84,
|
||||
lon=60.61,
|
||||
kind="house",
|
||||
)
|
||||
]
|
||||
|
||||
with (
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_house_match",
|
||||
return_value=None,
|
||||
) as mock_house,
|
||||
patch(
|
||||
"app.services.geocoder._cadastral_forward_sync",
|
||||
return_value=legacy,
|
||||
) as mock_forward,
|
||||
patch("app.services.geocoder._yandex_suggest", new_callable=AsyncMock) as mock_yandex,
|
||||
):
|
||||
results = await suggest("Серова 27", db=db, limit=8)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].lat == 56.84
|
||||
mock_house.assert_called_once()
|
||||
mock_forward.assert_called_once()
|
||||
mock_yandex.assert_not_called()
|
||||
Loading…
Add table
Reference in a new issue