diff --git a/tradein-mvp/backend/app/services/geocoder.py b/tradein-mvp/backend/app/services/geocoder.py index 5d2bd70b..42eaf764 100644 --- a/tradein-mvp/backend/app/services/geocoder.py +++ b/tradein-mvp/backend/app/services/geocoder.py @@ -691,6 +691,62 @@ def _cadastral_house_match(db: Session, street: str, house: str) -> GeocodeSugge ) +_RE_HOUSE_INTERNAL_SPACES = re.compile(r"\s+") + + +def _normalize_geoportal_house(house: str) -> str: + """house_norm как в ekb_geoportal_buildings: lower + удаление внутренних пробелов. + + «7 б» → «7б». ДОЛЖНО совпадать с app.tasks.ekb_geoportal_ingest.normalize_house. + """ + return _RE_HOUSE_INTERNAL_SPACES.sub("", house.strip().lower()) + + +def _geoportal_house_match(db: Session, street: str, house: str) -> GeocodeSuggestion | None: + """Точный матч по реестру зданий ЕКБ (городской геопортал, ekb_geoportal_buildings). + + Полнее чем NSPD cad_buildings (~70% зданий ЕКБ отсутствуют в NSPD) — поэтому это + ПЕРВЫЙ локальный tier геокодера, до cad_buildings. + + Нормализация: street_norm = lower(trim), house_norm = lower без внутренних пробелов + — ровно как наполняет лоадер. Параметры идут только bound-param'ами (без инъекций). + """ + street_norm = street.strip().lower() + house_norm = _normalize_geoportal_house(house) + if not street_norm or not house_norm: + return None + try: + row = db.execute( + text(""" + SELECT street, house, lat, lon + FROM ekb_geoportal_buildings + WHERE street_norm = lower(trim(CAST(:street AS text))) + AND house_norm = CAST(:house AS text) + LIMIT 1 + """), + {"street": street_norm, "house": house_norm}, + ).first() + except Exception: + logger.warning( + "geoportal house match failed for street=%r house=%r", + street, + house, + exc_info=True, + ) + return None + + if row is None: + return None + label = f"{row.street}, {row.house}" + return GeocodeSuggestion( + label=label, + full_address=label, + lat=float(row.lat), + lon=float(row.lon), + kind="house", + ) + + def _cadastral_reverse_sync(db: Session, lat: float, lon: float, radius_m: int = 200) -> str | None: """Reverse lookup via gendesign_cad_buildings FDW. @@ -810,11 +866,38 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: logger.info("geocode cache hit: %s", addr_norm) return cached - # 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API) - # 2a. Anchored house-match: парсим street+house → точный матч по дом-маркеру. - # Это primary cadastral путь — raw-ILIKE по полному readable_address давал 0 hits - # для «Серова 27» / DaData-форм (литеральная подстрока не совпадает). + # 2. Локальные источники по street+house (без внешнего API). parsed = _parse_street_house(address.strip()) + + # 2a. Геопортал ЕКБ — ПЕРВЫЙ локальный tier (полнее cad_buildings ~на 70%). + if parsed is not None: + street, house = parsed + try: + hit = await asyncio.to_thread(_geoportal_house_match, db, street, house) + except Exception: + logger.warning("geoportal house-match raised — fall through", exc_info=True) + hit = None + if hit is not None: + result = GeocodeResult( + lat=hit.lat, + lon=hit.lon, + full_address=hit.full_address, + provider="cache", + confidence="exact", + ) + await asyncio.to_thread(_cache_put, db, addr_norm, result) + logger.info( + "geocode geoportal house-match: %s → (%.5f, %.5f)", + addr_norm, + result.lat, + result.lon, + ) + return result + + # 2c. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API) + # Anchored house-match: парсим street+house → точный матч по дом-маркеру. + # raw-ILIKE по полному readable_address давал 0 hits для «Серова 27» / DaData-форм + # (литеральная подстрока не совпадает). if parsed is not None: street, house = parsed hit = await asyncio.to_thread(_cadastral_house_match, db, street, house) @@ -835,7 +918,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None: ) return result - # 2b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм) + # 2d. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм) cad_suggestions = await asyncio.to_thread(_cadastral_forward_sync, db, address.strip(), limit=1) if cad_suggestions: s = cad_suggestions[0] diff --git a/tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py b/tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py new file mode 100644 index 00000000..74667b51 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py @@ -0,0 +1,233 @@ +"""Клиент городского геопортала Екатеринбурга (геопортал.екатеринбург.рф). + +Открытый GeoServer WFS, без авторизации. Слой `portal:portal_geo_topo_build` — +футпринты зданий ЕКБ (намного полнее чем NSPD cad_buildings, который пропускает +~70% зданий города). + +ВАЖНО: сервер отдаёт 403 на дефолтный User-Agent curl/httpx. Нужны браузерные +заголовки (UA + Referer + Accept) — тогда 200. + +Геометрия `geoloc` в EPSG:4326 (MultiPolygon footprint). BBOX axis order = +lon_min,lat_min,lon_max,lat_max. Сервер режет каждый ответ на 20000 features — +лоадер тайлит мелко, чтобы не упереться. + +Центроид считается чистым Python'ом (area-weighted, без shapely — shapely не входит +в зависимости tradein-mvp backend). + +Используется лоадером app.tasks.ekb_geoportal_ingest. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass + +import httpx + +logger = logging.getLogger(__name__) + +# Punycode-хост геопортал.екатеринбург.рф +GEOPORTAL_HOST = "https://xn--80afgznagjs.xn--80acgfbsl1azdqr.xn--p1ai" +WFS_PATH = "/api/gis/ows/3/portal/wfs" +BUILD_LAYER = "portal:portal_geo_topo_build" + +# Браузерные заголовки — без них хост отдаёт 403. +_BROWSER_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/148.0.0.0 Safari/537.36" + ), + "Referer": f"{GEOPORTAL_HOST}/", + "Accept": "application/json", +} + + +@dataclass(frozen=True, slots=True) +class TopoBuilding: + """Одна часть-фича здания из слоя portal_geo_topo_build. + + Здание может быть разбито на несколько part-features (одинаковые street/house, + разные key) — лоадер группирует их по (street_norm, house_norm). + """ + + street: str + house: str + floors: int | None + purpose: str | None + material: str | None + key: int | None + lat: float + lon: float + + +def _parse_int(value: object) -> int | None: + if value is None: + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def _parse_str(value: object) -> str | None: + if value is None: + return None + s = str(value).strip() + return s or None + + +def _ring_centroid(ring: list) -> tuple[float, float, float] | None: + """Area-weighted centroid of one linear ring → (cx, cy, signed_area2). + + signed_area2 = 2× signed area (используется как вес при усреднении колец/полигонов). + Возвращает None если кольцо вырождено (нулевая площадь). + """ + n = len(ring) + if n < 3: + return None + a2 = 0.0 # 2× signed area + cx = 0.0 + cy = 0.0 + for i in range(n): + x0, y0 = ring[i][0], ring[i][1] + x1, y1 = ring[(i + 1) % n][0], ring[(i + 1) % n][1] + cross = x0 * y1 - x1 * y0 + a2 += cross + cx += (x0 + x1) * cross + cy += (y0 + y1) * cross + if a2 == 0.0: + return None + cx /= 3.0 * a2 + cy /= 3.0 * a2 + return (cx, cy, a2) + + +def _multipolygon_centroid(coords: list) -> tuple[float, float] | None: + """Центроид GeoJSON MultiPolygon/Polygon-частей: усреднение по |площади| внешних колец. + + coords — список полигонов, каждый = [outer_ring, hole1, ...]. Берём только + внешнее кольцо каждого полигона (дыр у footprint'ов зданий практически нет, + их вклад в положение центра пренебрежим). Возвращает (lon, lat) или None. + """ + sum_w = 0.0 + sum_x = 0.0 + sum_y = 0.0 + for polygon in coords: + if not polygon: + continue + outer = polygon[0] + c = _ring_centroid(outer) + if c is None: + continue + cx, cy, a2 = c + w = abs(a2) + sum_w += w + sum_x += cx * w + sum_y += cy * w + if sum_w == 0.0: + return None + return (sum_x / sum_w, sum_y / sum_w) + + +def _geometry_centroid(geom: dict) -> tuple[float, float] | None: + """Центроид GeoJSON geometry (MultiPolygon или Polygon) → (lon, lat) или None.""" + gtype = geom.get("type") + coords = geom.get("coordinates") + if not coords: + return None + if gtype == "MultiPolygon": + return _multipolygon_centroid(coords) + if gtype == "Polygon": + # Polygon coordinates = [outer_ring, hole, ...] → обернём в один полигон. + return _multipolygon_centroid([coords]) + return None + + +def _feature_to_building(feature: dict) -> TopoBuilding | None: + """GeoJSON Feature → TopoBuilding. None если нет улицы/дома или геометрии.""" + props = feature.get("properties") or {} + street = _parse_str(props.get("topo_street")) + house = _parse_str(props.get("topo_num_house")) + if not street or not house: + return None + + geom = feature.get("geometry") + if not isinstance(geom, dict): + return None + try: + centroid = _geometry_centroid(geom) + except Exception: + logger.warning("geoportal: bad geometry for street=%r house=%r", street, house) + return None + if centroid is None: + return None + lon, lat = centroid + + return TopoBuilding( + street=street, + house=house, + floors=_parse_int(props.get("topo_floor")), + purpose=_parse_str(props.get("topo_purpose")), + material=_parse_str(props.get("topo_material")), + key=_parse_int(props.get("key")), + lat=lat, + lon=lon, + ) + + +def parse_feature_collection(payload: dict) -> list[TopoBuilding]: + """GeoJSON FeatureCollection → list[TopoBuilding]. Пропускает невалидные.""" + features = payload.get("features") + if not isinstance(features, list): + return [] + out: list[TopoBuilding] = [] + for feature in features: + if not isinstance(feature, dict): + continue + building = _feature_to_building(feature) + if building is not None: + out.append(building) + return out + + +class EkbGeoportalClient: + """Async WFS-клиент геопортала ЕКБ. Один клиент переиспользуется лоадером.""" + + def __init__(self, *, timeout: float = 30.0) -> None: + self._timeout = timeout + + async def fetch_topo_buildings( + self, + bbox: tuple[float, float, float, float], + *, + count: int = 20000, + ) -> list[TopoBuilding]: + """Запрашивает здания в bbox (lon_min, lat_min, lon_max, lat_max). + + Graceful: на non-200/parse-error логирует warning и возвращает []. + """ + lon_min, lat_min, lon_max, lat_max = bbox + cql = f"BBOX(geoloc,{lon_min},{lat_min},{lon_max},{lat_max})" + params = { + "service": "WFS", + "version": "2.0.0", + "request": "GetFeature", + "typeNames": BUILD_LAYER, + "outputFormat": "application/json", + "CQL_FILTER": cql, + "count": str(count), + } + url = f"{GEOPORTAL_HOST}{WFS_PATH}" + try: + async with httpx.AsyncClient(timeout=self._timeout, headers=_BROWSER_HEADERS) as client: + resp = await client.get(url, params=params) + if resp.status_code != 200: + logger.warning("geoportal WFS → HTTP %d for bbox=%s", resp.status_code, bbox) + return [] + payload = resp.json() + except Exception as exc: + logger.warning("geoportal WFS fetch/parse failed for bbox=%s: %s", bbox, exc) + return [] + + return parse_feature_collection(payload) diff --git a/tradein-mvp/backend/app/tasks/ekb_geoportal_ingest.py b/tradein-mvp/backend/app/tasks/ekb_geoportal_ingest.py new file mode 100644 index 00000000..413e78b9 --- /dev/null +++ b/tradein-mvp/backend/app/tasks/ekb_geoportal_ingest.py @@ -0,0 +1,273 @@ +"""Лоадер реестра зданий ЕКБ из городского геопортала в ekb_geoportal_buildings. + +Запуск: `python -m app.tasks.ekb_geoportal_ingest` + +Тайлит bbox ЕКБ (lon 60.45–60.85, lat 56.70–56.95) на мелкую сетку (~250-300 м), +чтобы каждый тайл укладывался в server-cap 20000 features. Для каждого тайла +запрашивает слой portal_geo_topo_build, группирует part-features по +(street_norm, house_norm), считает центроид и UPSERT'ит в ekb_geoportal_buildings. + +Нормализация ОБЯЗАНА совпадать с geocoder._geoportal_house_match: + street_norm = lower(trim(street)) + house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б») + +Configurable через argv/env (для частичного/тестового прогона): + --lon-min/--lon-max/--lat-min/--lat-max или env BBOX="lon_min,lat_min,lon_max,lat_max" +По умолчанию — полный ЕКБ. +""" + +from __future__ import annotations + +import argparse +import asyncio +import logging +import os +import re + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import SessionLocal +from app.services.scrapers.ekb_geoportal_client import ( + EkbGeoportalClient, + TopoBuilding, +) + +logger = logging.getLogger(__name__) + +# Полный bbox ЕКБ. +EKB_LON_MIN = 60.45 +EKB_LON_MAX = 60.85 +EKB_LAT_MIN = 56.70 +EKB_LAT_MAX = 56.95 + +# Шаг сетки: ~0.004° lon × ~0.003° lat (≈250-300 м). Центральный тайл 0.005×0.004 +# давал 6825 зданий → 0.004×0.003 заведомо < server-cap 20000. +TILE_LON_STEP = 0.004 +TILE_LAT_STEP = 0.003 + +# Пауза между запросами — гос-портал, вежливый rate-limit. +REQUEST_SLEEP_SEC = 0.1 + +# Каждые N тайлов — лог прогресса + commit. +PROGRESS_EVERY = 50 + +_INTERNAL_SPACES = re.compile(r"\s+") + + +def normalize_street(street: str) -> str: + """street_norm = lower(trim(street)).""" + return street.strip().lower() + + +def normalize_house(house: str) -> str: + """house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»).""" + return _INTERNAL_SPACES.sub("", house.strip().lower()) + + +def _tile_grid( + lon_min: float, lat_min: float, lon_max: float, lat_max: float +) -> list[tuple[float, float, float, float]]: + """Сетка тайлов (lon_min, lat_min, lon_max, lat_max), шаг TILE_LON/LAT_STEP.""" + tiles: list[tuple[float, float, float, float]] = [] + lon = lon_min + while lon < lon_max: + lon_hi = min(lon + TILE_LON_STEP, lon_max) + lat = lat_min + while lat < lat_max: + lat_hi = min(lat + TILE_LAT_STEP, lat_max) + tiles.append((lon, lat, lon_hi, lat_hi)) + lat = lat_hi + lon = lon_hi + return tiles + + +def _group_buildings( + buildings: list[TopoBuilding], +) -> dict[tuple[str, str], dict]: + """Группирует part-features по (street_norm, house_norm). + + centroid = среднее центроидов частей; floors = max; purpose/material = первый + непустой; source_keys = собранные key. + """ + grouped: dict[tuple[str, str], dict] = {} + for b in buildings: + key = (normalize_street(b.street), normalize_house(b.house)) + if key not in grouped: + grouped[key] = { + "street": b.street.strip(), + "house": b.house.strip(), + "lat_sum": 0.0, + "lon_sum": 0.0, + "n": 0, + "floors": None, + "purpose": None, + "material": None, + "source_keys": [], + } + g = grouped[key] + g["lat_sum"] += b.lat + g["lon_sum"] += b.lon + g["n"] += 1 + if b.floors is not None: + g["floors"] = b.floors if g["floors"] is None else max(g["floors"], b.floors) + if g["purpose"] is None and b.purpose is not None: + g["purpose"] = b.purpose + if g["material"] is None and b.material is not None: + g["material"] = b.material + if b.key is not None and b.key not in g["source_keys"]: + g["source_keys"].append(b.key) + return grouped + + +def _upsert_building( + db: Session, + *, + street_norm: str, + house_norm: str, + agg: dict, +) -> None: + """UPSERT одного здания в ekb_geoportal_buildings (ON CONFLICT DO UPDATE).""" + lat = agg["lat_sum"] / agg["n"] + lon = agg["lon_sum"] / agg["n"] + db.execute( + text(""" + INSERT INTO ekb_geoportal_buildings ( + street, house, street_norm, house_norm, + lat, lon, floors, purpose, material, source_keys, updated_at + ) VALUES ( + :street, :house, :street_norm, :house_norm, + CAST(:lat AS double precision), CAST(:lon AS double precision), + CAST(:floors AS integer), :purpose, :material, + CAST(:source_keys AS bigint[]), now() + ) + ON CONFLICT (street_norm, house_norm) DO UPDATE SET + street = EXCLUDED.street, + house = EXCLUDED.house, + lat = EXCLUDED.lat, + lon = EXCLUDED.lon, + floors = EXCLUDED.floors, + purpose = EXCLUDED.purpose, + material = EXCLUDED.material, + source_keys = EXCLUDED.source_keys, + updated_at = now() + """), + { + "street": agg["street"], + "house": agg["house"], + "street_norm": street_norm, + "house_norm": house_norm, + "lat": lat, + "lon": lon, + "floors": agg["floors"], + "purpose": agg["purpose"], + "material": agg["material"], + "source_keys": agg["source_keys"], + }, + ) + + +async def ingest( + db: Session, + *, + lon_min: float, + lat_min: float, + lon_max: float, + lat_max: float, +) -> int: + """Тайлит bbox, тянет здания, UPSERT'ит. Возвращает число upsert'нутых зданий.""" + client = EkbGeoportalClient() + tiles = _tile_grid(lon_min, lat_min, lon_max, lat_max) + total_tiles = len(tiles) + logger.info( + "geoportal ingest: bbox=(%.4f,%.4f,%.4f,%.4f) → %d тайлов", + lon_min, + lat_min, + lon_max, + lat_max, + total_tiles, + ) + + upserted = 0 + for i, tile in enumerate(tiles, start=1): + buildings = await client.fetch_topo_buildings(tile) + if buildings: + grouped = _group_buildings(buildings) + try: + with db.begin_nested(): + for (street_norm, house_norm), agg in grouped.items(): + _upsert_building( + db, + street_norm=street_norm, + house_norm=house_norm, + agg=agg, + ) + upserted += len(grouped) + except Exception: + logger.warning("geoportal ingest: batch failed for tile=%s", tile, exc_info=True) + + await asyncio.sleep(REQUEST_SLEEP_SEC) + + if i % PROGRESS_EVERY == 0 or i == total_tiles: + db.commit() + logger.info( + "geoportal ingest: %d/%d тайлов, upsert'нуто %d зданий, tile=%s", + i, + total_tiles, + upserted, + tile, + ) + + db.commit() + logger.info("geoportal ingest: готово — upsert'нуто %d зданий", upserted) + return upserted + + +def _resolve_bbox(args: argparse.Namespace) -> tuple[float, float, float, float]: + """Разрешает bbox: argv → env BBOX → дефолтный ЕКБ.""" + env_bbox = os.environ.get("BBOX") + if env_bbox: + parts = [float(p) for p in env_bbox.split(",")] + if len(parts) == 4: + return (parts[0], parts[1], parts[2], parts[3]) + logger.warning("BBOX env malformed (%r) — игнорирую", env_bbox) + + return ( + args.lon_min if args.lon_min is not None else EKB_LON_MIN, + args.lat_min if args.lat_min is not None else EKB_LAT_MIN, + args.lon_max if args.lon_max is not None else EKB_LON_MAX, + args.lat_max if args.lat_max is not None else EKB_LAT_MAX, + ) + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s: %(message)s", + ) + parser = argparse.ArgumentParser(description="EKB geoportal building ingest") + parser.add_argument("--lon-min", type=float, default=None) + parser.add_argument("--lon-max", type=float, default=None) + parser.add_argument("--lat-min", type=float, default=None) + parser.add_argument("--lat-max", type=float, default=None) + args = parser.parse_args() + + lon_min, lat_min, lon_max, lat_max = _resolve_bbox(args) + + db = SessionLocal() + try: + asyncio.run( + ingest( + db, + lon_min=lon_min, + lat_min=lat_min, + lon_max=lon_max, + lat_max=lat_max, + ) + ) + finally: + db.close() + + +if __name__ == "__main__": + main() diff --git a/tradein-mvp/backend/data/sql/130_ekb_geoportal_buildings.sql b/tradein-mvp/backend/data/sql/130_ekb_geoportal_buildings.sql new file mode 100644 index 00000000..18422550 --- /dev/null +++ b/tradein-mvp/backend/data/sql/130_ekb_geoportal_buildings.sql @@ -0,0 +1,38 @@ +-- 130_ekb_geoportal_buildings.sql +-- Полный реестр зданий ЕКБ из городского геопортала (геопортал.екатеринбург.рф, +-- слой WFS portal:portal_geo_topo_build). Намного полнее чем NSPD cad_buildings +-- (который пропускает ~70% зданий ЕКБ) — используется как ПЕРВЫЙ локальный tier +-- геокодера для разрешения «улица + дом» → координаты. +-- +-- Наполняется лоадером app.tasks.ekb_geoportal_ingest (python -m). UPSERT по +-- (street_norm, house_norm); координаты = центроид футпринта здания. +-- +-- Нормализация (ДОЛЖНА совпадать с matcher'ом в geocoder._geoportal_house_match): +-- street_norm = lower(trim(street)) +-- house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б») +-- +-- Idempotent: CREATE TABLE IF NOT EXISTS + CREATE INDEX IF NOT EXISTS, +-- безопасно при повторном применении. + +BEGIN; + +CREATE TABLE IF NOT EXISTS ekb_geoportal_buildings ( + id bigserial PRIMARY KEY, + street text NOT NULL, + house text NOT NULL, + street_norm text NOT NULL, + house_norm text NOT NULL, + lat double precision NOT NULL, + lon double precision NOT NULL, + floors integer, + purpose text, + material text, + source_keys bigint[], + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (street_norm, house_norm) +); + +CREATE INDEX IF NOT EXISTS ix_ekb_geoportal_buildings_street_house + ON ekb_geoportal_buildings (street_norm, house_norm); + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py b/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py new file mode 100644 index 00000000..6496c2c2 --- /dev/null +++ b/tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py @@ -0,0 +1,329 @@ +"""Unit tests for the EKB geoportal building ingest pipeline. + +Covers: +- ekb_geoportal_client: GeoJSON FeatureCollection → TopoBuilding parsing + centroid, + skip of features without street/house, graceful empty on bad input. +- ekb_geoportal_ingest: house normalization ("7 б" → "7б"), part-feature grouping, + tile grid bounds. +- geocoder._geoportal_house_match + geocode() wiring: mock db row → GeocodeResult + (provider="cache", confidence="exact"); parse-fail / no-row → falls through. + +All network + DB mocked. +""" + +from __future__ import annotations + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +_wp_mock = MagicMock() +sys.modules.setdefault("weasyprint", _wp_mock) + + +from app.services.scrapers.ekb_geoportal_client import ( # noqa: E402 + TopoBuilding, + parse_feature_collection, +) +from app.tasks.ekb_geoportal_ingest import ( # noqa: E402 + _group_buildings, + _tile_grid, + normalize_house, + normalize_street, +) + +# ── client: GeoJSON parsing + centroid ─────────────────────────────────────── + + +def _multipolygon(coords: list) -> dict: + return {"type": "MultiPolygon", "coordinates": coords} + + +# A unit square [0,0]-[2,2] → centroid (1, 1) in (lon, lat). +_SQUARE = [[[[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 2.0], [0.0, 0.0]]]] + + +def test_parse_feature_collection_basic() -> None: + fc = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": _multipolygon(_SQUARE), + "properties": { + "topo_street": "Космонавтов", + "topo_num_house": "7б", + "topo_floor": 9, + "topo_purpose": "жилое", + "topo_material": "3", + "key": 12345, + }, + } + ], + } + out = parse_feature_collection(fc) + assert len(out) == 1 + b = out[0] + assert isinstance(b, TopoBuilding) + assert b.street == "Космонавтов" + assert b.house == "7б" + assert b.floors == 9 + assert b.purpose == "жилое" + assert b.material == "3" + assert b.key == 12345 + # centroid of the [0,0]-[2,2] square = (1, 1): lon=x, lat=y + assert b.lon == pytest.approx(1.0) + assert b.lat == pytest.approx(1.0) + + +def test_parse_skips_features_without_street_or_house() -> None: + fc = { + "features": [ + { + "geometry": _multipolygon(_SQUARE), + "properties": {"topo_street": "", "topo_num_house": "7б"}, + }, + { + "geometry": _multipolygon(_SQUARE), + "properties": {"topo_street": "Ленина", "topo_num_house": " "}, + }, + { + "geometry": _multipolygon(_SQUARE), + "properties": {"topo_num_house": "5"}, # no street key + }, + ] + } + assert parse_feature_collection(fc) == [] + + +def test_parse_skips_feature_without_geometry() -> None: + fc = { + "features": [ + { + "geometry": None, + "properties": {"topo_street": "Ленина", "topo_num_house": "5"}, + } + ] + } + assert parse_feature_collection(fc) == [] + + +def test_parse_empty_or_malformed_collection() -> None: + assert parse_feature_collection({}) == [] + assert parse_feature_collection({"features": "nope"}) == [] + assert parse_feature_collection({"features": [None, 42]}) == [] + + +# ── normalization ──────────────────────────────────────────────────────────── + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("7 б", "7б"), + ("7Б", "7б"), + (" 7б ", "7б"), + ("1в/1", "1в/1"), + ("12", "12"), + ("3 А", "3а"), + ], +) +def test_normalize_house(raw: str, expected: str) -> None: + assert normalize_house(raw) == expected + + +def test_normalize_street() -> None: + assert normalize_street(" Космонавтов ") == "космонавтов" + assert normalize_street("8 Марта") == "8 марта" + + +# ── grouping ───────────────────────────────────────────────────────────────── + + +def test_group_buildings_merges_parts() -> None: + parts = [ + TopoBuilding("Космонавтов", "7 б", 9, "жилое", "3", 1, lat=2.0, lon=4.0), + TopoBuilding("Космонавтов", "7б", None, None, None, 2, lat=4.0, lon=6.0), + TopoBuilding("Космонавтов", "7а", 5, None, None, 3, lat=1.0, lon=1.0), + ] + grouped = _group_buildings(parts) + # "7 б" and "7б" normalize to the same house → one merged entry + assert ("космонавтов", "7б") in grouped + assert ("космонавтов", "7а") in grouped + assert len(grouped) == 2 + + merged = grouped[("космонавтов", "7б")] + # centroid = mean of (2,4) and (4,6) + assert merged["lat_sum"] / merged["n"] == pytest.approx(3.0) + assert merged["lon_sum"] / merged["n"] == pytest.approx(5.0) + assert merged["floors"] == 9 # max across parts (other is None) + assert merged["purpose"] == "жилое" + assert merged["material"] == "3" + assert sorted(merged["source_keys"]) == [1, 2] + + +# ── tile grid ──────────────────────────────────────────────────────────────── + + +def test_tile_grid_covers_bbox() -> None: + tiles = _tile_grid(60.45, 56.70, 60.85, 56.95) + assert len(tiles) > 0 + # every tile within bbox bounds + for lon_lo, lat_lo, lon_hi, lat_hi in tiles: + assert 60.45 <= lon_lo < lon_hi <= 60.85 + 1e-9 + assert 56.70 <= lat_lo < lat_hi <= 56.95 + 1e-9 + # first tile anchored at SW corner + assert tiles[0][0] == pytest.approx(60.45) + assert tiles[0][1] == pytest.approx(56.70) + + +# ── geocoder._geoportal_house_match + geocode() wiring ─────────────────────── + + +def test_geoportal_house_match_returns_suggestion() -> None: + from app.services.geocoder import GeocodeSuggestion, _geoportal_house_match + + row = MagicMock() + row.street = "Космонавтов" + row.house = "7б" + row.lat = 56.864 + row.lon = 60.611 + + db = MagicMock() + result = MagicMock() + result.first.return_value = row + db.execute.return_value = result + + hit = _geoportal_house_match(db, "Космонавтов", "7 б") + assert isinstance(hit, GeocodeSuggestion) + assert hit.kind == "house" + assert hit.lat == 56.864 + assert hit.lon == 60.611 + # house param normalized: "7 б" → "7б" + params = db.execute.call_args.args[1] + assert params["house"] == "7б" + assert params["street"] == "космонавтов" + + +def test_geoportal_house_match_none_when_no_row() -> None: + from app.services.geocoder import _geoportal_house_match + + db = MagicMock() + result = MagicMock() + result.first.return_value = None + db.execute.return_value = result + assert _geoportal_house_match(db, "Космонавтов", "7б") is None + + +def test_geoportal_house_match_none_on_db_error() -> None: + from app.services.geocoder import _geoportal_house_match + + db = MagicMock() + db.execute.side_effect = RuntimeError("table missing") + assert _geoportal_house_match(db, "Космонавтов", "7б") is None + + +def test_geoportal_house_match_none_for_empty_house() -> None: + from app.services.geocoder import _geoportal_house_match + + db = MagicMock() + assert _geoportal_house_match(db, "Космонавтов", " ") is None + db.execute.assert_not_called() + + +async def test_geocode_uses_geoportal_first() -> None: + """Parse + geoportal hit → GeocodeResult(provider=cache, exact); cad NOT called.""" + from app.services.geocoder import GeocodeSuggestion, geocode + + db = MagicMock() + hit = GeocodeSuggestion( + label="Космонавтов, 7б", + full_address="Космонавтов, 7б", + lat=56.864, + lon=60.611, + kind="house", + ) + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch("app.services.geocoder._geoportal_house_match", return_value=hit) as mock_geo, + patch("app.services.geocoder._cadastral_house_match") as mock_cad, + patch("app.services.geocoder._cadastral_forward_sync") as mock_forward, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex, + ): + result = await geocode("Космонавтов 7б", db) + + assert result is not None + assert result.provider == "cache" + assert result.confidence == "exact" + assert result.lat == 56.864 + assert result.lon == 60.611 + mock_geo.assert_called_once() + # geoportal hit short-circuits everything downstream + mock_cad.assert_not_called() + mock_forward.assert_not_called() + mock_yandex.assert_not_called() + + +async def test_geocode_falls_through_to_cadastral_when_geoportal_misses() -> None: + """Geoportal None → cadastral house-match IS called next.""" + from app.services.geocoder import GeocodeSuggestion, geocode + + db = MagicMock() + cad_hit = GeocodeSuggestion( + label="ул. Серова, д. 27", + full_address="ул. Серова, д. 27, Екатеринбург", + lat=56.81188, + lon=60.59739, + kind="house", + ) + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch("app.services.geocoder._geoportal_house_match", return_value=None) as mock_geo, + patch("app.services.geocoder._cadastral_house_match", return_value=cad_hit) as mock_cad, + patch("app.services.geocoder._cadastral_forward_sync") as mock_forward, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex, + ): + result = await geocode("Серова 27", db) + + assert result is not None + assert result.lat == 56.81188 + mock_geo.assert_called_once() + mock_cad.assert_called_once() + mock_forward.assert_not_called() + mock_yandex.assert_not_called() + + +async def test_geocode_skips_geoportal_when_parse_fails() -> None: + """Unparseable address → geoportal NOT called.""" + from app.services.geocoder import geocode + + db = MagicMock() + + with ( + patch("app.services.geocoder._cache_get", return_value=None), + patch("app.services.geocoder._geoportal_house_match") as mock_geo, + patch("app.services.geocoder._cadastral_house_match") as mock_cad, + patch("app.services.geocoder._cadastral_forward_sync", return_value=[]) as mock_forward, + patch("app.services.geocoder._cache_put"), + patch("app.services.geocoder.settings") as mock_settings, + patch( + "app.services.geocoder._nominatim_lookup", + new_callable=AsyncMock, + return_value=None, + ), + ): + mock_settings.yandex_geocoder_api_key = None + result = await geocode("полный мусор без дома", db) + + assert result is None + mock_geo.assert_not_called() + mock_cad.assert_not_called() + mock_forward.assert_called_once() diff --git a/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py b/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py index 5bc6db9a..33481860 100644 --- a/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py +++ b/tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py @@ -178,6 +178,7 @@ async def test_geocode_uses_house_match_before_legacy_forward() -> None: with ( patch("app.services.geocoder._cache_get", return_value=None), + patch("app.services.geocoder._geoportal_house_match", return_value=None), patch( "app.services.geocoder._cadastral_house_match", return_value=hit, @@ -213,6 +214,7 @@ async def test_geocode_falls_back_to_legacy_forward_when_house_match_misses() -> with ( patch("app.services.geocoder._cache_get", return_value=None), + patch("app.services.geocoder._geoportal_house_match", return_value=None), patch( "app.services.geocoder._cadastral_house_match", return_value=None,