tradein-mvp/backend/app/services/geocoder.py

@@ -691,62 +691,6 @@ def _cadastral_house_match(db: Session, street: str, house: str) -> GeocodeSugge
     )
 
 
-_RE_HOUSE_INTERNAL_SPACES = re.compile(r"\s+")
-
-
-def _normalize_geoportal_house(house: str) -> str:
-    """house_norm как в ekb_geoportal_buildings: lower + удаление внутренних пробелов.
-
-    «7 б» → «7б». ДОЛЖНО совпадать с app.tasks.ekb_geoportal_ingest.normalize_house.
-    """
-    return _RE_HOUSE_INTERNAL_SPACES.sub("", house.strip().lower())
-
-
-def _geoportal_house_match(db: Session, street: str, house: str) -> GeocodeSuggestion | None:
-    """Точный матч по реестру зданий ЕКБ (городской геопортал, ekb_geoportal_buildings).
-
-    Полнее чем NSPD cad_buildings (~70% зданий ЕКБ отсутствуют в NSPD) — поэтому это
-    ПЕРВЫЙ локальный tier геокодера, до cad_buildings.
-
-    Нормализация: street_norm = lower(trim), house_norm = lower без внутренних пробелов
-    — ровно как наполняет лоадер. Параметры идут только bound-param'ами (без инъекций).
-    """
-    street_norm = street.strip().lower()
-    house_norm = _normalize_geoportal_house(house)
-    if not street_norm or not house_norm:
-        return None
-    try:
-        row = db.execute(
-            text("""
-                SELECT street, house, lat, lon
-                FROM ekb_geoportal_buildings
-                WHERE street_norm = lower(trim(CAST(:street AS text)))
-                  AND house_norm = CAST(:house AS text)
-                LIMIT 1
-            """),
-            {"street": street_norm, "house": house_norm},
-        ).first()
-    except Exception:
-        logger.warning(
-            "geoportal house match failed for street=%r house=%r",
-            street,
-            house,
-            exc_info=True,
-        )
-        return None
-
-    if row is None:
-        return None
-    label = f"{row.street}, {row.house}"
-    return GeocodeSuggestion(
-        label=label,
-        full_address=label,
-        lat=float(row.lat),
-        lon=float(row.lon),
-        kind="house",
-    )
-
-
 def _cadastral_reverse_sync(db: Session, lat: float, lon: float, radius_m: int = 200) -> str | None:
     """Reverse lookup via gendesign_cad_buildings FDW.
 
@@ -866,38 +810,11 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
         logger.info("geocode cache hit: %s", addr_norm)
         return cached
 
-    # 2. Локальные источники по street+house (без внешнего API).
+    # 2. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
+    # 2a. Anchored house-match: парсим street+house → точный матч по дом-маркеру.
+    # Это primary cadastral путь — raw-ILIKE по полному readable_address давал 0 hits
+    # для «Серова 27» / DaData-форм (литеральная подстрока не совпадает).
     parsed = _parse_street_house(address.strip())
-
-    # 2a. Геопортал ЕКБ — ПЕРВЫЙ локальный tier (полнее cad_buildings ~на 70%).
-    if parsed is not None:
-        street, house = parsed
-        try:
-            hit = await asyncio.to_thread(_geoportal_house_match, db, street, house)
-        except Exception:
-            logger.warning("geoportal house-match raised — fall through", exc_info=True)
-            hit = None
-        if hit is not None:
-            result = GeocodeResult(
-                lat=hit.lat,
-                lon=hit.lon,
-                full_address=hit.full_address,
-                provider="cache",
-                confidence="exact",
-            )
-            await asyncio.to_thread(_cache_put, db, addr_norm, result)
-            logger.info(
-                "geocode geoportal house-match: %s → (%.5f, %.5f)",
-                addr_norm,
-                result.lat,
-                result.lon,
-            )
-            return result
-
-    # 2c. Cadastral FDW (прямой запрос к gendesign_cad_buildings — без внешнего API)
-    # Anchored house-match: парсим street+house → точный матч по дом-маркеру.
-    # raw-ILIKE по полному readable_address давал 0 hits для «Серова 27» / DaData-форм
-    # (литеральная подстрока не совпадает).
     if parsed is not None:
         street, house = parsed
         hit = await asyncio.to_thread(_cadastral_house_match, db, street, house)
@@ -918,7 +835,7 @@ async def geocode(address: str, db: Session) -> GeocodeResult | None:
             )
             return result
 
-    # 2d. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм)
+    # 2b. Fallback: legacy raw-ILIKE forward search (для нераспарсенных форм)
     cad_suggestions = await asyncio.to_thread(_cadastral_forward_sync, db, address.strip(), limit=1)
     if cad_suggestions:
         s = cad_suggestions[0]

tradein-mvp/backend/app/services/scrapers/ekb_geoportal_client.py

@@ -1,233 +0,0 @@
-"""Клиент городского геопортала Екатеринбурга (геопортал.екатеринбург.рф).
-
-Открытый GeoServer WFS, без авторизации. Слой `portal:portal_geo_topo_build` —
-футпринты зданий ЕКБ (намного полнее чем NSPD cad_buildings, который пропускает
-~70% зданий города).
-
-ВАЖНО: сервер отдаёт 403 на дефолтный User-Agent curl/httpx. Нужны браузерные
-заголовки (UA + Referer + Accept) — тогда 200.
-
-Геометрия `geoloc` в EPSG:4326 (MultiPolygon footprint). BBOX axis order =
-lon_min,lat_min,lon_max,lat_max. Сервер режет каждый ответ на 20000 features —
-лоадер тайлит мелко, чтобы не упереться.
-
-Центроид считается чистым Python'ом (area-weighted, без shapely — shapely не входит
-в зависимости tradein-mvp backend).
-
-Используется лоадером app.tasks.ekb_geoportal_ingest.
-"""
-
-from __future__ import annotations
-
-import logging
-from dataclasses import dataclass
-
-import httpx
-
-logger = logging.getLogger(__name__)
-
-# Punycode-хост геопортал.екатеринбург.рф
-GEOPORTAL_HOST = "https://xn--80afgznagjs.xn--80acgfbsl1azdqr.xn--p1ai"
-WFS_PATH = "/api/gis/ows/3/portal/wfs"
-BUILD_LAYER = "portal:portal_geo_topo_build"
-
-# Браузерные заголовки — без них хост отдаёт 403.
-_BROWSER_HEADERS = {
-    "User-Agent": (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/148.0.0.0 Safari/537.36"
-    ),
-    "Referer": f"{GEOPORTAL_HOST}/",
-    "Accept": "application/json",
-}
-
-
-@dataclass(frozen=True, slots=True)
-class TopoBuilding:
-    """Одна часть-фича здания из слоя portal_geo_topo_build.
-
-    Здание может быть разбито на несколько part-features (одинаковые street/house,
-    разные key) — лоадер группирует их по (street_norm, house_norm).
-    """
-
-    street: str
-    house: str
-    floors: int | None
-    purpose: str | None
-    material: str | None
-    key: int | None
-    lat: float
-    lon: float
-
-
-def _parse_int(value: object) -> int | None:
-    if value is None:
-        return None
-    try:
-        return int(value)
-    except (ValueError, TypeError):
-        return None
-
-
-def _parse_str(value: object) -> str | None:
-    if value is None:
-        return None
-    s = str(value).strip()
-    return s or None
-
-
-def _ring_centroid(ring: list) -> tuple[float, float, float] | None:
-    """Area-weighted centroid of one linear ring → (cx, cy, signed_area2).
-
-    signed_area2 = 2× signed area (используется как вес при усреднении колец/полигонов).
-    Возвращает None если кольцо вырождено (нулевая площадь).
-    """
-    n = len(ring)
-    if n < 3:
-        return None
-    a2 = 0.0  # 2× signed area
-    cx = 0.0
-    cy = 0.0
-    for i in range(n):
-        x0, y0 = ring[i][0], ring[i][1]
-        x1, y1 = ring[(i + 1) % n][0], ring[(i + 1) % n][1]
-        cross = x0 * y1 - x1 * y0
-        a2 += cross
-        cx += (x0 + x1) * cross
-        cy += (y0 + y1) * cross
-    if a2 == 0.0:
-        return None
-    cx /= 3.0 * a2
-    cy /= 3.0 * a2
-    return (cx, cy, a2)
-
-
-def _multipolygon_centroid(coords: list) -> tuple[float, float] | None:
-    """Центроид GeoJSON MultiPolygon/Polygon-частей: усреднение по |площади| внешних колец.
-
-    coords — список полигонов, каждый = [outer_ring, hole1, ...]. Берём только
-    внешнее кольцо каждого полигона (дыр у footprint'ов зданий практически нет,
-    их вклад в положение центра пренебрежим). Возвращает (lon, lat) или None.
-    """
-    sum_w = 0.0
-    sum_x = 0.0
-    sum_y = 0.0
-    for polygon in coords:
-        if not polygon:
-            continue
-        outer = polygon[0]
-        c = _ring_centroid(outer)
-        if c is None:
-            continue
-        cx, cy, a2 = c
-        w = abs(a2)
-        sum_w += w
-        sum_x += cx * w
-        sum_y += cy * w
-    if sum_w == 0.0:
-        return None
-    return (sum_x / sum_w, sum_y / sum_w)
-
-
-def _geometry_centroid(geom: dict) -> tuple[float, float] | None:
-    """Центроид GeoJSON geometry (MultiPolygon или Polygon) → (lon, lat) или None."""
-    gtype = geom.get("type")
-    coords = geom.get("coordinates")
-    if not coords:
-        return None
-    if gtype == "MultiPolygon":
-        return _multipolygon_centroid(coords)
-    if gtype == "Polygon":
-        # Polygon coordinates = [outer_ring, hole, ...] → обернём в один полигон.
-        return _multipolygon_centroid([coords])
-    return None
-
-
-def _feature_to_building(feature: dict) -> TopoBuilding | None:
-    """GeoJSON Feature → TopoBuilding. None если нет улицы/дома или геометрии."""
-    props = feature.get("properties") or {}
-    street = _parse_str(props.get("topo_street"))
-    house = _parse_str(props.get("topo_num_house"))
-    if not street or not house:
-        return None
-
-    geom = feature.get("geometry")
-    if not isinstance(geom, dict):
-        return None
-    try:
-        centroid = _geometry_centroid(geom)
-    except Exception:
-        logger.warning("geoportal: bad geometry for street=%r house=%r", street, house)
-        return None
-    if centroid is None:
-        return None
-    lon, lat = centroid
-
-    return TopoBuilding(
-        street=street,
-        house=house,
-        floors=_parse_int(props.get("topo_floor")),
-        purpose=_parse_str(props.get("topo_purpose")),
-        material=_parse_str(props.get("topo_material")),
-        key=_parse_int(props.get("key")),
-        lat=lat,
-        lon=lon,
-    )
-
-
-def parse_feature_collection(payload: dict) -> list[TopoBuilding]:
-    """GeoJSON FeatureCollection → list[TopoBuilding]. Пропускает невалидные."""
-    features = payload.get("features")
-    if not isinstance(features, list):
-        return []
-    out: list[TopoBuilding] = []
-    for feature in features:
-        if not isinstance(feature, dict):
-            continue
-        building = _feature_to_building(feature)
-        if building is not None:
-            out.append(building)
-    return out
-
-
-class EkbGeoportalClient:
-    """Async WFS-клиент геопортала ЕКБ. Один клиент переиспользуется лоадером."""
-
-    def __init__(self, *, timeout: float = 30.0) -> None:
-        self._timeout = timeout
-
-    async def fetch_topo_buildings(
-        self,
-        bbox: tuple[float, float, float, float],
-        *,
-        count: int = 20000,
-    ) -> list[TopoBuilding]:
-        """Запрашивает здания в bbox (lon_min, lat_min, lon_max, lat_max).
-
-        Graceful: на non-200/parse-error логирует warning и возвращает [].
-        """
-        lon_min, lat_min, lon_max, lat_max = bbox
-        cql = f"BBOX(geoloc,{lon_min},{lat_min},{lon_max},{lat_max})"
-        params = {
-            "service": "WFS",
-            "version": "2.0.0",
-            "request": "GetFeature",
-            "typeNames": BUILD_LAYER,
-            "outputFormat": "application/json",
-            "CQL_FILTER": cql,
-            "count": str(count),
-        }
-        url = f"{GEOPORTAL_HOST}{WFS_PATH}"
-        try:
-            async with httpx.AsyncClient(timeout=self._timeout, headers=_BROWSER_HEADERS) as client:
-                resp = await client.get(url, params=params)
-            if resp.status_code != 200:
-                logger.warning("geoportal WFS → HTTP %d for bbox=%s", resp.status_code, bbox)
-                return []
-            payload = resp.json()
-        except Exception as exc:
-            logger.warning("geoportal WFS fetch/parse failed for bbox=%s: %s", bbox, exc)
-            return []
-
-        return parse_feature_collection(payload)

tradein-mvp/backend/app/tasks/ekb_geoportal_ingest.py

@@ -1,273 +0,0 @@
-"""Лоадер реестра зданий ЕКБ из городского геопортала в ekb_geoportal_buildings.
-
-Запуск: `python -m app.tasks.ekb_geoportal_ingest`
-
-Тайлит bbox ЕКБ (lon 60.45–60.85, lat 56.70–56.95) на мелкую сетку (~250-300 м),
-чтобы каждый тайл укладывался в server-cap 20000 features. Для каждого тайла
-запрашивает слой portal_geo_topo_build, группирует part-features по
-(street_norm, house_norm), считает центроид и UPSERT'ит в ekb_geoportal_buildings.
-
-Нормализация ОБЯЗАНА совпадать с geocoder._geoportal_house_match:
-  street_norm = lower(trim(street))
-  house_norm  = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)
-
-Configurable через argv/env (для частичного/тестового прогона):
-  --lon-min/--lon-max/--lat-min/--lat-max  или env BBOX="lon_min,lat_min,lon_max,lat_max"
-По умолчанию — полный ЕКБ.
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import logging
-import os
-import re
-
-from sqlalchemy import text
-from sqlalchemy.orm import Session
-
-from app.core.db import SessionLocal
-from app.services.scrapers.ekb_geoportal_client import (
-    EkbGeoportalClient,
-    TopoBuilding,
-)
-
-logger = logging.getLogger(__name__)
-
-# Полный bbox ЕКБ.
-EKB_LON_MIN = 60.45
-EKB_LON_MAX = 60.85
-EKB_LAT_MIN = 56.70
-EKB_LAT_MAX = 56.95
-
-# Шаг сетки: ~0.004° lon × ~0.003° lat (≈250-300 м). Центральный тайл 0.005×0.004
-# давал 6825 зданий → 0.004×0.003 заведомо < server-cap 20000.
-TILE_LON_STEP = 0.004
-TILE_LAT_STEP = 0.003
-
-# Пауза между запросами — гос-портал, вежливый rate-limit.
-REQUEST_SLEEP_SEC = 0.1
-
-# Каждые N тайлов — лог прогресса + commit.
-PROGRESS_EVERY = 50
-
-_INTERNAL_SPACES = re.compile(r"\s+")
-
-
-def normalize_street(street: str) -> str:
-    """street_norm = lower(trim(street))."""
-    return street.strip().lower()
-
-
-def normalize_house(house: str) -> str:
-    """house_norm = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)."""
-    return _INTERNAL_SPACES.sub("", house.strip().lower())
-
-
-def _tile_grid(
-    lon_min: float, lat_min: float, lon_max: float, lat_max: float
-) -> list[tuple[float, float, float, float]]:
-    """Сетка тайлов (lon_min, lat_min, lon_max, lat_max), шаг TILE_LON/LAT_STEP."""
-    tiles: list[tuple[float, float, float, float]] = []
-    lon = lon_min
-    while lon < lon_max:
-        lon_hi = min(lon + TILE_LON_STEP, lon_max)
-        lat = lat_min
-        while lat < lat_max:
-            lat_hi = min(lat + TILE_LAT_STEP, lat_max)
-            tiles.append((lon, lat, lon_hi, lat_hi))
-            lat = lat_hi
-        lon = lon_hi
-    return tiles
-
-
-def _group_buildings(
-    buildings: list[TopoBuilding],
-) -> dict[tuple[str, str], dict]:
-    """Группирует part-features по (street_norm, house_norm).
-
-    centroid = среднее центроидов частей; floors = max; purpose/material = первый
-    непустой; source_keys = собранные key.
-    """
-    grouped: dict[tuple[str, str], dict] = {}
-    for b in buildings:
-        key = (normalize_street(b.street), normalize_house(b.house))
-        if key not in grouped:
-            grouped[key] = {
-                "street": b.street.strip(),
-                "house": b.house.strip(),
-                "lat_sum": 0.0,
-                "lon_sum": 0.0,
-                "n": 0,
-                "floors": None,
-                "purpose": None,
-                "material": None,
-                "source_keys": [],
-            }
-        g = grouped[key]
-        g["lat_sum"] += b.lat
-        g["lon_sum"] += b.lon
-        g["n"] += 1
-        if b.floors is not None:
-            g["floors"] = b.floors if g["floors"] is None else max(g["floors"], b.floors)
-        if g["purpose"] is None and b.purpose is not None:
-            g["purpose"] = b.purpose
-        if g["material"] is None and b.material is not None:
-            g["material"] = b.material
-        if b.key is not None and b.key not in g["source_keys"]:
-            g["source_keys"].append(b.key)
-    return grouped
-
-
-def _upsert_building(
-    db: Session,
-    *,
-    street_norm: str,
-    house_norm: str,
-    agg: dict,
-) -> None:
-    """UPSERT одного здания в ekb_geoportal_buildings (ON CONFLICT DO UPDATE)."""
-    lat = agg["lat_sum"] / agg["n"]
-    lon = agg["lon_sum"] / agg["n"]
-    db.execute(
-        text("""
-            INSERT INTO ekb_geoportal_buildings (
-                street, house, street_norm, house_norm,
-                lat, lon, floors, purpose, material, source_keys, updated_at
-            ) VALUES (
-                :street, :house, :street_norm, :house_norm,
-                CAST(:lat AS double precision), CAST(:lon AS double precision),
-                CAST(:floors AS integer), :purpose, :material,
-                CAST(:source_keys AS bigint[]), now()
-            )
-            ON CONFLICT (street_norm, house_norm) DO UPDATE SET
-                street      = EXCLUDED.street,
-                house       = EXCLUDED.house,
-                lat         = EXCLUDED.lat,
-                lon         = EXCLUDED.lon,
-                floors      = EXCLUDED.floors,
-                purpose     = EXCLUDED.purpose,
-                material    = EXCLUDED.material,
-                source_keys = EXCLUDED.source_keys,
-                updated_at  = now()
-        """),
-        {
-            "street": agg["street"],
-            "house": agg["house"],
-            "street_norm": street_norm,
-            "house_norm": house_norm,
-            "lat": lat,
-            "lon": lon,
-            "floors": agg["floors"],
-            "purpose": agg["purpose"],
-            "material": agg["material"],
-            "source_keys": agg["source_keys"],
-        },
-    )
-
-
-async def ingest(
-    db: Session,
-    *,
-    lon_min: float,
-    lat_min: float,
-    lon_max: float,
-    lat_max: float,
-) -> int:
-    """Тайлит bbox, тянет здания, UPSERT'ит. Возвращает число upsert'нутых зданий."""
-    client = EkbGeoportalClient()
-    tiles = _tile_grid(lon_min, lat_min, lon_max, lat_max)
-    total_tiles = len(tiles)
-    logger.info(
-        "geoportal ingest: bbox=(%.4f,%.4f,%.4f,%.4f) → %d тайлов",
-        lon_min,
-        lat_min,
-        lon_max,
-        lat_max,
-        total_tiles,
-    )
-
-    upserted = 0
-    for i, tile in enumerate(tiles, start=1):
-        buildings = await client.fetch_topo_buildings(tile)
-        if buildings:
-            grouped = _group_buildings(buildings)
-            try:
-                with db.begin_nested():
-                    for (street_norm, house_norm), agg in grouped.items():
-                        _upsert_building(
-                            db,
-                            street_norm=street_norm,
-                            house_norm=house_norm,
-                            agg=agg,
-                        )
-                upserted += len(grouped)
-            except Exception:
-                logger.warning("geoportal ingest: batch failed for tile=%s", tile, exc_info=True)
-
-        await asyncio.sleep(REQUEST_SLEEP_SEC)
-
-        if i % PROGRESS_EVERY == 0 or i == total_tiles:
-            db.commit()
-            logger.info(
-                "geoportal ingest: %d/%d тайлов, upsert'нуто %d зданий, tile=%s",
-                i,
-                total_tiles,
-                upserted,
-                tile,
-            )
-
-    db.commit()
-    logger.info("geoportal ingest: готово — upsert'нуто %d зданий", upserted)
-    return upserted
-
-
-def _resolve_bbox(args: argparse.Namespace) -> tuple[float, float, float, float]:
-    """Разрешает bbox: argv → env BBOX → дефолтный ЕКБ."""
-    env_bbox = os.environ.get("BBOX")
-    if env_bbox:
-        parts = [float(p) for p in env_bbox.split(",")]
-        if len(parts) == 4:
-            return (parts[0], parts[1], parts[2], parts[3])
-        logger.warning("BBOX env malformed (%r) — игнорирую", env_bbox)
-
-    return (
-        args.lon_min if args.lon_min is not None else EKB_LON_MIN,
-        args.lat_min if args.lat_min is not None else EKB_LAT_MIN,
-        args.lon_max if args.lon_max is not None else EKB_LON_MAX,
-        args.lat_max if args.lat_max is not None else EKB_LAT_MAX,
-    )
-
-
-def main() -> None:
-    logging.basicConfig(
-        level=logging.INFO,
-        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
-    )
-    parser = argparse.ArgumentParser(description="EKB geoportal building ingest")
-    parser.add_argument("--lon-min", type=float, default=None)
-    parser.add_argument("--lon-max", type=float, default=None)
-    parser.add_argument("--lat-min", type=float, default=None)
-    parser.add_argument("--lat-max", type=float, default=None)
-    args = parser.parse_args()
-
-    lon_min, lat_min, lon_max, lat_max = _resolve_bbox(args)
-
-    db = SessionLocal()
-    try:
-        asyncio.run(
-            ingest(
-                db,
-                lon_min=lon_min,
-                lat_min=lat_min,
-                lon_max=lon_max,
-                lat_max=lat_max,
-            )
-        )
-    finally:
-        db.close()
-
-
-if __name__ == "__main__":
-    main()

tradein-mvp/backend/data/sql/130_ekb_geoportal_buildings.sql

@@ -1,38 +0,0 @@
--- 130_ekb_geoportal_buildings.sql
--- Полный реестр зданий ЕКБ из городского геопортала (геопортал.екатеринбург.рф,
--- слой WFS portal:portal_geo_topo_build). Намного полнее чем NSPD cad_buildings
--- (который пропускает ~70% зданий ЕКБ) — используется как ПЕРВЫЙ локальный tier
--- геокодера для разрешения «улица + дом» → координаты.
---
--- Наполняется лоадером app.tasks.ekb_geoportal_ingest (python -m). UPSERT по
--- (street_norm, house_norm); координаты = центроид футпринта здания.
---
--- Нормализация (ДОЛЖНА совпадать с matcher'ом в geocoder._geoportal_house_match):
---   street_norm = lower(trim(street))
---   house_norm  = lower(house) с удалёнными внутренними пробелами («7 б» → «7б»)
---
--- Idempotent: CREATE TABLE IF NOT EXISTS + CREATE INDEX IF NOT EXISTS,
--- безопасно при повторном применении.
-
-BEGIN;
-
-CREATE TABLE IF NOT EXISTS ekb_geoportal_buildings (
-    id           bigserial PRIMARY KEY,
-    street       text NOT NULL,
-    house        text NOT NULL,
-    street_norm  text NOT NULL,
-    house_norm   text NOT NULL,
-    lat          double precision NOT NULL,
-    lon          double precision NOT NULL,
-    floors       integer,
-    purpose      text,
-    material     text,
-    source_keys  bigint[],
-    updated_at   timestamptz NOT NULL DEFAULT now(),
-    UNIQUE (street_norm, house_norm)
-);
-
-CREATE INDEX IF NOT EXISTS ix_ekb_geoportal_buildings_street_house
-    ON ekb_geoportal_buildings (street_norm, house_norm);
-
-COMMIT;

tradein-mvp/backend/tests/test_ekb_geoportal_ingest.py

@@ -1,329 +0,0 @@
-"""Unit tests for the EKB geoportal building ingest pipeline.
-
-Covers:
-- ekb_geoportal_client: GeoJSON FeatureCollection → TopoBuilding parsing + centroid,
-  skip of features without street/house, graceful empty on bad input.
-- ekb_geoportal_ingest: house normalization ("7 б" → "7б"), part-feature grouping,
-  tile grid bounds.
-- geocoder._geoportal_house_match + geocode() wiring: mock db row → GeocodeResult
-  (provider="cache", confidence="exact"); parse-fail / no-row → falls through.
-
-All network + DB mocked.
-"""
-
-from __future__ import annotations
-
-import os
-import sys
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-_wp_mock = MagicMock()
-sys.modules.setdefault("weasyprint", _wp_mock)
-
-
-from app.services.scrapers.ekb_geoportal_client import (  # noqa: E402
-    TopoBuilding,
-    parse_feature_collection,
-)
-from app.tasks.ekb_geoportal_ingest import (  # noqa: E402
-    _group_buildings,
-    _tile_grid,
-    normalize_house,
-    normalize_street,
-)
-
-# ── client: GeoJSON parsing + centroid ───────────────────────────────────────
-
-
-def _multipolygon(coords: list) -> dict:
-    return {"type": "MultiPolygon", "coordinates": coords}
-
-
-# A unit square [0,0]-[2,2] → centroid (1, 1) in (lon, lat).
-_SQUARE = [[[[0.0, 0.0], [2.0, 0.0], [2.0, 2.0], [0.0, 2.0], [0.0, 0.0]]]]
-
-
-def test_parse_feature_collection_basic() -> None:
-    fc = {
-        "type": "FeatureCollection",
-        "features": [
-            {
-                "type": "Feature",
-                "geometry": _multipolygon(_SQUARE),
-                "properties": {
-                    "topo_street": "Космонавтов",
-                    "topo_num_house": "7б",
-                    "topo_floor": 9,
-                    "topo_purpose": "жилое",
-                    "topo_material": "3",
-                    "key": 12345,
-                },
-            }
-        ],
-    }
-    out = parse_feature_collection(fc)
-    assert len(out) == 1
-    b = out[0]
-    assert isinstance(b, TopoBuilding)
-    assert b.street == "Космонавтов"
-    assert b.house == "7б"
-    assert b.floors == 9
-    assert b.purpose == "жилое"
-    assert b.material == "3"
-    assert b.key == 12345
-    # centroid of the [0,0]-[2,2] square = (1, 1): lon=x, lat=y
-    assert b.lon == pytest.approx(1.0)
-    assert b.lat == pytest.approx(1.0)
-
-
-def test_parse_skips_features_without_street_or_house() -> None:
-    fc = {
-        "features": [
-            {
-                "geometry": _multipolygon(_SQUARE),
-                "properties": {"topo_street": "", "topo_num_house": "7б"},
-            },
-            {
-                "geometry": _multipolygon(_SQUARE),
-                "properties": {"topo_street": "Ленина", "topo_num_house": "  "},
-            },
-            {
-                "geometry": _multipolygon(_SQUARE),
-                "properties": {"topo_num_house": "5"},  # no street key
-            },
-        ]
-    }
-    assert parse_feature_collection(fc) == []
-
-
-def test_parse_skips_feature_without_geometry() -> None:
-    fc = {
-        "features": [
-            {
-                "geometry": None,
-                "properties": {"topo_street": "Ленина", "topo_num_house": "5"},
-            }
-        ]
-    }
-    assert parse_feature_collection(fc) == []
-
-
-def test_parse_empty_or_malformed_collection() -> None:
-    assert parse_feature_collection({}) == []
-    assert parse_feature_collection({"features": "nope"}) == []
-    assert parse_feature_collection({"features": [None, 42]}) == []
-
-
-# ── normalization ────────────────────────────────────────────────────────────
-
-
-@pytest.mark.parametrize(
-    ("raw", "expected"),
-    [
-        ("7 б", "7б"),
-        ("7Б", "7б"),
-        ("  7б  ", "7б"),
-        ("1в/1", "1в/1"),
-        ("12", "12"),
-        ("3 А", "3а"),
-    ],
-)
-def test_normalize_house(raw: str, expected: str) -> None:
-    assert normalize_house(raw) == expected
-
-
-def test_normalize_street() -> None:
-    assert normalize_street("  Космонавтов ") == "космонавтов"
-    assert normalize_street("8 Марта") == "8 марта"
-
-
-# ── grouping ─────────────────────────────────────────────────────────────────
-
-
-def test_group_buildings_merges_parts() -> None:
-    parts = [
-        TopoBuilding("Космонавтов", "7 б", 9, "жилое", "3", 1, lat=2.0, lon=4.0),
-        TopoBuilding("Космонавтов", "7б", None, None, None, 2, lat=4.0, lon=6.0),
-        TopoBuilding("Космонавтов", "7а", 5, None, None, 3, lat=1.0, lon=1.0),
-    ]
-    grouped = _group_buildings(parts)
-    # "7 б" and "7б" normalize to the same house → one merged entry
-    assert ("космонавтов", "7б") in grouped
-    assert ("космонавтов", "7а") in grouped
-    assert len(grouped) == 2
-
-    merged = grouped[("космонавтов", "7б")]
-    # centroid = mean of (2,4) and (4,6)
-    assert merged["lat_sum"] / merged["n"] == pytest.approx(3.0)
-    assert merged["lon_sum"] / merged["n"] == pytest.approx(5.0)
-    assert merged["floors"] == 9  # max across parts (other is None)
-    assert merged["purpose"] == "жилое"
-    assert merged["material"] == "3"
-    assert sorted(merged["source_keys"]) == [1, 2]
-
-
-# ── tile grid ────────────────────────────────────────────────────────────────
-
-
-def test_tile_grid_covers_bbox() -> None:
-    tiles = _tile_grid(60.45, 56.70, 60.85, 56.95)
-    assert len(tiles) > 0
-    # every tile within bbox bounds
-    for lon_lo, lat_lo, lon_hi, lat_hi in tiles:
-        assert 60.45 <= lon_lo < lon_hi <= 60.85 + 1e-9
-        assert 56.70 <= lat_lo < lat_hi <= 56.95 + 1e-9
-    # first tile anchored at SW corner
-    assert tiles[0][0] == pytest.approx(60.45)
-    assert tiles[0][1] == pytest.approx(56.70)
-
-
-# ── geocoder._geoportal_house_match + geocode() wiring ───────────────────────
-
-
-def test_geoportal_house_match_returns_suggestion() -> None:
-    from app.services.geocoder import GeocodeSuggestion, _geoportal_house_match
-
-    row = MagicMock()
-    row.street = "Космонавтов"
-    row.house = "7б"
-    row.lat = 56.864
-    row.lon = 60.611
-
-    db = MagicMock()
-    result = MagicMock()
-    result.first.return_value = row
-    db.execute.return_value = result
-
-    hit = _geoportal_house_match(db, "Космонавтов", "7 б")
-    assert isinstance(hit, GeocodeSuggestion)
-    assert hit.kind == "house"
-    assert hit.lat == 56.864
-    assert hit.lon == 60.611
-    # house param normalized: "7 б" → "7б"
-    params = db.execute.call_args.args[1]
-    assert params["house"] == "7б"
-    assert params["street"] == "космонавтов"
-
-
-def test_geoportal_house_match_none_when_no_row() -> None:
-    from app.services.geocoder import _geoportal_house_match
-
-    db = MagicMock()
-    result = MagicMock()
-    result.first.return_value = None
-    db.execute.return_value = result
-    assert _geoportal_house_match(db, "Космонавтов", "7б") is None
-
-
-def test_geoportal_house_match_none_on_db_error() -> None:
-    from app.services.geocoder import _geoportal_house_match
-
-    db = MagicMock()
-    db.execute.side_effect = RuntimeError("table missing")
-    assert _geoportal_house_match(db, "Космонавтов", "7б") is None
-
-
-def test_geoportal_house_match_none_for_empty_house() -> None:
-    from app.services.geocoder import _geoportal_house_match
-
-    db = MagicMock()
-    assert _geoportal_house_match(db, "Космонавтов", "   ") is None
-    db.execute.assert_not_called()
-
-
-async def test_geocode_uses_geoportal_first() -> None:
-    """Parse + geoportal hit → GeocodeResult(provider=cache, exact); cad NOT called."""
-    from app.services.geocoder import GeocodeSuggestion, geocode
-
-    db = MagicMock()
-    hit = GeocodeSuggestion(
-        label="Космонавтов, 7б",
-        full_address="Космонавтов, 7б",
-        lat=56.864,
-        lon=60.611,
-        kind="house",
-    )
-
-    with (
-        patch("app.services.geocoder._cache_get", return_value=None),
-        patch("app.services.geocoder._geoportal_house_match", return_value=hit) as mock_geo,
-        patch("app.services.geocoder._cadastral_house_match") as mock_cad,
-        patch("app.services.geocoder._cadastral_forward_sync") as mock_forward,
-        patch("app.services.geocoder._cache_put"),
-        patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
-    ):
-        result = await geocode("Космонавтов 7б", db)
-
-    assert result is not None
-    assert result.provider == "cache"
-    assert result.confidence == "exact"
-    assert result.lat == 56.864
-    assert result.lon == 60.611
-    mock_geo.assert_called_once()
-    # geoportal hit short-circuits everything downstream
-    mock_cad.assert_not_called()
-    mock_forward.assert_not_called()
-    mock_yandex.assert_not_called()
-
-
-async def test_geocode_falls_through_to_cadastral_when_geoportal_misses() -> None:
-    """Geoportal None → cadastral house-match IS called next."""
-    from app.services.geocoder import GeocodeSuggestion, geocode
-
-    db = MagicMock()
-    cad_hit = GeocodeSuggestion(
-        label="ул. Серова, д. 27",
-        full_address="ул. Серова, д. 27, Екатеринбург",
-        lat=56.81188,
-        lon=60.59739,
-        kind="house",
-    )
-
-    with (
-        patch("app.services.geocoder._cache_get", return_value=None),
-        patch("app.services.geocoder._geoportal_house_match", return_value=None) as mock_geo,
-        patch("app.services.geocoder._cadastral_house_match", return_value=cad_hit) as mock_cad,
-        patch("app.services.geocoder._cadastral_forward_sync") as mock_forward,
-        patch("app.services.geocoder._cache_put"),
-        patch("app.services.geocoder._yandex_lookup", new_callable=AsyncMock) as mock_yandex,
-    ):
-        result = await geocode("Серова 27", db)
-
-    assert result is not None
-    assert result.lat == 56.81188
-    mock_geo.assert_called_once()
-    mock_cad.assert_called_once()
-    mock_forward.assert_not_called()
-    mock_yandex.assert_not_called()
-
-
-async def test_geocode_skips_geoportal_when_parse_fails() -> None:
-    """Unparseable address → geoportal NOT called."""
-    from app.services.geocoder import geocode
-
-    db = MagicMock()
-
-    with (
-        patch("app.services.geocoder._cache_get", return_value=None),
-        patch("app.services.geocoder._geoportal_house_match") as mock_geo,
-        patch("app.services.geocoder._cadastral_house_match") as mock_cad,
-        patch("app.services.geocoder._cadastral_forward_sync", return_value=[]) as mock_forward,
-        patch("app.services.geocoder._cache_put"),
-        patch("app.services.geocoder.settings") as mock_settings,
-        patch(
-            "app.services.geocoder._nominatim_lookup",
-            new_callable=AsyncMock,
-            return_value=None,
-        ),
-    ):
-        mock_settings.yandex_geocoder_api_key = None
-        result = await geocode("полный мусор без дома", db)
-
-    assert result is None
-    mock_geo.assert_not_called()
-    mock_cad.assert_not_called()
-    mock_forward.assert_called_once()

tradein-mvp/backend/tests/test_geocoder_cadastral_matcher.py

@@ -178,7 +178,6 @@ async def test_geocode_uses_house_match_before_legacy_forward() -> None:
 
     with (
         patch("app.services.geocoder._cache_get", return_value=None),
-        patch("app.services.geocoder._geoportal_house_match", return_value=None),
         patch(
             "app.services.geocoder._cadastral_house_match",
             return_value=hit,
@@ -214,7 +213,6 @@ async def test_geocode_falls_back_to_legacy_forward_when_house_match_misses() ->
 
     with (
         patch("app.services.geocoder._cache_get", return_value=None),
-        patch("app.services.geocoder._geoportal_house_match", return_value=None),
         patch(
             "app.services.geocoder._cadastral_house_match",
             return_value=None,